diff --git a/.agents/skills/instrumentation/SKILL.md b/.agents/skills/instrumentation/SKILL.md index 42fcbeeb6..c59a7859b 100644 --- a/.agents/skills/instrumentation/SKILL.md +++ b/.agents/skills/instrumentation/SKILL.md @@ -35,6 +35,7 @@ Map the change before editing: - Promise/stream behavior must be preserved. Patches need to keep subclass/helper semantics intact. - Contain instrumentation failures. Extraction/logging bugs should be logged or ignored as appropriate, but must not break the user call path. - Log only the useful surface. Prefer narrow, stable payloads over dumping full request/response objects; exclude redundant or overly large data when possible. +- We want to limit our instrumentation to operations that are relevant for AI generations and operations (LLMs, embeddings, media generation, ...). Things like creating entities on platforms (CRUD for Workflows of Agent entities) is irrelevant to us. - When building instrumentation, we should always have a vendored type/interface for what we are wrapping. The type or interface should not be larger than what is relevant to the instrumentation. The type or interface should be used for typing tracing channels and also should be used to assert the type on whatever is passed into wrappers as soon as the wrapper has verified that the passed in value is plausibly what should be wrapped. ## Process diff --git a/.env.example b/.env.example index 2fc0f5f70..7ade92540 100644 --- a/.env.example +++ b/.env.example @@ -1,4 +1,6 @@ - -BRAINTRUST_API_KEY=your_braintrust_api_key -OPENAI_API_KEY=your_openai_api_key -ANTHROPIC_API_KEY=your_anthropic_key +BRAINTRUST_API_KEY= +OPENAI_API_KEY= +ANTHROPIC_API_KEY= +GEMINI_API_KEY= +OPENROUTER_API_KEY= +MISTRAL_API_KEY= diff --git a/.github/workflows/e2e-canary.yaml b/.github/workflows/e2e-canary.yaml index 71266780e..0aa472263 100644 --- a/.github/workflows/e2e-canary.yaml +++ b/.github/workflows/e2e-canary.yaml @@ -37,6 +37,7 @@ jobs: GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }} OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} + MISTRAL_API_KEY: ${{ secrets.MISTRAL_API_KEY }} run: pnpm test:e2e:canary - name: Create or update nightly canary issue if: ${{ failure() && github.event_name == 'schedule' }} diff --git a/.github/workflows/integration-tests.yaml b/.github/workflows/integration-tests.yaml index 44119c35a..120301dd7 100644 --- a/.github/workflows/integration-tests.yaml +++ b/.github/workflows/integration-tests.yaml @@ -30,6 +30,7 @@ jobs: OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }} OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} + MISTRAL_API_KEY: ${{ secrets.MISTRAL_API_KEY }} steps: - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0 @@ -57,6 +58,7 @@ jobs: GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }} OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} + MISTRAL_API_KEY: ${{ secrets.MISTRAL_API_KEY }} steps: - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 - uses: denoland/setup-deno@ff4860f9d7236f320afa0f82b7e6457384805d05 # v2.0.4 @@ -105,6 +107,7 @@ jobs: GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }} OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} + MISTRAL_API_KEY: ${{ secrets.MISTRAL_API_KEY }} steps: - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0 diff --git a/.vscode/settings.json b/.vscode/settings.json index 52877f4d9..c0693da20 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -1,4 +1,4 @@ { - "typescript.tsdk": "js/node_modules/typescript/lib", - "typescript.enablePromptUseWorkspaceTsdk": false + "js/ts.tsdk.promptToUseWorkspaceVersion": false, + "js/ts.tsdk.path": "js/node_modules/typescript/lib" } diff --git a/e2e/README.md b/e2e/README.md index d9d56751d..3ec5a95f7 100644 --- a/e2e/README.md +++ b/e2e/README.md @@ -116,6 +116,7 @@ Non-hermetic scenarios require provider credentials in addition to the mock Brai - `ANTHROPIC_API_KEY` - `GEMINI_API_KEY` or `GOOGLE_API_KEY` - `OPENROUTER_API_KEY` +- `MISTRAL_API_KEY` `claude-agent-sdk-instrumentation` also uses `ANTHROPIC_API_KEY`, because it runs the real Claude Agent SDK against Anthropic in the same style as the existing live Anthropic wrapper coverage. diff --git a/e2e/config/pr-comment-scenarios.json b/e2e/config/pr-comment-scenarios.json index 02435c1ed..94a29964f 100644 --- a/e2e/config/pr-comment-scenarios.json +++ b/e2e/config/pr-comment-scenarios.json @@ -33,6 +33,19 @@ { "variantKey": "google-genai-v1460", "label": "v1.46.0" } ] }, + { + "scenarioDirName": "mistral-instrumentation", + "label": "Mistral Instrumentation", + "metadataScenario": "mistral-instrumentation", + "variants": [ + { "variantKey": "mistral-v1-3-4", "label": "v1.3.4" }, + { "variantKey": "mistral-v1-10-0", "label": "v1.10.0" }, + { "variantKey": "mistral-v1-14-1", "label": "v1.14.1" }, + { "variantKey": "mistral-v1-15-1", "label": "v1.15.1" }, + { "variantKey": "mistral-v1", "label": "latest v1" }, + { "variantKey": "mistral-v2", "label": "latest v2" } + ] + }, { "scenarioDirName": "openrouter-instrumentation", "label": "OpenRouter Instrumentation", diff --git a/e2e/helpers/scenario-installer.ts b/e2e/helpers/scenario-installer.ts index 1d29fb4ac..bd387f701 100644 --- a/e2e/helpers/scenario-installer.ts +++ b/e2e/helpers/scenario-installer.ts @@ -26,6 +26,7 @@ const INSTALL_SECRET_ENV_VARS = [ "GH_TOKEN", "OPENAI_API_KEY", "OPENROUTER_API_KEY", + "MISTRAL_API_KEY", ] as const; const cleanupDirs = new Set(); diff --git a/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-10-0.log-payloads.json b/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-10-0.log-payloads.json new file mode 100644 index 000000000..c435f5788 --- /dev/null +++ b/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-10-0.log-payloads.json @@ -0,0 +1,421 @@ +[ + { + "has_input": false, + "has_output": false, + "input": null, + "metadata": { + "scenario": "mistral-instrumentation" + }, + "metric_keys": [], + "output": null, + "span_id": "" + }, + { + "has_input": false, + "has_output": false, + "input": null, + "metadata": { + "operation": "chat-complete" + }, + "metric_keys": [], + "output": null, + "span_id": "" + }, + { + "has_input": true, + "has_output": true, + "input": { + "item_count": 2, + "roles": [ + "system", + "user" + ], + "type": "messages" + }, + "metadata": { + "model": "mistral-small-latest", + "provider": "mistral" + }, + "metric_keys": [ + "completion_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "output": { + "choice_count": 1, + "finish_reason": "stop", + "has_content": true, + "role": "assistant", + "tool_call_count": 0, + "type": "choices" + }, + "span_id": "" + }, + { + "has_input": false, + "has_output": false, + "input": null, + "metadata": { + "operation": "chat-stream" + }, + "metric_keys": [], + "output": null, + "span_id": "" + }, + { + "has_input": true, + "has_output": true, + "input": { + "item_count": 1, + "roles": [ + "user" + ], + "type": "messages" + }, + "metadata": { + "model": "mistral-small-latest", + "provider": "mistral" + }, + "metric_keys": [ + "completion_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "output": { + "choice_count": 1, + "finish_reason": "stop", + "has_content": true, + "role": "assistant", + "tool_call_count": 0, + "type": "choices" + }, + "span_id": "" + }, + { + "has_input": false, + "has_output": false, + "input": null, + "metadata": { + "operation": "chat-tool-call" + }, + "metric_keys": [], + "output": null, + "span_id": "" + }, + { + "has_input": true, + "has_output": true, + "input": { + "item_count": 1, + "roles": [ + "user" + ], + "type": "messages" + }, + "metadata": { + "model": "mistral-small-latest", + "provider": "mistral" + }, + "metric_keys": [ + "completion_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "output": { + "choice_count": 1, + "finish_reason": "tool_calls", + "has_content": false, + "role": "assistant", + "tool_call_count": 1, + "type": "choices" + }, + "span_id": "" + }, + { + "has_input": true, + "has_output": true, + "input": { + "item_count": 1, + "roles": [ + "user" + ], + "type": "messages" + }, + "metadata": { + "model": "mistral-small-latest", + "provider": "mistral" + }, + "metric_keys": [ + "completion_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "output": { + "choice_count": 1, + "finish_reason": "tool_calls", + "has_content": false, + "role": "assistant", + "tool_call_count": 1, + "type": "choices" + }, + "span_id": "" + }, + { + "has_input": true, + "has_output": true, + "input": { + "item_count": 2, + "roles": [ + "system", + "user" + ], + "type": "messages" + }, + "metadata": { + "model": "mistral-small-latest", + "provider": "mistral" + }, + "metric_keys": [ + "completion_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "output": { + "choice_count": 1, + "finish_reason": "tool_calls", + "has_content": false, + "role": "assistant", + "tool_call_count": 2, + "type": "choices" + }, + "span_id": "" + }, + { + "has_input": false, + "has_output": false, + "input": null, + "metadata": { + "operation": "fim-complete" + }, + "metric_keys": [], + "output": null, + "span_id": "" + }, + { + "has_input": true, + "has_output": true, + "input": "", + "metadata": { + "model": "codestral-latest", + "provider": "mistral" + }, + "metric_keys": [ + "completion_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "output": { + "choice_count": 1, + "finish_reason": "stop", + "has_content": true, + "role": "assistant", + "tool_call_count": 0, + "type": "choices" + }, + "span_id": "" + }, + { + "has_input": false, + "has_output": false, + "input": null, + "metadata": { + "operation": "fim-stream" + }, + "metric_keys": [], + "output": null, + "span_id": "" + }, + { + "has_input": true, + "has_output": true, + "input": "", + "metadata": { + "model": "codestral-latest", + "provider": "mistral" + }, + "metric_keys": [ + "completion_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "output": { + "choice_count": 1, + "finish_reason": "length", + "has_content": true, + "role": "assistant", + "tool_call_count": 0, + "type": "choices" + }, + "span_id": "" + }, + { + "has_input": false, + "has_output": false, + "input": null, + "metadata": { + "operation": "agents-complete" + }, + "metric_keys": [], + "output": null, + "span_id": "" + }, + { + "has_input": true, + "has_output": true, + "input": { + "item_count": 1, + "roles": [ + "user" + ], + "type": "messages" + }, + "metadata": { + "model": "mistral-small-latest", + "provider": "mistral" + }, + "metric_keys": [ + "completion_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "output": { + "choice_count": 1, + "finish_reason": "stop", + "has_content": true, + "role": "assistant", + "tool_call_count": 0, + "type": "choices" + }, + "span_id": "" + }, + { + "has_input": false, + "has_output": false, + "input": null, + "metadata": { + "operation": "agents-tool-call" + }, + "metric_keys": [], + "output": null, + "span_id": "" + }, + { + "has_input": true, + "has_output": true, + "input": { + "item_count": 1, + "roles": [ + "user" + ], + "type": "messages" + }, + "metadata": { + "model": "mistral-small-latest", + "provider": "mistral" + }, + "metric_keys": [ + "completion_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "output": { + "choice_count": 1, + "finish_reason": "tool_calls", + "has_content": false, + "role": "assistant", + "tool_call_count": 1, + "type": "choices" + }, + "span_id": "" + }, + { + "has_input": false, + "has_output": false, + "input": null, + "metadata": { + "operation": "agents-stream" + }, + "metric_keys": [], + "output": null, + "span_id": "" + }, + { + "has_input": true, + "has_output": true, + "input": { + "item_count": 1, + "roles": [ + "user" + ], + "type": "messages" + }, + "metadata": { + "model": "mistral-small-latest", + "provider": "mistral" + }, + "metric_keys": [ + "completion_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "output": { + "choice_count": 1, + "finish_reason": "stop", + "has_content": true, + "role": "assistant", + "tool_call_count": 0, + "type": "choices" + }, + "span_id": "" + }, + { + "has_input": false, + "has_output": false, + "input": null, + "metadata": { + "operation": "embeddings-create" + }, + "metric_keys": [], + "output": null, + "span_id": "" + }, + { + "has_input": true, + "has_output": true, + "input": "", + "metadata": { + "model": "mistral-embed", + "provider": "mistral" + }, + "metric_keys": [ + "completion_tokens", + "prompt_tokens", + "tokens" + ], + "output": { + "embedding_length": "", + "type": "embedding" + }, + "span_id": "" + } +] diff --git a/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-10-0.span-events.json b/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-10-0.span-events.json new file mode 100644 index 000000000..b7d2919bc --- /dev/null +++ b/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-10-0.span-events.json @@ -0,0 +1,380 @@ +[ + { + "has_input": false, + "has_output": false, + "metadata": { + "scenario": "mistral-instrumentation" + }, + "metric_keys": [], + "name": "mistral-root", + "root_span_id": "", + "span_id": "", + "span_parents": [], + "type": "task" + }, + { + "has_input": false, + "has_output": false, + "metadata": { + "operation": "chat-complete" + }, + "metric_keys": [], + "name": "mistral-chat-complete-operation", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": null + }, + { + "has_input": true, + "has_output": true, + "metadata": { + "model": "mistral-small-latest", + "provider": "mistral" + }, + "metric_keys": [ + "completion_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "name": "mistral.chat.complete", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": "llm" + }, + { + "has_input": false, + "has_output": false, + "metadata": { + "operation": "chat-stream" + }, + "metric_keys": [], + "name": "mistral-chat-stream-operation", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": null + }, + { + "has_input": true, + "has_output": true, + "metadata": { + "model": "mistral-small-latest", + "provider": "mistral" + }, + "metric_keys": [ + "completion_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "name": "mistral.chat.stream", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": "llm" + }, + { + "has_input": false, + "has_output": false, + "metadata": { + "operation": "chat-tool-call" + }, + "metric_keys": [], + "name": "mistral-chat-tool-call-operation", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": null + }, + { + "has_input": true, + "has_output": true, + "metadata": { + "model": "mistral-small-latest", + "provider": "mistral" + }, + "metric_keys": [ + "completion_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "name": "mistral.chat.complete", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": "llm" + }, + { + "has_input": true, + "has_output": true, + "metadata": { + "model": "mistral-small-latest", + "provider": "mistral" + }, + "metric_keys": [ + "completion_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "name": "mistral.chat.complete", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": "llm" + }, + { + "has_input": true, + "has_output": true, + "metadata": { + "model": "mistral-small-latest", + "provider": "mistral" + }, + "metric_keys": [ + "completion_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "name": "mistral.chat.complete", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": "llm" + }, + { + "has_input": false, + "has_output": false, + "metadata": { + "operation": "fim-complete" + }, + "metric_keys": [], + "name": "mistral-fim-complete-operation", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": null + }, + { + "has_input": true, + "has_output": true, + "metadata": { + "model": "codestral-latest", + "provider": "mistral" + }, + "metric_keys": [ + "completion_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "name": "mistral.fim.complete", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": "llm" + }, + { + "has_input": false, + "has_output": false, + "metadata": { + "operation": "fim-stream" + }, + "metric_keys": [], + "name": "mistral-fim-stream-operation", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": null + }, + { + "has_input": true, + "has_output": true, + "metadata": { + "model": "codestral-latest", + "provider": "mistral" + }, + "metric_keys": [ + "completion_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "name": "mistral.fim.stream", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": "llm" + }, + { + "has_input": false, + "has_output": false, + "metadata": { + "operation": "agents-complete" + }, + "metric_keys": [], + "name": "mistral-agents-complete-operation", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": null + }, + { + "has_input": true, + "has_output": true, + "metadata": { + "model": "mistral-small-latest", + "provider": "mistral" + }, + "metric_keys": [ + "completion_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "name": "mistral.agents.complete", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": "llm" + }, + { + "has_input": false, + "has_output": false, + "metadata": { + "operation": "agents-tool-call" + }, + "metric_keys": [], + "name": "mistral-agents-tool-call-operation", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": null + }, + { + "has_input": true, + "has_output": true, + "metadata": { + "model": "mistral-small-latest", + "provider": "mistral" + }, + "metric_keys": [ + "completion_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "name": "mistral.agents.complete", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": "llm" + }, + { + "has_input": false, + "has_output": false, + "metadata": { + "operation": "agents-stream" + }, + "metric_keys": [], + "name": "mistral-agents-stream-operation", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": null + }, + { + "has_input": true, + "has_output": true, + "metadata": { + "model": "mistral-small-latest", + "provider": "mistral" + }, + "metric_keys": [ + "completion_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "name": "mistral.agents.stream", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": "llm" + }, + { + "has_input": false, + "has_output": false, + "metadata": { + "operation": "embeddings-create" + }, + "metric_keys": [], + "name": "mistral-embeddings-operation", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": null + }, + { + "has_input": true, + "has_output": true, + "metadata": { + "model": "mistral-embed", + "provider": "mistral" + }, + "metric_keys": [ + "completion_tokens", + "prompt_tokens", + "tokens" + ], + "name": "mistral.embeddings.create", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": "llm" + } +] diff --git a/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-14-1.log-payloads.json b/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-14-1.log-payloads.json new file mode 100644 index 000000000..c435f5788 --- /dev/null +++ b/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-14-1.log-payloads.json @@ -0,0 +1,421 @@ +[ + { + "has_input": false, + "has_output": false, + "input": null, + "metadata": { + "scenario": "mistral-instrumentation" + }, + "metric_keys": [], + "output": null, + "span_id": "" + }, + { + "has_input": false, + "has_output": false, + "input": null, + "metadata": { + "operation": "chat-complete" + }, + "metric_keys": [], + "output": null, + "span_id": "" + }, + { + "has_input": true, + "has_output": true, + "input": { + "item_count": 2, + "roles": [ + "system", + "user" + ], + "type": "messages" + }, + "metadata": { + "model": "mistral-small-latest", + "provider": "mistral" + }, + "metric_keys": [ + "completion_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "output": { + "choice_count": 1, + "finish_reason": "stop", + "has_content": true, + "role": "assistant", + "tool_call_count": 0, + "type": "choices" + }, + "span_id": "" + }, + { + "has_input": false, + "has_output": false, + "input": null, + "metadata": { + "operation": "chat-stream" + }, + "metric_keys": [], + "output": null, + "span_id": "" + }, + { + "has_input": true, + "has_output": true, + "input": { + "item_count": 1, + "roles": [ + "user" + ], + "type": "messages" + }, + "metadata": { + "model": "mistral-small-latest", + "provider": "mistral" + }, + "metric_keys": [ + "completion_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "output": { + "choice_count": 1, + "finish_reason": "stop", + "has_content": true, + "role": "assistant", + "tool_call_count": 0, + "type": "choices" + }, + "span_id": "" + }, + { + "has_input": false, + "has_output": false, + "input": null, + "metadata": { + "operation": "chat-tool-call" + }, + "metric_keys": [], + "output": null, + "span_id": "" + }, + { + "has_input": true, + "has_output": true, + "input": { + "item_count": 1, + "roles": [ + "user" + ], + "type": "messages" + }, + "metadata": { + "model": "mistral-small-latest", + "provider": "mistral" + }, + "metric_keys": [ + "completion_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "output": { + "choice_count": 1, + "finish_reason": "tool_calls", + "has_content": false, + "role": "assistant", + "tool_call_count": 1, + "type": "choices" + }, + "span_id": "" + }, + { + "has_input": true, + "has_output": true, + "input": { + "item_count": 1, + "roles": [ + "user" + ], + "type": "messages" + }, + "metadata": { + "model": "mistral-small-latest", + "provider": "mistral" + }, + "metric_keys": [ + "completion_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "output": { + "choice_count": 1, + "finish_reason": "tool_calls", + "has_content": false, + "role": "assistant", + "tool_call_count": 1, + "type": "choices" + }, + "span_id": "" + }, + { + "has_input": true, + "has_output": true, + "input": { + "item_count": 2, + "roles": [ + "system", + "user" + ], + "type": "messages" + }, + "metadata": { + "model": "mistral-small-latest", + "provider": "mistral" + }, + "metric_keys": [ + "completion_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "output": { + "choice_count": 1, + "finish_reason": "tool_calls", + "has_content": false, + "role": "assistant", + "tool_call_count": 2, + "type": "choices" + }, + "span_id": "" + }, + { + "has_input": false, + "has_output": false, + "input": null, + "metadata": { + "operation": "fim-complete" + }, + "metric_keys": [], + "output": null, + "span_id": "" + }, + { + "has_input": true, + "has_output": true, + "input": "", + "metadata": { + "model": "codestral-latest", + "provider": "mistral" + }, + "metric_keys": [ + "completion_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "output": { + "choice_count": 1, + "finish_reason": "stop", + "has_content": true, + "role": "assistant", + "tool_call_count": 0, + "type": "choices" + }, + "span_id": "" + }, + { + "has_input": false, + "has_output": false, + "input": null, + "metadata": { + "operation": "fim-stream" + }, + "metric_keys": [], + "output": null, + "span_id": "" + }, + { + "has_input": true, + "has_output": true, + "input": "", + "metadata": { + "model": "codestral-latest", + "provider": "mistral" + }, + "metric_keys": [ + "completion_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "output": { + "choice_count": 1, + "finish_reason": "length", + "has_content": true, + "role": "assistant", + "tool_call_count": 0, + "type": "choices" + }, + "span_id": "" + }, + { + "has_input": false, + "has_output": false, + "input": null, + "metadata": { + "operation": "agents-complete" + }, + "metric_keys": [], + "output": null, + "span_id": "" + }, + { + "has_input": true, + "has_output": true, + "input": { + "item_count": 1, + "roles": [ + "user" + ], + "type": "messages" + }, + "metadata": { + "model": "mistral-small-latest", + "provider": "mistral" + }, + "metric_keys": [ + "completion_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "output": { + "choice_count": 1, + "finish_reason": "stop", + "has_content": true, + "role": "assistant", + "tool_call_count": 0, + "type": "choices" + }, + "span_id": "" + }, + { + "has_input": false, + "has_output": false, + "input": null, + "metadata": { + "operation": "agents-tool-call" + }, + "metric_keys": [], + "output": null, + "span_id": "" + }, + { + "has_input": true, + "has_output": true, + "input": { + "item_count": 1, + "roles": [ + "user" + ], + "type": "messages" + }, + "metadata": { + "model": "mistral-small-latest", + "provider": "mistral" + }, + "metric_keys": [ + "completion_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "output": { + "choice_count": 1, + "finish_reason": "tool_calls", + "has_content": false, + "role": "assistant", + "tool_call_count": 1, + "type": "choices" + }, + "span_id": "" + }, + { + "has_input": false, + "has_output": false, + "input": null, + "metadata": { + "operation": "agents-stream" + }, + "metric_keys": [], + "output": null, + "span_id": "" + }, + { + "has_input": true, + "has_output": true, + "input": { + "item_count": 1, + "roles": [ + "user" + ], + "type": "messages" + }, + "metadata": { + "model": "mistral-small-latest", + "provider": "mistral" + }, + "metric_keys": [ + "completion_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "output": { + "choice_count": 1, + "finish_reason": "stop", + "has_content": true, + "role": "assistant", + "tool_call_count": 0, + "type": "choices" + }, + "span_id": "" + }, + { + "has_input": false, + "has_output": false, + "input": null, + "metadata": { + "operation": "embeddings-create" + }, + "metric_keys": [], + "output": null, + "span_id": "" + }, + { + "has_input": true, + "has_output": true, + "input": "", + "metadata": { + "model": "mistral-embed", + "provider": "mistral" + }, + "metric_keys": [ + "completion_tokens", + "prompt_tokens", + "tokens" + ], + "output": { + "embedding_length": "", + "type": "embedding" + }, + "span_id": "" + } +] diff --git a/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-14-1.span-events.json b/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-14-1.span-events.json new file mode 100644 index 000000000..b7d2919bc --- /dev/null +++ b/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-14-1.span-events.json @@ -0,0 +1,380 @@ +[ + { + "has_input": false, + "has_output": false, + "metadata": { + "scenario": "mistral-instrumentation" + }, + "metric_keys": [], + "name": "mistral-root", + "root_span_id": "", + "span_id": "", + "span_parents": [], + "type": "task" + }, + { + "has_input": false, + "has_output": false, + "metadata": { + "operation": "chat-complete" + }, + "metric_keys": [], + "name": "mistral-chat-complete-operation", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": null + }, + { + "has_input": true, + "has_output": true, + "metadata": { + "model": "mistral-small-latest", + "provider": "mistral" + }, + "metric_keys": [ + "completion_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "name": "mistral.chat.complete", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": "llm" + }, + { + "has_input": false, + "has_output": false, + "metadata": { + "operation": "chat-stream" + }, + "metric_keys": [], + "name": "mistral-chat-stream-operation", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": null + }, + { + "has_input": true, + "has_output": true, + "metadata": { + "model": "mistral-small-latest", + "provider": "mistral" + }, + "metric_keys": [ + "completion_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "name": "mistral.chat.stream", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": "llm" + }, + { + "has_input": false, + "has_output": false, + "metadata": { + "operation": "chat-tool-call" + }, + "metric_keys": [], + "name": "mistral-chat-tool-call-operation", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": null + }, + { + "has_input": true, + "has_output": true, + "metadata": { + "model": "mistral-small-latest", + "provider": "mistral" + }, + "metric_keys": [ + "completion_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "name": "mistral.chat.complete", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": "llm" + }, + { + "has_input": true, + "has_output": true, + "metadata": { + "model": "mistral-small-latest", + "provider": "mistral" + }, + "metric_keys": [ + "completion_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "name": "mistral.chat.complete", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": "llm" + }, + { + "has_input": true, + "has_output": true, + "metadata": { + "model": "mistral-small-latest", + "provider": "mistral" + }, + "metric_keys": [ + "completion_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "name": "mistral.chat.complete", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": "llm" + }, + { + "has_input": false, + "has_output": false, + "metadata": { + "operation": "fim-complete" + }, + "metric_keys": [], + "name": "mistral-fim-complete-operation", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": null + }, + { + "has_input": true, + "has_output": true, + "metadata": { + "model": "codestral-latest", + "provider": "mistral" + }, + "metric_keys": [ + "completion_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "name": "mistral.fim.complete", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": "llm" + }, + { + "has_input": false, + "has_output": false, + "metadata": { + "operation": "fim-stream" + }, + "metric_keys": [], + "name": "mistral-fim-stream-operation", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": null + }, + { + "has_input": true, + "has_output": true, + "metadata": { + "model": "codestral-latest", + "provider": "mistral" + }, + "metric_keys": [ + "completion_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "name": "mistral.fim.stream", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": "llm" + }, + { + "has_input": false, + "has_output": false, + "metadata": { + "operation": "agents-complete" + }, + "metric_keys": [], + "name": "mistral-agents-complete-operation", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": null + }, + { + "has_input": true, + "has_output": true, + "metadata": { + "model": "mistral-small-latest", + "provider": "mistral" + }, + "metric_keys": [ + "completion_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "name": "mistral.agents.complete", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": "llm" + }, + { + "has_input": false, + "has_output": false, + "metadata": { + "operation": "agents-tool-call" + }, + "metric_keys": [], + "name": "mistral-agents-tool-call-operation", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": null + }, + { + "has_input": true, + "has_output": true, + "metadata": { + "model": "mistral-small-latest", + "provider": "mistral" + }, + "metric_keys": [ + "completion_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "name": "mistral.agents.complete", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": "llm" + }, + { + "has_input": false, + "has_output": false, + "metadata": { + "operation": "agents-stream" + }, + "metric_keys": [], + "name": "mistral-agents-stream-operation", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": null + }, + { + "has_input": true, + "has_output": true, + "metadata": { + "model": "mistral-small-latest", + "provider": "mistral" + }, + "metric_keys": [ + "completion_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "name": "mistral.agents.stream", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": "llm" + }, + { + "has_input": false, + "has_output": false, + "metadata": { + "operation": "embeddings-create" + }, + "metric_keys": [], + "name": "mistral-embeddings-operation", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": null + }, + { + "has_input": true, + "has_output": true, + "metadata": { + "model": "mistral-embed", + "provider": "mistral" + }, + "metric_keys": [ + "completion_tokens", + "prompt_tokens", + "tokens" + ], + "name": "mistral.embeddings.create", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": "llm" + } +] diff --git a/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-15-1.log-payloads.json b/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-15-1.log-payloads.json new file mode 100644 index 000000000..c435f5788 --- /dev/null +++ b/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-15-1.log-payloads.json @@ -0,0 +1,421 @@ +[ + { + "has_input": false, + "has_output": false, + "input": null, + "metadata": { + "scenario": "mistral-instrumentation" + }, + "metric_keys": [], + "output": null, + "span_id": "" + }, + { + "has_input": false, + "has_output": false, + "input": null, + "metadata": { + "operation": "chat-complete" + }, + "metric_keys": [], + "output": null, + "span_id": "" + }, + { + "has_input": true, + "has_output": true, + "input": { + "item_count": 2, + "roles": [ + "system", + "user" + ], + "type": "messages" + }, + "metadata": { + "model": "mistral-small-latest", + "provider": "mistral" + }, + "metric_keys": [ + "completion_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "output": { + "choice_count": 1, + "finish_reason": "stop", + "has_content": true, + "role": "assistant", + "tool_call_count": 0, + "type": "choices" + }, + "span_id": "" + }, + { + "has_input": false, + "has_output": false, + "input": null, + "metadata": { + "operation": "chat-stream" + }, + "metric_keys": [], + "output": null, + "span_id": "" + }, + { + "has_input": true, + "has_output": true, + "input": { + "item_count": 1, + "roles": [ + "user" + ], + "type": "messages" + }, + "metadata": { + "model": "mistral-small-latest", + "provider": "mistral" + }, + "metric_keys": [ + "completion_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "output": { + "choice_count": 1, + "finish_reason": "stop", + "has_content": true, + "role": "assistant", + "tool_call_count": 0, + "type": "choices" + }, + "span_id": "" + }, + { + "has_input": false, + "has_output": false, + "input": null, + "metadata": { + "operation": "chat-tool-call" + }, + "metric_keys": [], + "output": null, + "span_id": "" + }, + { + "has_input": true, + "has_output": true, + "input": { + "item_count": 1, + "roles": [ + "user" + ], + "type": "messages" + }, + "metadata": { + "model": "mistral-small-latest", + "provider": "mistral" + }, + "metric_keys": [ + "completion_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "output": { + "choice_count": 1, + "finish_reason": "tool_calls", + "has_content": false, + "role": "assistant", + "tool_call_count": 1, + "type": "choices" + }, + "span_id": "" + }, + { + "has_input": true, + "has_output": true, + "input": { + "item_count": 1, + "roles": [ + "user" + ], + "type": "messages" + }, + "metadata": { + "model": "mistral-small-latest", + "provider": "mistral" + }, + "metric_keys": [ + "completion_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "output": { + "choice_count": 1, + "finish_reason": "tool_calls", + "has_content": false, + "role": "assistant", + "tool_call_count": 1, + "type": "choices" + }, + "span_id": "" + }, + { + "has_input": true, + "has_output": true, + "input": { + "item_count": 2, + "roles": [ + "system", + "user" + ], + "type": "messages" + }, + "metadata": { + "model": "mistral-small-latest", + "provider": "mistral" + }, + "metric_keys": [ + "completion_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "output": { + "choice_count": 1, + "finish_reason": "tool_calls", + "has_content": false, + "role": "assistant", + "tool_call_count": 2, + "type": "choices" + }, + "span_id": "" + }, + { + "has_input": false, + "has_output": false, + "input": null, + "metadata": { + "operation": "fim-complete" + }, + "metric_keys": [], + "output": null, + "span_id": "" + }, + { + "has_input": true, + "has_output": true, + "input": "", + "metadata": { + "model": "codestral-latest", + "provider": "mistral" + }, + "metric_keys": [ + "completion_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "output": { + "choice_count": 1, + "finish_reason": "stop", + "has_content": true, + "role": "assistant", + "tool_call_count": 0, + "type": "choices" + }, + "span_id": "" + }, + { + "has_input": false, + "has_output": false, + "input": null, + "metadata": { + "operation": "fim-stream" + }, + "metric_keys": [], + "output": null, + "span_id": "" + }, + { + "has_input": true, + "has_output": true, + "input": "", + "metadata": { + "model": "codestral-latest", + "provider": "mistral" + }, + "metric_keys": [ + "completion_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "output": { + "choice_count": 1, + "finish_reason": "length", + "has_content": true, + "role": "assistant", + "tool_call_count": 0, + "type": "choices" + }, + "span_id": "" + }, + { + "has_input": false, + "has_output": false, + "input": null, + "metadata": { + "operation": "agents-complete" + }, + "metric_keys": [], + "output": null, + "span_id": "" + }, + { + "has_input": true, + "has_output": true, + "input": { + "item_count": 1, + "roles": [ + "user" + ], + "type": "messages" + }, + "metadata": { + "model": "mistral-small-latest", + "provider": "mistral" + }, + "metric_keys": [ + "completion_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "output": { + "choice_count": 1, + "finish_reason": "stop", + "has_content": true, + "role": "assistant", + "tool_call_count": 0, + "type": "choices" + }, + "span_id": "" + }, + { + "has_input": false, + "has_output": false, + "input": null, + "metadata": { + "operation": "agents-tool-call" + }, + "metric_keys": [], + "output": null, + "span_id": "" + }, + { + "has_input": true, + "has_output": true, + "input": { + "item_count": 1, + "roles": [ + "user" + ], + "type": "messages" + }, + "metadata": { + "model": "mistral-small-latest", + "provider": "mistral" + }, + "metric_keys": [ + "completion_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "output": { + "choice_count": 1, + "finish_reason": "tool_calls", + "has_content": false, + "role": "assistant", + "tool_call_count": 1, + "type": "choices" + }, + "span_id": "" + }, + { + "has_input": false, + "has_output": false, + "input": null, + "metadata": { + "operation": "agents-stream" + }, + "metric_keys": [], + "output": null, + "span_id": "" + }, + { + "has_input": true, + "has_output": true, + "input": { + "item_count": 1, + "roles": [ + "user" + ], + "type": "messages" + }, + "metadata": { + "model": "mistral-small-latest", + "provider": "mistral" + }, + "metric_keys": [ + "completion_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "output": { + "choice_count": 1, + "finish_reason": "stop", + "has_content": true, + "role": "assistant", + "tool_call_count": 0, + "type": "choices" + }, + "span_id": "" + }, + { + "has_input": false, + "has_output": false, + "input": null, + "metadata": { + "operation": "embeddings-create" + }, + "metric_keys": [], + "output": null, + "span_id": "" + }, + { + "has_input": true, + "has_output": true, + "input": "", + "metadata": { + "model": "mistral-embed", + "provider": "mistral" + }, + "metric_keys": [ + "completion_tokens", + "prompt_tokens", + "tokens" + ], + "output": { + "embedding_length": "", + "type": "embedding" + }, + "span_id": "" + } +] diff --git a/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-15-1.span-events.json b/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-15-1.span-events.json new file mode 100644 index 000000000..b7d2919bc --- /dev/null +++ b/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-15-1.span-events.json @@ -0,0 +1,380 @@ +[ + { + "has_input": false, + "has_output": false, + "metadata": { + "scenario": "mistral-instrumentation" + }, + "metric_keys": [], + "name": "mistral-root", + "root_span_id": "", + "span_id": "", + "span_parents": [], + "type": "task" + }, + { + "has_input": false, + "has_output": false, + "metadata": { + "operation": "chat-complete" + }, + "metric_keys": [], + "name": "mistral-chat-complete-operation", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": null + }, + { + "has_input": true, + "has_output": true, + "metadata": { + "model": "mistral-small-latest", + "provider": "mistral" + }, + "metric_keys": [ + "completion_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "name": "mistral.chat.complete", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": "llm" + }, + { + "has_input": false, + "has_output": false, + "metadata": { + "operation": "chat-stream" + }, + "metric_keys": [], + "name": "mistral-chat-stream-operation", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": null + }, + { + "has_input": true, + "has_output": true, + "metadata": { + "model": "mistral-small-latest", + "provider": "mistral" + }, + "metric_keys": [ + "completion_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "name": "mistral.chat.stream", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": "llm" + }, + { + "has_input": false, + "has_output": false, + "metadata": { + "operation": "chat-tool-call" + }, + "metric_keys": [], + "name": "mistral-chat-tool-call-operation", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": null + }, + { + "has_input": true, + "has_output": true, + "metadata": { + "model": "mistral-small-latest", + "provider": "mistral" + }, + "metric_keys": [ + "completion_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "name": "mistral.chat.complete", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": "llm" + }, + { + "has_input": true, + "has_output": true, + "metadata": { + "model": "mistral-small-latest", + "provider": "mistral" + }, + "metric_keys": [ + "completion_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "name": "mistral.chat.complete", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": "llm" + }, + { + "has_input": true, + "has_output": true, + "metadata": { + "model": "mistral-small-latest", + "provider": "mistral" + }, + "metric_keys": [ + "completion_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "name": "mistral.chat.complete", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": "llm" + }, + { + "has_input": false, + "has_output": false, + "metadata": { + "operation": "fim-complete" + }, + "metric_keys": [], + "name": "mistral-fim-complete-operation", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": null + }, + { + "has_input": true, + "has_output": true, + "metadata": { + "model": "codestral-latest", + "provider": "mistral" + }, + "metric_keys": [ + "completion_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "name": "mistral.fim.complete", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": "llm" + }, + { + "has_input": false, + "has_output": false, + "metadata": { + "operation": "fim-stream" + }, + "metric_keys": [], + "name": "mistral-fim-stream-operation", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": null + }, + { + "has_input": true, + "has_output": true, + "metadata": { + "model": "codestral-latest", + "provider": "mistral" + }, + "metric_keys": [ + "completion_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "name": "mistral.fim.stream", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": "llm" + }, + { + "has_input": false, + "has_output": false, + "metadata": { + "operation": "agents-complete" + }, + "metric_keys": [], + "name": "mistral-agents-complete-operation", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": null + }, + { + "has_input": true, + "has_output": true, + "metadata": { + "model": "mistral-small-latest", + "provider": "mistral" + }, + "metric_keys": [ + "completion_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "name": "mistral.agents.complete", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": "llm" + }, + { + "has_input": false, + "has_output": false, + "metadata": { + "operation": "agents-tool-call" + }, + "metric_keys": [], + "name": "mistral-agents-tool-call-operation", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": null + }, + { + "has_input": true, + "has_output": true, + "metadata": { + "model": "mistral-small-latest", + "provider": "mistral" + }, + "metric_keys": [ + "completion_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "name": "mistral.agents.complete", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": "llm" + }, + { + "has_input": false, + "has_output": false, + "metadata": { + "operation": "agents-stream" + }, + "metric_keys": [], + "name": "mistral-agents-stream-operation", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": null + }, + { + "has_input": true, + "has_output": true, + "metadata": { + "model": "mistral-small-latest", + "provider": "mistral" + }, + "metric_keys": [ + "completion_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "name": "mistral.agents.stream", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": "llm" + }, + { + "has_input": false, + "has_output": false, + "metadata": { + "operation": "embeddings-create" + }, + "metric_keys": [], + "name": "mistral-embeddings-operation", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": null + }, + { + "has_input": true, + "has_output": true, + "metadata": { + "model": "mistral-embed", + "provider": "mistral" + }, + "metric_keys": [ + "completion_tokens", + "prompt_tokens", + "tokens" + ], + "name": "mistral.embeddings.create", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": "llm" + } +] diff --git a/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-3-4.log-payloads.json b/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-3-4.log-payloads.json new file mode 100644 index 000000000..cf15e9eec --- /dev/null +++ b/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-3-4.log-payloads.json @@ -0,0 +1,418 @@ +[ + { + "has_input": false, + "has_output": false, + "input": null, + "metadata": { + "scenario": "mistral-instrumentation" + }, + "metric_keys": [], + "output": null, + "span_id": "" + }, + { + "has_input": false, + "has_output": false, + "input": null, + "metadata": { + "operation": "chat-complete" + }, + "metric_keys": [], + "output": null, + "span_id": "" + }, + { + "has_input": true, + "has_output": true, + "input": { + "item_count": 2, + "roles": [ + "system", + "user" + ], + "type": "messages" + }, + "metadata": { + "model": "mistral-small-latest", + "provider": "mistral" + }, + "metric_keys": [ + "completion_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "output": { + "choice_count": 1, + "finish_reason": "stop", + "has_content": true, + "role": "assistant", + "tool_call_count": 0, + "type": "choices" + }, + "span_id": "" + }, + { + "has_input": false, + "has_output": false, + "input": null, + "metadata": { + "operation": "chat-stream" + }, + "metric_keys": [], + "output": null, + "span_id": "" + }, + { + "has_input": true, + "has_output": true, + "input": { + "item_count": 1, + "roles": [ + "user" + ], + "type": "messages" + }, + "metadata": { + "model": "mistral-small-latest", + "provider": "mistral" + }, + "metric_keys": [ + "completion_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "output": { + "choice_count": 1, + "finish_reason": "stop", + "has_content": true, + "role": "assistant", + "tool_call_count": 0, + "type": "choices" + }, + "span_id": "" + }, + { + "has_input": false, + "has_output": false, + "input": null, + "metadata": { + "operation": "chat-tool-call" + }, + "metric_keys": [], + "output": null, + "span_id": "" + }, + { + "has_input": true, + "has_output": true, + "input": { + "item_count": 1, + "roles": [ + "user" + ], + "type": "messages" + }, + "metadata": { + "model": "mistral-small-latest", + "provider": "mistral" + }, + "metric_keys": [ + "completion_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "output": { + "choice_count": 1, + "finish_reason": "tool_calls", + "has_content": false, + "role": "assistant", + "tool_call_count": 1, + "type": "choices" + }, + "span_id": "" + }, + { + "has_input": true, + "has_output": true, + "input": { + "item_count": 1, + "roles": [ + "user" + ], + "type": "messages" + }, + "metadata": { + "model": "mistral-small-latest", + "provider": "mistral" + }, + "metric_keys": [ + "completion_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "output": { + "choice_count": 1, + "finish_reason": "tool_calls", + "has_content": false, + "role": "assistant", + "tool_call_count": 1, + "type": "choices" + }, + "span_id": "" + }, + { + "has_input": true, + "has_output": true, + "input": { + "item_count": 2, + "roles": [ + "system", + "user" + ], + "type": "messages" + }, + "metadata": { + "model": "mistral-small-latest", + "provider": "mistral" + }, + "metric_keys": [ + "completion_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "output": { + "choice_count": 1, + "finish_reason": "tool_calls", + "has_content": false, + "role": "assistant", + "tool_call_count": 2, + "type": "choices" + }, + "span_id": "" + }, + { + "has_input": false, + "has_output": false, + "input": null, + "metadata": { + "operation": "fim-complete" + }, + "metric_keys": [], + "output": null, + "span_id": "" + }, + { + "has_input": true, + "has_output": true, + "input": "", + "metadata": { + "model": "codestral-latest", + "provider": "mistral" + }, + "metric_keys": [ + "completion_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "output": { + "choice_count": 1, + "finish_reason": "stop", + "has_content": true, + "role": "assistant", + "tool_call_count": 0, + "type": "choices" + }, + "span_id": "" + }, + { + "has_input": false, + "has_output": false, + "input": null, + "metadata": { + "operation": "fim-stream" + }, + "metric_keys": [], + "output": null, + "span_id": "" + }, + { + "has_input": true, + "has_output": true, + "input": "", + "metadata": { + "model": "codestral-latest", + "provider": "mistral" + }, + "metric_keys": [ + "time_to_first_token" + ], + "output": { + "choice_count": 1, + "finish_reason": null, + "has_content": true, + "role": "assistant", + "tool_call_count": 0, + "type": "choices" + }, + "span_id": "" + }, + { + "has_input": false, + "has_output": false, + "input": null, + "metadata": { + "operation": "agents-complete" + }, + "metric_keys": [], + "output": null, + "span_id": "" + }, + { + "has_input": true, + "has_output": true, + "input": { + "item_count": 1, + "roles": [ + "user" + ], + "type": "messages" + }, + "metadata": { + "model": "mistral-small-latest", + "provider": "mistral" + }, + "metric_keys": [ + "completion_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "output": { + "choice_count": 1, + "finish_reason": "stop", + "has_content": true, + "role": "assistant", + "tool_call_count": 0, + "type": "choices" + }, + "span_id": "" + }, + { + "has_input": false, + "has_output": false, + "input": null, + "metadata": { + "operation": "agents-tool-call" + }, + "metric_keys": [], + "output": null, + "span_id": "" + }, + { + "has_input": true, + "has_output": true, + "input": { + "item_count": 1, + "roles": [ + "user" + ], + "type": "messages" + }, + "metadata": { + "model": "mistral-small-latest", + "provider": "mistral" + }, + "metric_keys": [ + "completion_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "output": { + "choice_count": 1, + "finish_reason": "tool_calls", + "has_content": false, + "role": "assistant", + "tool_call_count": 1, + "type": "choices" + }, + "span_id": "" + }, + { + "has_input": false, + "has_output": false, + "input": null, + "metadata": { + "operation": "agents-stream" + }, + "metric_keys": [], + "output": null, + "span_id": "" + }, + { + "has_input": true, + "has_output": true, + "input": { + "item_count": 1, + "roles": [ + "user" + ], + "type": "messages" + }, + "metadata": { + "model": "mistral-small-latest", + "provider": "mistral" + }, + "metric_keys": [ + "completion_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "output": { + "choice_count": 1, + "finish_reason": "stop", + "has_content": true, + "role": "assistant", + "tool_call_count": 0, + "type": "choices" + }, + "span_id": "" + }, + { + "has_input": false, + "has_output": false, + "input": null, + "metadata": { + "operation": "embeddings-create" + }, + "metric_keys": [], + "output": null, + "span_id": "" + }, + { + "has_input": true, + "has_output": true, + "input": "", + "metadata": { + "model": "mistral-embed", + "provider": "mistral" + }, + "metric_keys": [ + "completion_tokens", + "prompt_tokens", + "tokens" + ], + "output": { + "embedding_length": "", + "type": "embedding" + }, + "span_id": "" + } +] diff --git a/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-3-4.span-events.json b/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-3-4.span-events.json new file mode 100644 index 000000000..909d73766 --- /dev/null +++ b/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-3-4.span-events.json @@ -0,0 +1,377 @@ +[ + { + "has_input": false, + "has_output": false, + "metadata": { + "scenario": "mistral-instrumentation" + }, + "metric_keys": [], + "name": "mistral-root", + "root_span_id": "", + "span_id": "", + "span_parents": [], + "type": "task" + }, + { + "has_input": false, + "has_output": false, + "metadata": { + "operation": "chat-complete" + }, + "metric_keys": [], + "name": "mistral-chat-complete-operation", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": null + }, + { + "has_input": true, + "has_output": true, + "metadata": { + "model": "mistral-small-latest", + "provider": "mistral" + }, + "metric_keys": [ + "completion_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "name": "mistral.chat.complete", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": "llm" + }, + { + "has_input": false, + "has_output": false, + "metadata": { + "operation": "chat-stream" + }, + "metric_keys": [], + "name": "mistral-chat-stream-operation", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": null + }, + { + "has_input": true, + "has_output": true, + "metadata": { + "model": "mistral-small-latest", + "provider": "mistral" + }, + "metric_keys": [ + "completion_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "name": "mistral.chat.stream", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": "llm" + }, + { + "has_input": false, + "has_output": false, + "metadata": { + "operation": "chat-tool-call" + }, + "metric_keys": [], + "name": "mistral-chat-tool-call-operation", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": null + }, + { + "has_input": true, + "has_output": true, + "metadata": { + "model": "mistral-small-latest", + "provider": "mistral" + }, + "metric_keys": [ + "completion_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "name": "mistral.chat.complete", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": "llm" + }, + { + "has_input": true, + "has_output": true, + "metadata": { + "model": "mistral-small-latest", + "provider": "mistral" + }, + "metric_keys": [ + "completion_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "name": "mistral.chat.complete", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": "llm" + }, + { + "has_input": true, + "has_output": true, + "metadata": { + "model": "mistral-small-latest", + "provider": "mistral" + }, + "metric_keys": [ + "completion_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "name": "mistral.chat.complete", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": "llm" + }, + { + "has_input": false, + "has_output": false, + "metadata": { + "operation": "fim-complete" + }, + "metric_keys": [], + "name": "mistral-fim-complete-operation", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": null + }, + { + "has_input": true, + "has_output": true, + "metadata": { + "model": "codestral-latest", + "provider": "mistral" + }, + "metric_keys": [ + "completion_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "name": "mistral.fim.complete", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": "llm" + }, + { + "has_input": false, + "has_output": false, + "metadata": { + "operation": "fim-stream" + }, + "metric_keys": [], + "name": "mistral-fim-stream-operation", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": null + }, + { + "has_input": true, + "has_output": true, + "metadata": { + "model": "codestral-latest", + "provider": "mistral" + }, + "metric_keys": [ + "time_to_first_token" + ], + "name": "mistral.fim.stream", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": "llm" + }, + { + "has_input": false, + "has_output": false, + "metadata": { + "operation": "agents-complete" + }, + "metric_keys": [], + "name": "mistral-agents-complete-operation", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": null + }, + { + "has_input": true, + "has_output": true, + "metadata": { + "model": "mistral-small-latest", + "provider": "mistral" + }, + "metric_keys": [ + "completion_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "name": "mistral.agents.complete", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": "llm" + }, + { + "has_input": false, + "has_output": false, + "metadata": { + "operation": "agents-tool-call" + }, + "metric_keys": [], + "name": "mistral-agents-tool-call-operation", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": null + }, + { + "has_input": true, + "has_output": true, + "metadata": { + "model": "mistral-small-latest", + "provider": "mistral" + }, + "metric_keys": [ + "completion_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "name": "mistral.agents.complete", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": "llm" + }, + { + "has_input": false, + "has_output": false, + "metadata": { + "operation": "agents-stream" + }, + "metric_keys": [], + "name": "mistral-agents-stream-operation", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": null + }, + { + "has_input": true, + "has_output": true, + "metadata": { + "model": "mistral-small-latest", + "provider": "mistral" + }, + "metric_keys": [ + "completion_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "name": "mistral.agents.stream", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": "llm" + }, + { + "has_input": false, + "has_output": false, + "metadata": { + "operation": "embeddings-create" + }, + "metric_keys": [], + "name": "mistral-embeddings-operation", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": null + }, + { + "has_input": true, + "has_output": true, + "metadata": { + "model": "mistral-embed", + "provider": "mistral" + }, + "metric_keys": [ + "completion_tokens", + "prompt_tokens", + "tokens" + ], + "name": "mistral.embeddings.create", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": "llm" + } +] diff --git a/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1.log-payloads.json b/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1.log-payloads.json new file mode 100644 index 000000000..c435f5788 --- /dev/null +++ b/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1.log-payloads.json @@ -0,0 +1,421 @@ +[ + { + "has_input": false, + "has_output": false, + "input": null, + "metadata": { + "scenario": "mistral-instrumentation" + }, + "metric_keys": [], + "output": null, + "span_id": "" + }, + { + "has_input": false, + "has_output": false, + "input": null, + "metadata": { + "operation": "chat-complete" + }, + "metric_keys": [], + "output": null, + "span_id": "" + }, + { + "has_input": true, + "has_output": true, + "input": { + "item_count": 2, + "roles": [ + "system", + "user" + ], + "type": "messages" + }, + "metadata": { + "model": "mistral-small-latest", + "provider": "mistral" + }, + "metric_keys": [ + "completion_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "output": { + "choice_count": 1, + "finish_reason": "stop", + "has_content": true, + "role": "assistant", + "tool_call_count": 0, + "type": "choices" + }, + "span_id": "" + }, + { + "has_input": false, + "has_output": false, + "input": null, + "metadata": { + "operation": "chat-stream" + }, + "metric_keys": [], + "output": null, + "span_id": "" + }, + { + "has_input": true, + "has_output": true, + "input": { + "item_count": 1, + "roles": [ + "user" + ], + "type": "messages" + }, + "metadata": { + "model": "mistral-small-latest", + "provider": "mistral" + }, + "metric_keys": [ + "completion_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "output": { + "choice_count": 1, + "finish_reason": "stop", + "has_content": true, + "role": "assistant", + "tool_call_count": 0, + "type": "choices" + }, + "span_id": "" + }, + { + "has_input": false, + "has_output": false, + "input": null, + "metadata": { + "operation": "chat-tool-call" + }, + "metric_keys": [], + "output": null, + "span_id": "" + }, + { + "has_input": true, + "has_output": true, + "input": { + "item_count": 1, + "roles": [ + "user" + ], + "type": "messages" + }, + "metadata": { + "model": "mistral-small-latest", + "provider": "mistral" + }, + "metric_keys": [ + "completion_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "output": { + "choice_count": 1, + "finish_reason": "tool_calls", + "has_content": false, + "role": "assistant", + "tool_call_count": 1, + "type": "choices" + }, + "span_id": "" + }, + { + "has_input": true, + "has_output": true, + "input": { + "item_count": 1, + "roles": [ + "user" + ], + "type": "messages" + }, + "metadata": { + "model": "mistral-small-latest", + "provider": "mistral" + }, + "metric_keys": [ + "completion_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "output": { + "choice_count": 1, + "finish_reason": "tool_calls", + "has_content": false, + "role": "assistant", + "tool_call_count": 1, + "type": "choices" + }, + "span_id": "" + }, + { + "has_input": true, + "has_output": true, + "input": { + "item_count": 2, + "roles": [ + "system", + "user" + ], + "type": "messages" + }, + "metadata": { + "model": "mistral-small-latest", + "provider": "mistral" + }, + "metric_keys": [ + "completion_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "output": { + "choice_count": 1, + "finish_reason": "tool_calls", + "has_content": false, + "role": "assistant", + "tool_call_count": 2, + "type": "choices" + }, + "span_id": "" + }, + { + "has_input": false, + "has_output": false, + "input": null, + "metadata": { + "operation": "fim-complete" + }, + "metric_keys": [], + "output": null, + "span_id": "" + }, + { + "has_input": true, + "has_output": true, + "input": "", + "metadata": { + "model": "codestral-latest", + "provider": "mistral" + }, + "metric_keys": [ + "completion_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "output": { + "choice_count": 1, + "finish_reason": "stop", + "has_content": true, + "role": "assistant", + "tool_call_count": 0, + "type": "choices" + }, + "span_id": "" + }, + { + "has_input": false, + "has_output": false, + "input": null, + "metadata": { + "operation": "fim-stream" + }, + "metric_keys": [], + "output": null, + "span_id": "" + }, + { + "has_input": true, + "has_output": true, + "input": "", + "metadata": { + "model": "codestral-latest", + "provider": "mistral" + }, + "metric_keys": [ + "completion_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "output": { + "choice_count": 1, + "finish_reason": "length", + "has_content": true, + "role": "assistant", + "tool_call_count": 0, + "type": "choices" + }, + "span_id": "" + }, + { + "has_input": false, + "has_output": false, + "input": null, + "metadata": { + "operation": "agents-complete" + }, + "metric_keys": [], + "output": null, + "span_id": "" + }, + { + "has_input": true, + "has_output": true, + "input": { + "item_count": 1, + "roles": [ + "user" + ], + "type": "messages" + }, + "metadata": { + "model": "mistral-small-latest", + "provider": "mistral" + }, + "metric_keys": [ + "completion_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "output": { + "choice_count": 1, + "finish_reason": "stop", + "has_content": true, + "role": "assistant", + "tool_call_count": 0, + "type": "choices" + }, + "span_id": "" + }, + { + "has_input": false, + "has_output": false, + "input": null, + "metadata": { + "operation": "agents-tool-call" + }, + "metric_keys": [], + "output": null, + "span_id": "" + }, + { + "has_input": true, + "has_output": true, + "input": { + "item_count": 1, + "roles": [ + "user" + ], + "type": "messages" + }, + "metadata": { + "model": "mistral-small-latest", + "provider": "mistral" + }, + "metric_keys": [ + "completion_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "output": { + "choice_count": 1, + "finish_reason": "tool_calls", + "has_content": false, + "role": "assistant", + "tool_call_count": 1, + "type": "choices" + }, + "span_id": "" + }, + { + "has_input": false, + "has_output": false, + "input": null, + "metadata": { + "operation": "agents-stream" + }, + "metric_keys": [], + "output": null, + "span_id": "" + }, + { + "has_input": true, + "has_output": true, + "input": { + "item_count": 1, + "roles": [ + "user" + ], + "type": "messages" + }, + "metadata": { + "model": "mistral-small-latest", + "provider": "mistral" + }, + "metric_keys": [ + "completion_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "output": { + "choice_count": 1, + "finish_reason": "stop", + "has_content": true, + "role": "assistant", + "tool_call_count": 0, + "type": "choices" + }, + "span_id": "" + }, + { + "has_input": false, + "has_output": false, + "input": null, + "metadata": { + "operation": "embeddings-create" + }, + "metric_keys": [], + "output": null, + "span_id": "" + }, + { + "has_input": true, + "has_output": true, + "input": "", + "metadata": { + "model": "mistral-embed", + "provider": "mistral" + }, + "metric_keys": [ + "completion_tokens", + "prompt_tokens", + "tokens" + ], + "output": { + "embedding_length": "", + "type": "embedding" + }, + "span_id": "" + } +] diff --git a/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1.span-events.json b/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1.span-events.json new file mode 100644 index 000000000..b7d2919bc --- /dev/null +++ b/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1.span-events.json @@ -0,0 +1,380 @@ +[ + { + "has_input": false, + "has_output": false, + "metadata": { + "scenario": "mistral-instrumentation" + }, + "metric_keys": [], + "name": "mistral-root", + "root_span_id": "", + "span_id": "", + "span_parents": [], + "type": "task" + }, + { + "has_input": false, + "has_output": false, + "metadata": { + "operation": "chat-complete" + }, + "metric_keys": [], + "name": "mistral-chat-complete-operation", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": null + }, + { + "has_input": true, + "has_output": true, + "metadata": { + "model": "mistral-small-latest", + "provider": "mistral" + }, + "metric_keys": [ + "completion_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "name": "mistral.chat.complete", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": "llm" + }, + { + "has_input": false, + "has_output": false, + "metadata": { + "operation": "chat-stream" + }, + "metric_keys": [], + "name": "mistral-chat-stream-operation", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": null + }, + { + "has_input": true, + "has_output": true, + "metadata": { + "model": "mistral-small-latest", + "provider": "mistral" + }, + "metric_keys": [ + "completion_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "name": "mistral.chat.stream", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": "llm" + }, + { + "has_input": false, + "has_output": false, + "metadata": { + "operation": "chat-tool-call" + }, + "metric_keys": [], + "name": "mistral-chat-tool-call-operation", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": null + }, + { + "has_input": true, + "has_output": true, + "metadata": { + "model": "mistral-small-latest", + "provider": "mistral" + }, + "metric_keys": [ + "completion_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "name": "mistral.chat.complete", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": "llm" + }, + { + "has_input": true, + "has_output": true, + "metadata": { + "model": "mistral-small-latest", + "provider": "mistral" + }, + "metric_keys": [ + "completion_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "name": "mistral.chat.complete", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": "llm" + }, + { + "has_input": true, + "has_output": true, + "metadata": { + "model": "mistral-small-latest", + "provider": "mistral" + }, + "metric_keys": [ + "completion_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "name": "mistral.chat.complete", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": "llm" + }, + { + "has_input": false, + "has_output": false, + "metadata": { + "operation": "fim-complete" + }, + "metric_keys": [], + "name": "mistral-fim-complete-operation", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": null + }, + { + "has_input": true, + "has_output": true, + "metadata": { + "model": "codestral-latest", + "provider": "mistral" + }, + "metric_keys": [ + "completion_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "name": "mistral.fim.complete", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": "llm" + }, + { + "has_input": false, + "has_output": false, + "metadata": { + "operation": "fim-stream" + }, + "metric_keys": [], + "name": "mistral-fim-stream-operation", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": null + }, + { + "has_input": true, + "has_output": true, + "metadata": { + "model": "codestral-latest", + "provider": "mistral" + }, + "metric_keys": [ + "completion_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "name": "mistral.fim.stream", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": "llm" + }, + { + "has_input": false, + "has_output": false, + "metadata": { + "operation": "agents-complete" + }, + "metric_keys": [], + "name": "mistral-agents-complete-operation", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": null + }, + { + "has_input": true, + "has_output": true, + "metadata": { + "model": "mistral-small-latest", + "provider": "mistral" + }, + "metric_keys": [ + "completion_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "name": "mistral.agents.complete", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": "llm" + }, + { + "has_input": false, + "has_output": false, + "metadata": { + "operation": "agents-tool-call" + }, + "metric_keys": [], + "name": "mistral-agents-tool-call-operation", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": null + }, + { + "has_input": true, + "has_output": true, + "metadata": { + "model": "mistral-small-latest", + "provider": "mistral" + }, + "metric_keys": [ + "completion_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "name": "mistral.agents.complete", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": "llm" + }, + { + "has_input": false, + "has_output": false, + "metadata": { + "operation": "agents-stream" + }, + "metric_keys": [], + "name": "mistral-agents-stream-operation", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": null + }, + { + "has_input": true, + "has_output": true, + "metadata": { + "model": "mistral-small-latest", + "provider": "mistral" + }, + "metric_keys": [ + "completion_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "name": "mistral.agents.stream", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": "llm" + }, + { + "has_input": false, + "has_output": false, + "metadata": { + "operation": "embeddings-create" + }, + "metric_keys": [], + "name": "mistral-embeddings-operation", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": null + }, + { + "has_input": true, + "has_output": true, + "metadata": { + "model": "mistral-embed", + "provider": "mistral" + }, + "metric_keys": [ + "completion_tokens", + "prompt_tokens", + "tokens" + ], + "name": "mistral.embeddings.create", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": "llm" + } +] diff --git a/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v2.log-payloads.json b/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v2.log-payloads.json new file mode 100644 index 000000000..85b50c53d --- /dev/null +++ b/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v2.log-payloads.json @@ -0,0 +1,431 @@ +[ + { + "has_input": false, + "has_output": false, + "input": null, + "metadata": { + "scenario": "mistral-instrumentation" + }, + "metric_keys": [], + "output": null, + "span_id": "" + }, + { + "has_input": false, + "has_output": false, + "input": null, + "metadata": { + "operation": "chat-complete" + }, + "metric_keys": [], + "output": null, + "span_id": "" + }, + { + "has_input": true, + "has_output": true, + "input": { + "item_count": 2, + "roles": [ + "system", + "user" + ], + "type": "messages" + }, + "metadata": { + "model": "mistral-small-latest", + "provider": "mistral" + }, + "metric_keys": [ + "completion_tokens", + "prompt_cached_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "output": { + "choice_count": 1, + "finish_reason": "stop", + "has_content": true, + "role": "assistant", + "tool_call_count": 0, + "type": "choices" + }, + "span_id": "" + }, + { + "has_input": false, + "has_output": false, + "input": null, + "metadata": { + "operation": "chat-stream" + }, + "metric_keys": [], + "output": null, + "span_id": "" + }, + { + "has_input": true, + "has_output": true, + "input": { + "item_count": 1, + "roles": [ + "user" + ], + "type": "messages" + }, + "metadata": { + "model": "mistral-small-latest", + "provider": "mistral" + }, + "metric_keys": [ + "completion_tokens", + "prompt_cached_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "output": { + "choice_count": 1, + "finish_reason": "stop", + "has_content": true, + "role": "assistant", + "tool_call_count": 0, + "type": "choices" + }, + "span_id": "" + }, + { + "has_input": false, + "has_output": false, + "input": null, + "metadata": { + "operation": "chat-tool-call" + }, + "metric_keys": [], + "output": null, + "span_id": "" + }, + { + "has_input": true, + "has_output": true, + "input": { + "item_count": 1, + "roles": [ + "user" + ], + "type": "messages" + }, + "metadata": { + "model": "mistral-small-latest", + "provider": "mistral" + }, + "metric_keys": [ + "completion_tokens", + "prompt_cached_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "output": { + "choice_count": 1, + "finish_reason": "tool_calls", + "has_content": false, + "role": "assistant", + "tool_call_count": 1, + "type": "choices" + }, + "span_id": "" + }, + { + "has_input": true, + "has_output": true, + "input": { + "item_count": 1, + "roles": [ + "user" + ], + "type": "messages" + }, + "metadata": { + "model": "mistral-small-latest", + "provider": "mistral" + }, + "metric_keys": [ + "completion_tokens", + "prompt_cached_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "output": { + "choice_count": 1, + "finish_reason": "tool_calls", + "has_content": false, + "role": "assistant", + "tool_call_count": 1, + "type": "choices" + }, + "span_id": "" + }, + { + "has_input": true, + "has_output": true, + "input": { + "item_count": 2, + "roles": [ + "system", + "user" + ], + "type": "messages" + }, + "metadata": { + "model": "mistral-small-latest", + "provider": "mistral" + }, + "metric_keys": [ + "completion_tokens", + "prompt_cached_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "output": { + "choice_count": 1, + "finish_reason": "tool_calls", + "has_content": false, + "role": "assistant", + "tool_call_count": 2, + "type": "choices" + }, + "span_id": "" + }, + { + "has_input": false, + "has_output": false, + "input": null, + "metadata": { + "operation": "fim-complete" + }, + "metric_keys": [], + "output": null, + "span_id": "" + }, + { + "has_input": true, + "has_output": true, + "input": "", + "metadata": { + "model": "codestral-latest", + "provider": "mistral" + }, + "metric_keys": [ + "completion_tokens", + "prompt_cached_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "output": { + "choice_count": 1, + "finish_reason": "stop", + "has_content": true, + "role": "assistant", + "tool_call_count": 0, + "type": "choices" + }, + "span_id": "" + }, + { + "has_input": false, + "has_output": false, + "input": null, + "metadata": { + "operation": "fim-stream" + }, + "metric_keys": [], + "output": null, + "span_id": "" + }, + { + "has_input": true, + "has_output": true, + "input": "", + "metadata": { + "model": "codestral-latest", + "provider": "mistral" + }, + "metric_keys": [ + "completion_tokens", + "prompt_cached_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "output": { + "choice_count": 1, + "finish_reason": "length", + "has_content": true, + "role": "assistant", + "tool_call_count": 0, + "type": "choices" + }, + "span_id": "" + }, + { + "has_input": false, + "has_output": false, + "input": null, + "metadata": { + "operation": "agents-complete" + }, + "metric_keys": [], + "output": null, + "span_id": "" + }, + { + "has_input": true, + "has_output": true, + "input": { + "item_count": 1, + "roles": [ + "user" + ], + "type": "messages" + }, + "metadata": { + "model": "mistral-small-latest", + "provider": "mistral" + }, + "metric_keys": [ + "completion_tokens", + "prompt_cached_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "output": { + "choice_count": 1, + "finish_reason": "stop", + "has_content": true, + "role": "assistant", + "tool_call_count": 0, + "type": "choices" + }, + "span_id": "" + }, + { + "has_input": false, + "has_output": false, + "input": null, + "metadata": { + "operation": "agents-tool-call" + }, + "metric_keys": [], + "output": null, + "span_id": "" + }, + { + "has_input": true, + "has_output": true, + "input": { + "item_count": 1, + "roles": [ + "user" + ], + "type": "messages" + }, + "metadata": { + "model": "mistral-small-latest", + "provider": "mistral" + }, + "metric_keys": [ + "completion_tokens", + "prompt_cached_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "output": { + "choice_count": 1, + "finish_reason": "tool_calls", + "has_content": false, + "role": "assistant", + "tool_call_count": 1, + "type": "choices" + }, + "span_id": "" + }, + { + "has_input": false, + "has_output": false, + "input": null, + "metadata": { + "operation": "agents-stream" + }, + "metric_keys": [], + "output": null, + "span_id": "" + }, + { + "has_input": true, + "has_output": true, + "input": { + "item_count": 1, + "roles": [ + "user" + ], + "type": "messages" + }, + "metadata": { + "model": "mistral-small-latest", + "provider": "mistral" + }, + "metric_keys": [ + "completion_tokens", + "prompt_cached_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "output": { + "choice_count": 1, + "finish_reason": "stop", + "has_content": true, + "role": "assistant", + "tool_call_count": 0, + "type": "choices" + }, + "span_id": "" + }, + { + "has_input": false, + "has_output": false, + "input": null, + "metadata": { + "operation": "embeddings-create" + }, + "metric_keys": [], + "output": null, + "span_id": "" + }, + { + "has_input": true, + "has_output": true, + "input": "", + "metadata": { + "model": "mistral-embed", + "provider": "mistral" + }, + "metric_keys": [ + "completion_tokens", + "prompt_tokens", + "tokens" + ], + "output": { + "embedding_length": "", + "type": "embedding" + }, + "span_id": "" + } +] diff --git a/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v2.span-events.json b/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v2.span-events.json new file mode 100644 index 000000000..6227c9374 --- /dev/null +++ b/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v2.span-events.json @@ -0,0 +1,390 @@ +[ + { + "has_input": false, + "has_output": false, + "metadata": { + "scenario": "mistral-instrumentation" + }, + "metric_keys": [], + "name": "mistral-root", + "root_span_id": "", + "span_id": "", + "span_parents": [], + "type": "task" + }, + { + "has_input": false, + "has_output": false, + "metadata": { + "operation": "chat-complete" + }, + "metric_keys": [], + "name": "mistral-chat-complete-operation", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": null + }, + { + "has_input": true, + "has_output": true, + "metadata": { + "model": "mistral-small-latest", + "provider": "mistral" + }, + "metric_keys": [ + "completion_tokens", + "prompt_cached_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "name": "mistral.chat.complete", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": "llm" + }, + { + "has_input": false, + "has_output": false, + "metadata": { + "operation": "chat-stream" + }, + "metric_keys": [], + "name": "mistral-chat-stream-operation", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": null + }, + { + "has_input": true, + "has_output": true, + "metadata": { + "model": "mistral-small-latest", + "provider": "mistral" + }, + "metric_keys": [ + "completion_tokens", + "prompt_cached_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "name": "mistral.chat.stream", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": "llm" + }, + { + "has_input": false, + "has_output": false, + "metadata": { + "operation": "chat-tool-call" + }, + "metric_keys": [], + "name": "mistral-chat-tool-call-operation", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": null + }, + { + "has_input": true, + "has_output": true, + "metadata": { + "model": "mistral-small-latest", + "provider": "mistral" + }, + "metric_keys": [ + "completion_tokens", + "prompt_cached_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "name": "mistral.chat.complete", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": "llm" + }, + { + "has_input": true, + "has_output": true, + "metadata": { + "model": "mistral-small-latest", + "provider": "mistral" + }, + "metric_keys": [ + "completion_tokens", + "prompt_cached_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "name": "mistral.chat.complete", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": "llm" + }, + { + "has_input": true, + "has_output": true, + "metadata": { + "model": "mistral-small-latest", + "provider": "mistral" + }, + "metric_keys": [ + "completion_tokens", + "prompt_cached_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "name": "mistral.chat.complete", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": "llm" + }, + { + "has_input": false, + "has_output": false, + "metadata": { + "operation": "fim-complete" + }, + "metric_keys": [], + "name": "mistral-fim-complete-operation", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": null + }, + { + "has_input": true, + "has_output": true, + "metadata": { + "model": "codestral-latest", + "provider": "mistral" + }, + "metric_keys": [ + "completion_tokens", + "prompt_cached_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "name": "mistral.fim.complete", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": "llm" + }, + { + "has_input": false, + "has_output": false, + "metadata": { + "operation": "fim-stream" + }, + "metric_keys": [], + "name": "mistral-fim-stream-operation", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": null + }, + { + "has_input": true, + "has_output": true, + "metadata": { + "model": "codestral-latest", + "provider": "mistral" + }, + "metric_keys": [ + "completion_tokens", + "prompt_cached_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "name": "mistral.fim.stream", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": "llm" + }, + { + "has_input": false, + "has_output": false, + "metadata": { + "operation": "agents-complete" + }, + "metric_keys": [], + "name": "mistral-agents-complete-operation", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": null + }, + { + "has_input": true, + "has_output": true, + "metadata": { + "model": "mistral-small-latest", + "provider": "mistral" + }, + "metric_keys": [ + "completion_tokens", + "prompt_cached_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "name": "mistral.agents.complete", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": "llm" + }, + { + "has_input": false, + "has_output": false, + "metadata": { + "operation": "agents-tool-call" + }, + "metric_keys": [], + "name": "mistral-agents-tool-call-operation", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": null + }, + { + "has_input": true, + "has_output": true, + "metadata": { + "model": "mistral-small-latest", + "provider": "mistral" + }, + "metric_keys": [ + "completion_tokens", + "prompt_cached_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "name": "mistral.agents.complete", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": "llm" + }, + { + "has_input": false, + "has_output": false, + "metadata": { + "operation": "agents-stream" + }, + "metric_keys": [], + "name": "mistral-agents-stream-operation", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": null + }, + { + "has_input": true, + "has_output": true, + "metadata": { + "model": "mistral-small-latest", + "provider": "mistral" + }, + "metric_keys": [ + "completion_tokens", + "prompt_cached_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "name": "mistral.agents.stream", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": "llm" + }, + { + "has_input": false, + "has_output": false, + "metadata": { + "operation": "embeddings-create" + }, + "metric_keys": [], + "name": "mistral-embeddings-operation", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": null + }, + { + "has_input": true, + "has_output": true, + "metadata": { + "model": "mistral-embed", + "provider": "mistral" + }, + "metric_keys": [ + "completion_tokens", + "prompt_tokens", + "tokens" + ], + "name": "mistral.embeddings.create", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": "llm" + } +] diff --git a/e2e/scenarios/mistral-instrumentation/assertions.ts b/e2e/scenarios/mistral-instrumentation/assertions.ts new file mode 100644 index 000000000..da757b9f9 --- /dev/null +++ b/e2e/scenarios/mistral-instrumentation/assertions.ts @@ -0,0 +1,894 @@ +import { beforeAll, describe, expect, test } from "vitest"; +import type { + CapturedLogEvent, + CapturedLogPayload, + CapturedLogRow, +} from "../../helpers/mock-braintrust-server"; +import type { Json } from "../../helpers/normalize"; +import { normalizeForSnapshot } from "../../helpers/normalize"; +import { + formatJsonFileSnapshot, + resolveFileSnapshotPath, +} from "../../helpers/file-snapshot"; +import { withScenarioHarness } from "../../helpers/scenario-harness"; +import { findChildSpans, findLatestSpan } from "../../helpers/trace-selectors"; +import { + payloadRowsForRootSpan, + summarizeWrapperContract, +} from "../../helpers/wrapper-contract"; +import { + FIM_MODEL, + CHAT_MODEL, + AGENT_MODEL, + EMBEDDING_MODEL, + ROOT_NAME, + SCENARIO_NAME, +} from "./constants.mjs"; + +type RunMistralScenario = (harness: { + runNodeScenarioDir: (options: { + entry: string; + nodeArgs: string[]; + runContext?: { variantKey: string }; + scenarioDir: string; + timeoutMs: number; + }) => Promise; + runScenarioDir: (options: { + entry: string; + runContext?: { variantKey: string }; + scenarioDir: string; + timeoutMs: number; + }) => Promise; +}) => Promise; + +function findMistralSpan( + events: CapturedLogEvent[], + parentId: string | undefined, + names: string[], +) { + for (const name of names) { + const spans = findChildSpans(events, name, parentId); + const spanWithOutput = spans.find((candidate) => candidate.output != null); + if (spanWithOutput) { + return spanWithOutput; + } + if (spans[0]) { + return spans[0]; + } + } + + return undefined; +} + +function isRecord(value: Json | undefined): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value); +} + +function pickMetadata( + metadata: Record | undefined, + keys: string[], +): Json { + if (!metadata) { + return null; + } + + const picked = Object.fromEntries( + keys.flatMap((key) => + key in metadata ? [[key, metadata[key] as Json]] : [], + ), + ); + + return Object.keys(picked).length > 0 ? (picked as Json) : null; +} + +function summarizeInput( + input: unknown, + options?: { omitObjectKeys?: boolean }, +): Json { + if (Array.isArray(input)) { + const roles = input + .map((message) => + isRecord(message as Json) && typeof message.role === "string" + ? message.role + : null, + ) + .filter((role): role is string => role !== null); + + return { + item_count: input.length, + roles, + type: "messages", + }; + } + + if (typeof input === "string") { + return ""; + } + + if (!isRecord(input as Json)) { + return null; + } + + if (options?.omitObjectKeys) { + return { + type: "object", + }; + } + + return { + keys: Object.keys(input).sort(), + type: "object", + }; +} + +function summarizeOutput(output: unknown): Json { + if (Array.isArray(output)) { + const firstChoice = output[0] as Json; + if (!isRecord(firstChoice) || !isRecord(firstChoice.message)) { + return { + choice_count: output.length, + finish_reason: null, + type: "array", + }; + } + + const message = firstChoice.message; + const finishReason = + typeof firstChoice.finishReason === "string" || + firstChoice.finishReason === null + ? firstChoice.finishReason + : typeof firstChoice.finish_reason === "string" || + firstChoice.finish_reason === null + ? firstChoice.finish_reason + : null; + const toolCalls = + (Array.isArray(message.tool_calls) && message.tool_calls) || + (Array.isArray(message.toolCalls) && message.toolCalls) || + []; + + return { + choice_count: output.length, + finish_reason: finishReason, + has_content: + typeof message.content === "string" + ? message.content.length > 0 + : false, + role: typeof message.role === "string" ? message.role : null, + tool_call_count: toolCalls.length, + type: "choices", + }; + } + + if (!isRecord(output as Json)) { + return output === undefined ? null : ""; + } + + if (typeof output.embedding_length === "number") { + return { + embedding_length: "", + type: "embedding", + }; + } + + return { + keys: Object.keys(output).sort(), + type: "object", + }; +} + +function summarizePayloadRow(row: CapturedLogRow): Json { + const metrics = + typeof row.metrics === "object" && + row.metrics !== null && + !Array.isArray(row.metrics) + ? (row.metrics as Record) + : {}; + return { + has_input: row.input !== undefined && row.input !== null, + has_output: row.output !== undefined && row.output !== null, + input: summarizeInput(row.input), + metadata: pickMetadata( + row.metadata as Record | undefined, + ["model", "operation", "provider", "scenario"], + ), + metric_keys: Object.keys(metrics) + .filter((key) => key !== "start" && key !== "end") + .sort(), + output: summarizeOutput(row.output), + span_id: typeof row.span_id === "string" ? row.span_id : null, + } satisfies Json; +} + +function normalizeLegacyV134MetricKeys(metricKeys: Json): Json { + if (!Array.isArray(metricKeys)) { + return metricKeys; + } + + return metricKeys.includes("time_to_first_token") + ? ["time_to_first_token"] + : metricKeys; +} + +function normalizeLegacyV134SpanSummaryRow( + summaryRow: Json, + snapshotName: string, +): Json { + if (snapshotName !== "mistral-v1-3-4" || !isRecord(summaryRow)) { + return summaryRow; + } + + if (summaryRow.name !== "mistral.fim.stream") { + return summaryRow; + } + + return { + ...summaryRow, + metric_keys: normalizeLegacyV134MetricKeys(summaryRow.metric_keys), + }; +} + +function normalizeLegacyV134PayloadSummaryRow( + summaryRow: Json, + snapshotName: string, + spanName: string | undefined, +): Json { + if ( + snapshotName !== "mistral-v1-3-4" || + spanName !== "mistral.fim.stream" || + !isRecord(summaryRow) + ) { + return summaryRow; + } + + const output = isRecord(summaryRow.output) ? summaryRow.output : null; + + return { + ...summaryRow, + metric_keys: normalizeLegacyV134MetricKeys(summaryRow.metric_keys), + ...(output + ? { + output: { + ...output, + finish_reason: null, + }, + } + : {}), + }; +} + +function mergeRecordValues( + left: Record | undefined, + right: unknown, +): Record | undefined { + if (!right || typeof right !== "object" || Array.isArray(right)) { + return left; + } + + return { + ...(left || {}), + ...(right as Record), + }; +} + +function mergePayloadRows(rows: CapturedLogRow[]): CapturedLogRow[] { + const mergedBySpan = new Map(); + const spanOrder: string[] = []; + + for (const row of rows) { + const spanId = + typeof row.span_id === "string" + ? row.span_id + : `unknown-${spanOrder.length}`; + const existing = mergedBySpan.get(spanId); + + if (!existing) { + mergedBySpan.set(spanId, { + ...row, + }); + spanOrder.push(spanId); + continue; + } + + mergedBySpan.set(spanId, { + ...existing, + ...(row.input !== undefined && row.input !== null + ? { + input: row.input, + } + : {}), + ...(row.output !== undefined && row.output !== null + ? { + output: row.output, + } + : {}), + metadata: mergeRecordValues( + existing.metadata as Record | undefined, + row.metadata, + ), + metrics: mergeRecordValues( + existing.metrics as Record | undefined, + row.metrics, + ), + }); + } + + return spanOrder + .map((spanId) => mergedBySpan.get(spanId)) + .filter((row): row is CapturedLogRow => row !== undefined); +} + +function pickOutputSpans(spans: T[]): T[] { + const spansWithOutput = spans.filter( + (span) => span.output !== undefined && span.output !== null, + ); + return spansWithOutput.length > 0 ? spansWithOutput : spans; +} + +function buildSpanSummary( + events: CapturedLogEvent[], + snapshotName: string, +): Json { + const chatCompleteOperation = findLatestSpan( + events, + "mistral-chat-complete-operation", + ); + const chatStreamOperation = findLatestSpan( + events, + "mistral-chat-stream-operation", + ); + const chatToolCallOperation = findLatestSpan( + events, + "mistral-chat-tool-call-operation", + ); + const chatToolCallSpans = findChildSpans( + events, + "mistral.chat.complete", + chatToolCallOperation?.span.id, + ); + const selectedChatToolCallSpans = pickOutputSpans(chatToolCallSpans); + const fimCompleteOperation = findLatestSpan( + events, + "mistral-fim-complete-operation", + ); + const fimStreamOperation = findLatestSpan( + events, + "mistral-fim-stream-operation", + ); + const agentsCompleteOperation = findLatestSpan( + events, + "mistral-agents-complete-operation", + ); + const agentsToolCallOperation = findLatestSpan( + events, + "mistral-agents-tool-call-operation", + ); + const agentsToolCallSpans = findChildSpans( + events, + "mistral.agents.complete", + agentsToolCallOperation?.span.id, + ); + const selectedAgentsToolCallSpans = pickOutputSpans(agentsToolCallSpans); + const agentsStreamOperation = findLatestSpan( + events, + "mistral-agents-stream-operation", + ); + const embeddingsOperation = findLatestSpan( + events, + "mistral-embeddings-operation", + ); + + return normalizeForSnapshot( + [ + findLatestSpan(events, ROOT_NAME), + chatCompleteOperation, + findMistralSpan(events, chatCompleteOperation?.span.id, [ + "mistral.chat.complete", + ]), + chatStreamOperation, + findMistralSpan(events, chatStreamOperation?.span.id, [ + "mistral.chat.stream", + ]), + chatToolCallOperation, + ...selectedChatToolCallSpans, + fimCompleteOperation, + findMistralSpan(events, fimCompleteOperation?.span.id, [ + "mistral.fim.complete", + ]), + fimStreamOperation, + findMistralSpan(events, fimStreamOperation?.span.id, [ + "mistral.fim.stream", + ]), + agentsCompleteOperation, + findMistralSpan(events, agentsCompleteOperation?.span.id, [ + "mistral.agents.complete", + ]), + agentsToolCallOperation, + ...selectedAgentsToolCallSpans, + agentsStreamOperation, + findMistralSpan(events, agentsStreamOperation?.span.id, [ + "mistral.agents.stream", + ]), + embeddingsOperation, + findMistralSpan(events, embeddingsOperation?.span.id, [ + "mistral.embeddings.create", + ]), + ].map((event) => + normalizeLegacyV134SpanSummaryRow( + summarizeWrapperContract(event!, [ + "model", + "operation", + "provider", + "scenario", + ]), + snapshotName, + ), + ) as Json, + ); +} + +function buildPayloadSummary( + events: CapturedLogEvent[], + payloads: CapturedLogPayload[], + snapshotName: string, +): Json { + const parentIdBySpanId = new Map(); + const spanNameById = new Map(); + for (const event of events) { + if ( + typeof event?.span?.id === "string" && + typeof event?.span?.name === "string" + ) { + const parentId = + Array.isArray(event.span.parentIds) && + typeof event.span.parentIds[0] === "string" + ? event.span.parentIds[0] + : ""; + parentIdBySpanId.set(event.span.id, parentId); + spanNameById.set(event.span.id, event.span.name); + } + } + + const root = findLatestSpan(events, ROOT_NAME); + const payloadRows = payloadRowsForRootSpan(payloads, root?.span.id); + const mergedRows = mergePayloadRows(payloadRows); + const llmSpanNames = new Set([ + "mistral.chat.complete", + "mistral.chat.stream", + "mistral.fim.complete", + "mistral.fim.stream", + "mistral.agents.complete", + "mistral.agents.stream", + "mistral.embeddings.create", + ]); + const parentAndNameWithOutput = new Set(); + + for (const row of mergedRows) { + if (typeof row.span_id !== "string") { + continue; + } + const spanName = spanNameById.get(row.span_id); + if (!spanName || !llmSpanNames.has(spanName)) { + continue; + } + if (row.output === undefined || row.output === null) { + continue; + } + + const parentId = parentIdBySpanId.get(row.span_id) || ""; + parentAndNameWithOutput.add(`${parentId}:${spanName}`); + } + + const stableRows = mergedRows.filter((row) => { + if (typeof row.span_id !== "string") { + return true; + } + + const spanName = spanNameById.get(row.span_id); + if (!spanName || !llmSpanNames.has(spanName)) { + return true; + } + + if (row.output !== undefined && row.output !== null) { + return true; + } + + const parentId = parentIdBySpanId.get(row.span_id) || ""; + return !parentAndNameWithOutput.has(`${parentId}:${spanName}`); + }); + + return normalizeForSnapshot( + stableRows.map((row) => { + const spanName = + typeof row.span_id === "string" + ? spanNameById.get(row.span_id) + : undefined; + + return normalizeLegacyV134PayloadSummaryRow( + summarizePayloadRow(row), + snapshotName, + spanName, + ); + }), + ); +} + +export function defineMistralInstrumentationAssertions(options: { + name: string; + runScenario: RunMistralScenario; + snapshotName: string; + testFileUrl: string; + timeoutMs: number; +}): void { + const spanSnapshotPath = resolveFileSnapshotPath( + options.testFileUrl, + `${options.snapshotName}.span-events.json`, + ); + const payloadSnapshotPath = resolveFileSnapshotPath( + options.testFileUrl, + `${options.snapshotName}.log-payloads.json`, + ); + const testConfig = { + timeout: options.timeoutMs, + }; + + describe(options.name, () => { + let events: CapturedLogEvent[] = []; + let payloads: CapturedLogPayload[] = []; + + beforeAll(async () => { + await withScenarioHarness(async (harness) => { + await options.runScenario(harness); + events = harness.events(); + payloads = harness.payloads(); + }); + }, options.timeoutMs); + + test("captures the root trace for the scenario", testConfig, () => { + const root = findLatestSpan(events, ROOT_NAME); + + expect(root).toBeDefined(); + expect(root?.row.metadata).toMatchObject({ + scenario: SCENARIO_NAME, + }); + }); + + test("captures trace for chat.complete()", testConfig, () => { + const root = findLatestSpan(events, ROOT_NAME); + const operation = findLatestSpan( + events, + "mistral-chat-complete-operation", + ); + const span = findMistralSpan(events, operation?.span.id, [ + "mistral.chat.complete", + ]); + + expect(operation).toBeDefined(); + expect(span).toBeDefined(); + expect(operation?.span.parentIds).toEqual([root?.span.id ?? ""]); + expect(span?.span.type).toBe("llm"); + expect(span?.row.metadata).toMatchObject({ + model: CHAT_MODEL, + provider: "mistral", + }); + expect(span?.output).toBeDefined(); + }); + + test("captures trace for chat.stream()", testConfig, () => { + const root = findLatestSpan(events, ROOT_NAME); + const operation = findLatestSpan(events, "mistral-chat-stream-operation"); + const span = findMistralSpan(events, operation?.span.id, [ + "mistral.chat.stream", + ]); + + expect(operation).toBeDefined(); + expect(span).toBeDefined(); + expect(operation?.span.parentIds).toEqual([root?.span.id ?? ""]); + expect(span?.span.type).toBe("llm"); + expect(span?.row.metadata).toMatchObject({ + model: CHAT_MODEL, + provider: "mistral", + }); + expect(span?.metrics?.time_to_first_token).toEqual(expect.any(Number)); + expect(span?.output).toBeDefined(); + }); + + test("captures trace for chat.complete() tool calling", testConfig, () => { + const root = findLatestSpan(events, ROOT_NAME); + const operation = findLatestSpan( + events, + "mistral-chat-tool-call-operation", + ); + const spans = findChildSpans( + events, + "mistral.chat.complete", + operation?.span.id, + ); + const selectedSpans = pickOutputSpans(spans); + const toolCallCountBySpanId = new Map( + selectedSpans.map((span) => { + const output = span.output as + | Array<{ + message?: { + toolCalls?: unknown; + tool_calls?: unknown; + }; + }> + | undefined; + const firstChoice = Array.isArray(output) ? output[0] : undefined; + const toolCalls = + (Array.isArray(firstChoice?.message?.tool_calls) && + firstChoice.message.tool_calls) || + (Array.isArray(firstChoice?.message?.toolCalls) && + firstChoice.message.toolCalls) || + []; + return [span.span.id, toolCalls.length]; + }), + ); + const spansWithToolCalls = selectedSpans.filter((span) => { + return (toolCallCountBySpanId.get(span.span.id) || 0) > 0; + }); + const finishReasons = selectedSpans + .map((span) => { + const output = span.output as + | Array<{ + finishReason?: unknown; + finish_reason?: unknown; + }> + | undefined; + const firstChoice = Array.isArray(output) ? output[0] : undefined; + if (typeof firstChoice?.finishReason === "string") { + return firstChoice.finishReason; + } + if (typeof firstChoice?.finish_reason === "string") { + return firstChoice.finish_reason; + } + return undefined; + }) + .filter((value): value is string => typeof value === "string"); + const toolNames = new Set( + selectedSpans.flatMap((span) => { + const output = span.output as + | Array<{ + message?: { + toolCalls?: unknown; + tool_calls?: unknown; + }; + }> + | undefined; + const firstChoice = Array.isArray(output) ? output[0] : undefined; + const toolCalls = + (Array.isArray(firstChoice?.message?.tool_calls) && + firstChoice.message.tool_calls) || + (Array.isArray(firstChoice?.message?.toolCalls) && + firstChoice.message.toolCalls) || + []; + + return toolCalls + .map((toolCall) => { + if ( + !toolCall || + typeof toolCall !== "object" || + Array.isArray(toolCall) + ) { + return undefined; + } + const toolFunction = ( + toolCall as { function?: { name?: unknown } } + ).function; + return typeof toolFunction?.name === "string" + ? toolFunction.name + : undefined; + }) + .filter((name): name is string => typeof name === "string"); + }), + ); + + expect(operation).toBeDefined(); + expect(operation?.span.parentIds).toEqual([root?.span.id ?? ""]); + expect(selectedSpans.length).toBeGreaterThanOrEqual(2); + expect(spansWithToolCalls.length).toBeGreaterThanOrEqual(2); + expect( + Array.from(toolCallCountBySpanId.values()).some((count) => count >= 2), + ).toBe(true); + expect(finishReasons.length).toBeGreaterThanOrEqual(2); + + for (const span of selectedSpans) { + expect(span.span.type).toBe("llm"); + expect(span.row.metadata).toMatchObject({ + model: CHAT_MODEL, + provider: "mistral", + }); + } + + expect(toolNames.has("get_weather")).toBe(true); + expect(toolNames.has("get_exchange_rate")).toBe(true); + }); + + test("captures trace for fim.complete()", testConfig, () => { + const root = findLatestSpan(events, ROOT_NAME); + const operation = findLatestSpan( + events, + "mistral-fim-complete-operation", + ); + const span = findMistralSpan(events, operation?.span.id, [ + "mistral.fim.complete", + ]); + + expect(operation).toBeDefined(); + expect(span).toBeDefined(); + expect(operation?.span.parentIds).toEqual([root?.span.id ?? ""]); + expect(span?.span.type).toBe("llm"); + expect(span?.row.metadata).toMatchObject({ + model: FIM_MODEL, + provider: "mistral", + }); + expect(span?.input).toEqual(expect.any(String)); + expect(span?.metrics?.time_to_first_token).toEqual(expect.any(Number)); + expect(span?.output).toBeDefined(); + }); + + test("captures trace for fim.stream()", testConfig, () => { + const root = findLatestSpan(events, ROOT_NAME); + const operation = findLatestSpan(events, "mistral-fim-stream-operation"); + const span = findMistralSpan(events, operation?.span.id, [ + "mistral.fim.stream", + ]); + + expect(operation).toBeDefined(); + expect(span).toBeDefined(); + expect(operation?.span.parentIds).toEqual([root?.span.id ?? ""]); + expect(span?.span.type).toBe("llm"); + expect(span?.row.metadata).toMatchObject({ + model: FIM_MODEL, + provider: "mistral", + }); + expect(span?.input).toEqual(expect.any(String)); + expect(span?.metrics?.time_to_first_token).toEqual(expect.any(Number)); + expect(span?.output).toBeDefined(); + }); + + test("captures trace for agents.complete()", testConfig, () => { + const root = findLatestSpan(events, ROOT_NAME); + const operation = findLatestSpan( + events, + "mistral-agents-complete-operation", + ); + const span = findMistralSpan(events, operation?.span.id, [ + "mistral.agents.complete", + ]); + const metadata = span?.row.metadata as + | Record + | undefined; + const input = span?.input as Array<{ role?: string }> | undefined; + + expect(operation).toBeDefined(); + expect(span).toBeDefined(); + expect(operation?.span.parentIds).toEqual([root?.span.id ?? ""]); + expect(span?.span.type).toBe("llm"); + expect(metadata).toMatchObject({ + provider: "mistral", + agentId: expect.any(String), + }); + if (typeof metadata?.model === "string") { + expect(metadata.model).toBe(AGENT_MODEL); + } + expect(input).toEqual(expect.any(Array)); + expect(input?.[0]?.role).toBe("user"); + expect(span?.metrics?.time_to_first_token).toEqual(expect.any(Number)); + expect(span?.output).toBeDefined(); + }); + + test( + "captures trace for agents.complete() tool calling", + testConfig, + () => { + const root = findLatestSpan(events, ROOT_NAME); + const operation = findLatestSpan( + events, + "mistral-agents-tool-call-operation", + ); + const spans = findChildSpans( + events, + "mistral.agents.complete", + operation?.span.id, + ); + const selectedSpans = pickOutputSpans(spans); + const spansWithToolCalls = selectedSpans.filter((span) => { + const output = span.output as + | Array<{ + message?: { + toolCalls?: unknown; + tool_calls?: unknown; + }; + }> + | undefined; + const firstChoice = Array.isArray(output) ? output[0] : undefined; + const toolCalls = + (Array.isArray(firstChoice?.message?.tool_calls) && + firstChoice.message.tool_calls) || + (Array.isArray(firstChoice?.message?.toolCalls) && + firstChoice.message.toolCalls) || + []; + return toolCalls.length > 0; + }); + + expect(operation).toBeDefined(); + expect(operation?.span.parentIds).toEqual([root?.span.id ?? ""]); + expect(selectedSpans.length).toBeGreaterThanOrEqual(1); + expect(spansWithToolCalls.length).toBeGreaterThanOrEqual(1); + + for (const span of selectedSpans) { + expect(span.span.type).toBe("llm"); + expect(span.row.metadata).toMatchObject({ + provider: "mistral", + }); + } + }, + ); + + test("captures trace for agents.stream()", testConfig, () => { + const root = findLatestSpan(events, ROOT_NAME); + const operation = findLatestSpan( + events, + "mistral-agents-stream-operation", + ); + const span = findMistralSpan(events, operation?.span.id, [ + "mistral.agents.stream", + ]); + const metadata = span?.row.metadata as + | Record + | undefined; + const input = span?.input as Array<{ role?: string }> | undefined; + + expect(operation).toBeDefined(); + expect(span).toBeDefined(); + expect(operation?.span.parentIds).toEqual([root?.span.id ?? ""]); + expect(span?.span.type).toBe("llm"); + expect(metadata).toMatchObject({ + provider: "mistral", + agentId: expect.any(String), + }); + if (typeof metadata?.model === "string") { + expect(metadata.model).toBe(AGENT_MODEL); + } + expect(input).toEqual(expect.any(Array)); + expect(input?.[0]?.role).toBe("user"); + expect(span?.metrics?.time_to_first_token).toEqual(expect.any(Number)); + expect(span?.output).toBeDefined(); + }); + + test("captures trace for embeddings.create()", testConfig, () => { + const root = findLatestSpan(events, ROOT_NAME); + const operation = findLatestSpan(events, "mistral-embeddings-operation"); + const span = findMistralSpan(events, operation?.span.id, [ + "mistral.embeddings.create", + ]); + const output = span?.output as { embedding_length?: number } | undefined; + + expect(operation).toBeDefined(); + expect(span).toBeDefined(); + expect(operation?.span.parentIds).toEqual([root?.span.id ?? ""]); + expect(span?.span.type).toBe("llm"); + expect(span?.row.metadata).toMatchObject({ + model: EMBEDDING_MODEL, + provider: "mistral", + }); + expect(output?.embedding_length).toEqual(expect.any(Number)); + expect(output?.embedding_length).toBeGreaterThan(0); + }); + + test("matches the shared span snapshot", testConfig, async () => { + await expect( + formatJsonFileSnapshot(buildSpanSummary(events, options.snapshotName)), + ).toMatchFileSnapshot(spanSnapshotPath); + }); + + test("matches the shared payload snapshot", testConfig, async () => { + await expect( + formatJsonFileSnapshot( + buildPayloadSummary(events, payloads, options.snapshotName), + ), + ).toMatchFileSnapshot(payloadSnapshotPath); + }); + }); +} diff --git a/e2e/scenarios/mistral-instrumentation/constants.mjs b/e2e/scenarios/mistral-instrumentation/constants.mjs new file mode 100644 index 000000000..b4eb0ab83 --- /dev/null +++ b/e2e/scenarios/mistral-instrumentation/constants.mjs @@ -0,0 +1,15 @@ +const CHAT_MODEL = "mistral-small-latest"; +const EMBEDDING_MODEL = "mistral-embed"; +const FIM_MODEL = "codestral-latest"; +const AGENT_MODEL = CHAT_MODEL; +const ROOT_NAME = "mistral-root"; +const SCENARIO_NAME = "mistral-instrumentation"; + +export { + AGENT_MODEL, + CHAT_MODEL, + EMBEDDING_MODEL, + FIM_MODEL, + ROOT_NAME, + SCENARIO_NAME, +}; diff --git a/e2e/scenarios/mistral-instrumentation/package.json b/e2e/scenarios/mistral-instrumentation/package.json new file mode 100644 index 000000000..031cccf46 --- /dev/null +++ b/e2e/scenarios/mistral-instrumentation/package.json @@ -0,0 +1,20 @@ +{ + "name": "@braintrust/e2e-mistral-instrumentation", + "private": true, + "braintrustScenario": { + "canary": { + "dependencies": { + "mistral-sdk-v1": "@mistralai/mistralai@1", + "mistral-sdk-v2": "@mistralai/mistralai@2" + } + } + }, + "dependencies": { + "mistral-sdk-v1": "npm:@mistralai/mistralai@1.15.1", + "mistral-sdk-v1-10-0": "npm:@mistralai/mistralai@1.10.0", + "mistral-sdk-v1-14-1": "npm:@mistralai/mistralai@1.14.1", + "mistral-sdk-v1-15-1": "npm:@mistralai/mistralai@1.15.1", + "mistral-sdk-v1-3-4": "npm:@mistralai/mistralai@1.3.4", + "mistral-sdk-v2": "npm:@mistralai/mistralai@2.1.2" + } +} diff --git a/e2e/scenarios/mistral-instrumentation/pnpm-lock.yaml b/e2e/scenarios/mistral-instrumentation/pnpm-lock.yaml new file mode 100644 index 000000000..e94ea4b92 --- /dev/null +++ b/e2e/scenarios/mistral-instrumentation/pnpm-lock.yaml @@ -0,0 +1,122 @@ +lockfileVersion: '9.0' + +settings: + autoInstallPeers: true + excludeLinksFromLockfile: false + +importers: + + .: + dependencies: + mistral-sdk-v1: + specifier: npm:@mistralai/mistralai@1.15.1 + version: '@mistralai/mistralai@1.15.1' + mistral-sdk-v1-10-0: + specifier: npm:@mistralai/mistralai@1.10.0 + version: '@mistralai/mistralai@1.10.0' + mistral-sdk-v1-14-1: + specifier: npm:@mistralai/mistralai@1.14.1 + version: '@mistralai/mistralai@1.14.1' + mistral-sdk-v1-15-1: + specifier: npm:@mistralai/mistralai@1.15.1 + version: '@mistralai/mistralai@1.15.1' + mistral-sdk-v1-3-4: + specifier: npm:@mistralai/mistralai@1.3.4 + version: '@mistralai/mistralai@1.3.4(zod@4.3.6)' + mistral-sdk-v2: + specifier: npm:@mistralai/mistralai@2.1.2 + version: '@mistralai/mistralai@2.1.2' + +packages: + + '@mistralai/mistralai@1.10.0': + resolution: {integrity: sha512-tdIgWs4Le8vpvPiUEWne6tK0qbVc+jMenujnvTqOjogrJUsCSQhus0tHTU1avDDh5//Rq2dFgP9mWRAdIEoBqg==} + + '@mistralai/mistralai@1.14.1': + resolution: {integrity: sha512-IiLmmZFCCTReQgPAT33r7KQ1nYo5JPdvGkrkZqA8qQ2qB1GHgs5LoP5K2ICyrjnpw2n8oSxMM/VP+liiKcGNlQ==} + + '@mistralai/mistralai@1.15.1': + resolution: {integrity: sha512-fb995eiz3r0KsBGtRjFV+/iLbX+UpfalxpF+YitT3R6ukrPD4PN+FGwwmYcRFhNAzVzDUtTVxQYnjQWEnwV5nw==} + + '@mistralai/mistralai@1.3.4': + resolution: {integrity: sha512-db5UhCXqH0N05XbXMR/2bSiGKIFUzS6p0sI9Nl2XDmJuDZIm+WRGTlsq60ALwhvKpHcQKzN5L58HIneksRrn9g==} + peerDependencies: + zod: '>= 3' + + '@mistralai/mistralai@2.1.2': + resolution: {integrity: sha512-0NlXBNdTMmi4z0oSUWa8KseBHlli+/FkaUTaAyLYTJzkKh3mYM/D3DnJzRil8VD6QC7W3IteARjd3LnxAZGPUw==} + + ws@8.20.0: + resolution: {integrity: sha512-sAt8BhgNbzCtgGbt2OxmpuryO63ZoDk/sqaB/znQm94T4fCEsy/yV+7CdC1kJhOU9lboAEU7R3kquuycDoibVA==} + engines: {node: '>=10.0.0'} + peerDependencies: + bufferutil: ^4.0.1 + utf-8-validate: '>=5.0.2' + peerDependenciesMeta: + bufferutil: + optional: true + utf-8-validate: + optional: true + + zod-to-json-schema@3.25.2: + resolution: {integrity: sha512-O/PgfnpT1xKSDeQYSCfRI5Gy3hPf91mKVDuYLUHZJMiDFptvP41MSnWofm8dnCm0256ZNfZIM7DSzuSMAFnjHA==} + peerDependencies: + zod: ^3.25.28 || ^4 + + zod@3.25.76: + resolution: {integrity: sha512-gzUt/qt81nXsFGKIFcC3YnfEAx5NkunCfnDlvuBSSFS02bcXu4Lmea0AFIUwbLWxWPx3d9p8S5QoaujKcNQxcQ==} + + zod@4.3.6: + resolution: {integrity: sha512-rftlrkhHZOcjDwkGlnUtZZkvaPHCsDATp4pGpuOOMDaTdDDXF91wuVDJoWoPsKX/3YPQ5fHuF3STjcYyKr+Qhg==} + +snapshots: + + '@mistralai/mistralai@1.10.0': + dependencies: + zod: 3.25.76 + zod-to-json-schema: 3.25.2(zod@3.25.76) + + '@mistralai/mistralai@1.14.1': + dependencies: + ws: 8.20.0 + zod: 4.3.6 + zod-to-json-schema: 3.25.2(zod@4.3.6) + transitivePeerDependencies: + - bufferutil + - utf-8-validate + + '@mistralai/mistralai@1.15.1': + dependencies: + ws: 8.20.0 + zod: 4.3.6 + zod-to-json-schema: 3.25.2(zod@4.3.6) + transitivePeerDependencies: + - bufferutil + - utf-8-validate + + '@mistralai/mistralai@1.3.4(zod@4.3.6)': + dependencies: + zod: 4.3.6 + + '@mistralai/mistralai@2.1.2': + dependencies: + ws: 8.20.0 + zod: 4.3.6 + zod-to-json-schema: 3.25.2(zod@4.3.6) + transitivePeerDependencies: + - bufferutil + - utf-8-validate + + ws@8.20.0: {} + + zod-to-json-schema@3.25.2(zod@3.25.76): + dependencies: + zod: 3.25.76 + + zod-to-json-schema@3.25.2(zod@4.3.6): + dependencies: + zod: 4.3.6 + + zod@3.25.76: {} + + zod@4.3.6: {} diff --git a/e2e/scenarios/mistral-instrumentation/scenario.impl.mjs b/e2e/scenarios/mistral-instrumentation/scenario.impl.mjs new file mode 100644 index 000000000..ee61b661c --- /dev/null +++ b/e2e/scenarios/mistral-instrumentation/scenario.impl.mjs @@ -0,0 +1,681 @@ +import { wrapMistral } from "braintrust"; +import { + collectAsync, + runOperation, + runTracedScenario, +} from "../../helpers/provider-runtime.mjs"; +import { + AGENT_MODEL, + CHAT_MODEL, + EMBEDDING_MODEL, + FIM_MODEL, + ROOT_NAME, + SCENARIO_NAME, +} from "./constants.mjs"; + +export const MISTRAL_SCENARIO_TIMEOUT_MS = 240_000; +const TEST_TOOL_DELAY_MS = 50; +const MISTRAL_REQUEST_RETRY_OPTIONS = { + attempts: 5, + delayMs: 2_000, + maxDelayMs: 10_000, +}; +export const MISTRAL_SCENARIO_SPECS = [ + { + autoEntry: "scenario.mistral-v1-3-4.mjs", + dependencyName: "mistral-sdk-v1-3-4", + snapshotName: "mistral-v1-3-4", + wrapperEntry: "scenario.mistral-v1-3-4.ts", + }, + { + autoEntry: "scenario.mistral-v1-10-0.mjs", + dependencyName: "mistral-sdk-v1-10-0", + snapshotName: "mistral-v1-10-0", + wrapperEntry: "scenario.mistral-v1-10-0.ts", + }, + { + autoEntry: "scenario.mistral-v1-14-1.mjs", + dependencyName: "mistral-sdk-v1-14-1", + snapshotName: "mistral-v1-14-1", + wrapperEntry: "scenario.mistral-v1-14-1.ts", + }, + { + autoEntry: "scenario.mistral-v1-15-1.mjs", + dependencyName: "mistral-sdk-v1-15-1", + snapshotName: "mistral-v1-15-1", + wrapperEntry: "scenario.mistral-v1-15-1.ts", + }, + { + autoEntry: "scenario.mistral-v1.mjs", + dependencyName: "mistral-sdk-v1", + snapshotName: "mistral-v1", + wrapperEntry: "scenario.mistral-v1.ts", + }, + { + autoEntry: "scenario.mjs", + dependencyName: "mistral-sdk-v2", + snapshotName: "mistral-v2", + wrapperEntry: "scenario.ts", + }, +]; + +function nonEmptyString(value) { + return typeof value === "string" && value.trim().length > 0 + ? value.trim() + : null; +} + +function isMistralInputValidationError(error) { + return ( + error instanceof Error && + typeof error.message === "string" && + error.message.includes("Input validation failed") + ); +} + +function getWeatherToolDefinition({ legacy = false } = {}) { + return { + type: "function", + function: { + name: "get_weather", + description: "Get weather for a city.", + parameters: legacy + ? {} + : { + type: "object", + properties: { + location: { + type: "string", + description: "City name, e.g. Vienna.", + }, + }, + required: ["location"], + }, + }, + }; +} + +function getExchangeRateToolDefinition({ legacy = false } = {}) { + return { + type: "function", + function: { + name: "get_exchange_rate", + description: "Get currency exchange rate.", + parameters: legacy + ? {} + : { + type: "object", + properties: { + from_currency: { + type: "string", + description: "Base currency code, e.g. USD.", + }, + to_currency: { + type: "string", + description: "Target currency code, e.g. EUR.", + }, + }, + required: ["from_currency", "to_currency"], + }, + }, + }; +} + +function getAgentTimeToolDefinition({ legacy = false } = {}) { + return { + type: "function", + function: { + name: "get_time_in_city", + description: "Get the local time in a city.", + parameters: legacy + ? {} + : { + type: "object", + properties: { + city: { + type: "string", + description: "City name, e.g. Vienna.", + }, + }, + required: ["city"], + }, + }, + }; +} + +function isRecord(value) { + return typeof value === "object" && value !== null; +} + +function getAgentId(agent) { + if (!isRecord(agent)) { + return null; + } + + if (nonEmptyString(agent.id)) { + return agent.id.trim(); + } + + if (nonEmptyString(agent.agentId)) { + return agent.agentId.trim(); + } + + if (nonEmptyString(agent.agent_id)) { + return agent.agent_id.trim(); + } + + return null; +} + +function getMistralApiBaseUrl(client) { + const envBaseUrl = + nonEmptyString(process.env.MISTRAL_API_URL) || + nonEmptyString(process.env.MISTRAL_BASE_URL); + if (envBaseUrl) { + return envBaseUrl.replace(/\/+$/g, ""); + } + + const options = + isRecord(client) && isRecord(client._options) ? client._options : undefined; + const optionBaseUrl = + (isRecord(options) && nonEmptyString(options.serverURL)) || + (isRecord(options) && nonEmptyString(options.serverUrl)) || + (isRecord(options) && nonEmptyString(options.baseURL)) || + (isRecord(options) && nonEmptyString(options.baseUrl)); + + return (optionBaseUrl || "https://api.mistral.ai").replace(/\/+$/g, ""); +} + +function getAgentCreatePayload() { + return { + model: AGENT_MODEL, + name: `braintrust-e2e-${Date.now().toString(36)}`, + instructions: "You are concise. Keep responses under five words.", + }; +} + +async function withRetry( + callback, + { attempts = 3, delayMs = 1_000, maxDelayMs = Number.POSITIVE_INFINITY } = {}, +) { + let lastError; + for (let attempt = 1; attempt <= attempts; attempt++) { + try { + return await callback(); + } catch (error) { + lastError = error; + if (attempt === attempts) { + throw error; + } + const retryDelayMs = Math.min(delayMs * attempt, maxDelayMs); + await new Promise((resolve) => setTimeout(resolve, retryDelayMs)); + } + } + + throw lastError; +} + +async function simulateToolExecutionDelay() { + await new Promise((resolve) => setTimeout(resolve, TEST_TOOL_DELAY_MS)); +} + +async function createAgentViaHttp(client, apiKey) { + const baseUrl = getMistralApiBaseUrl(client); + const response = await withRetry( + async () => + fetch(`${baseUrl}/v1/agents`, { + body: JSON.stringify(getAgentCreatePayload()), + headers: { + Authorization: `Bearer ${apiKey}`, + "Content-Type": "application/json", + }, + method: "POST", + }), + MISTRAL_REQUEST_RETRY_OPTIONS, + ); + const responseBody = await response.text(); + if (!response.ok) { + throw new Error( + `Failed to create temporary Mistral agent (${response.status}): ${responseBody}`, + ); + } + + const parsed = JSON.parse(responseBody); + const createdAgentId = getAgentId(parsed); + if (!createdAgentId) { + throw new Error("Mistral agent creation response did not include an id."); + } + + return { + agentId: createdAgentId, + cleanup: async () => { + try { + await fetch(`${baseUrl}/v1/agents/${createdAgentId}`, { + headers: { + Authorization: `Bearer ${apiKey}`, + }, + method: "DELETE", + }); + } catch { + // Ignore cleanup failures for temporary e2e agents. + } + }, + }; +} + +async function createAgentViaSdk(client, apiKey) { + const beta = isRecord(client) && isRecord(client.beta) ? client.beta : null; + const agentManager = beta && isRecord(beta.agents) ? beta.agents : null; + const createAgent = agentManager?.create; + if (typeof createAgent !== "function") { + return null; + } + + const created = await withRetry( + async () => createAgent.call(agentManager, getAgentCreatePayload()), + MISTRAL_REQUEST_RETRY_OPTIONS, + ); + const createdAgentId = getAgentId(created); + if (!createdAgentId) { + throw new Error("beta.agents.create() did not return an agent id."); + } + + const deleteAgent = agentManager?.delete; + if (typeof deleteAgent === "function") { + return { + agentId: createdAgentId, + cleanup: async () => { + try { + await deleteAgent.call(agentManager, { agentId: createdAgentId }); + } catch { + // Ignore cleanup failures for temporary e2e agents. + } + }, + }; + } + + const baseUrl = getMistralApiBaseUrl(client); + return { + agentId: createdAgentId, + cleanup: async () => { + try { + await fetch(`${baseUrl}/v1/agents/${createdAgentId}`, { + headers: { + Authorization: `Bearer ${apiKey}`, + }, + method: "DELETE", + }); + } catch { + // Ignore cleanup failures for temporary e2e agents. + } + }, + }; +} + +async function resolveAgentRuntime(client) { + const configuredAgentId = nonEmptyString(process.env.MISTRAL_AGENT_ID); + if (configuredAgentId) { + return { + agentId: configuredAgentId, + cleanup: async () => {}, + }; + } + + const apiKey = nonEmptyString(process.env.MISTRAL_API_KEY); + if (!apiKey) { + throw new Error("MISTRAL_API_KEY is required for Mistral e2e scenarios."); + } + + try { + const sdkRuntime = await createAgentViaSdk(client, apiKey); + if (sdkRuntime) { + return sdkRuntime; + } + } catch { + // Fall back to direct API provisioning when SDK beta management is unavailable. + } + + return await createAgentViaHttp(client, apiKey); +} + +async function runMistralInstrumentationScenario( + Mistral, + { decorateClient } = {}, +) { + const baseClient = new Mistral({ + apiKey: process.env.MISTRAL_API_KEY, + }); + const client = decorateClient ? decorateClient(baseClient) : baseClient; + const { agentId, cleanup } = await resolveAgentRuntime(baseClient); + + try { + await runTracedScenario({ + callback: async () => { + await runOperation( + "mistral-chat-complete-operation", + "chat-complete", + async () => { + await withRetry( + async () => + client.chat.complete({ + model: CHAT_MODEL, + messages: [ + { + role: "system", + content: + "You are concise. Keep responses under five words.", + }, + { + role: "user", + content: "Reply with exactly: observability", + }, + ], + maxTokens: 16, + temperature: 0, + }), + MISTRAL_REQUEST_RETRY_OPTIONS, + ); + }, + ); + + await runOperation( + "mistral-chat-stream-operation", + "chat-stream", + async () => { + await withRetry(async () => { + const stream = await client.chat.stream({ + model: CHAT_MODEL, + messages: [ + { + role: "user", + content: "Reply with exactly: streamed output", + }, + ], + maxTokens: 24, + stream: true, + temperature: 0, + }); + await collectAsync(stream); + }, MISTRAL_REQUEST_RETRY_OPTIONS); + }, + ); + + await runOperation( + "mistral-chat-tool-call-operation", + "chat-tool-call", + async () => { + await withRetry(async () => { + const request = { + model: CHAT_MODEL, + messages: [ + { + role: "user", + content: + "Call the get_weather tool for Vienna. Do not answer with plain text.", + }, + ], + toolChoice: "required", + maxTokens: 48, + temperature: 0, + }; + + try { + return await client.chat.complete({ + ...request, + tools: [getWeatherToolDefinition()], + }); + } catch (error) { + if (!isMistralInputValidationError(error)) { + throw error; + } + + return await client.chat.complete({ + ...request, + tools: [getWeatherToolDefinition({ legacy: true })], + }); + } + }, MISTRAL_REQUEST_RETRY_OPTIONS); + await simulateToolExecutionDelay(); + + await withRetry(async () => { + const request = { + model: CHAT_MODEL, + messages: [ + { + role: "user", + content: + "Call the get_exchange_rate tool for USD to EUR. Do not answer with plain text.", + }, + ], + toolChoice: "required", + maxTokens: 48, + temperature: 0, + }; + + try { + return await client.chat.complete({ + ...request, + tools: [getExchangeRateToolDefinition()], + }); + } catch (error) { + if (!isMistralInputValidationError(error)) { + throw error; + } + + return await client.chat.complete({ + ...request, + tools: [getExchangeRateToolDefinition({ legacy: true })], + }); + } + }, MISTRAL_REQUEST_RETRY_OPTIONS); + await simulateToolExecutionDelay(); + + await withRetry(async () => { + const request = { + model: CHAT_MODEL, + messages: [ + { + role: "system", + content: + "You must return only tool calls and no plain text.", + }, + { + role: "user", + content: + "In a single assistant response, call exactly two tools: get_weather with location Vienna and get_exchange_rate with from_currency USD and to_currency EUR.", + }, + ], + toolChoice: "required", + maxTokens: 96, + temperature: 0, + }; + + try { + return await client.chat.complete({ + ...request, + tools: [ + getWeatherToolDefinition(), + getExchangeRateToolDefinition(), + ], + }); + } catch (error) { + if (!isMistralInputValidationError(error)) { + throw error; + } + + return await client.chat.complete({ + ...request, + tools: [ + getWeatherToolDefinition({ legacy: true }), + getExchangeRateToolDefinition({ legacy: true }), + ], + }); + } + }, MISTRAL_REQUEST_RETRY_OPTIONS); + await simulateToolExecutionDelay(); + }, + ); + + await runOperation( + "mistral-fim-complete-operation", + "fim-complete", + async () => { + await withRetry( + async () => + client.fim.complete({ + model: FIM_MODEL, + prompt: "function add(a, b) {", + suffix: "}", + maxTokens: 24, + temperature: 0, + }), + MISTRAL_REQUEST_RETRY_OPTIONS, + ); + }, + ); + + await runOperation( + "mistral-fim-stream-operation", + "fim-stream", + async () => { + await withRetry(async () => { + const stream = await client.fim.stream({ + model: FIM_MODEL, + prompt: "const project = ", + suffix: ";", + maxTokens: 16, + stream: true, + temperature: 0, + }); + await collectAsync(stream); + }, MISTRAL_REQUEST_RETRY_OPTIONS); + }, + ); + + await runOperation( + "mistral-agents-complete-operation", + "agents-complete", + async () => { + await withRetry( + async () => + client.agents.complete({ + agentId, + messages: [ + { + role: "user", + content: "Reply with exactly: agent complete", + }, + ], + responseFormat: { + type: "text", + }, + maxTokens: 12, + temperature: 0, + }), + MISTRAL_REQUEST_RETRY_OPTIONS, + ); + }, + ); + + await runOperation( + "mistral-agents-tool-call-operation", + "agents-tool-call", + async () => { + await withRetry(async () => { + const request = { + agentId, + messages: [ + { + role: "user", + content: + "Call the get_time_in_city tool for Vienna. Do not answer with plain text.", + }, + ], + responseFormat: { + type: "text", + }, + toolChoice: "required", + maxTokens: 32, + temperature: 0, + }; + + try { + return await client.agents.complete({ + ...request, + tools: [getAgentTimeToolDefinition()], + }); + } catch (error) { + if (!isMistralInputValidationError(error)) { + throw error; + } + + return await client.agents.complete({ + ...request, + tools: [getAgentTimeToolDefinition({ legacy: true })], + }); + } + }, MISTRAL_REQUEST_RETRY_OPTIONS); + await simulateToolExecutionDelay(); + }, + ); + + await runOperation( + "mistral-agents-stream-operation", + "agents-stream", + async () => { + await withRetry(async () => { + const stream = await client.agents.stream({ + agentId, + messages: [ + { + role: "user", + content: "Reply with exactly: agent stream", + }, + ], + responseFormat: { + type: "text", + }, + maxTokens: 12, + stream: true, + temperature: 0, + }); + await collectAsync(stream); + }, MISTRAL_REQUEST_RETRY_OPTIONS); + }, + ); + + await runOperation( + "mistral-embeddings-operation", + "embeddings-create", + async () => { + await withRetry( + async () => + client.embeddings.create({ + model: EMBEDDING_MODEL, + inputs: "braintrust mistral instrumentation", + }), + MISTRAL_REQUEST_RETRY_OPTIONS, + ); + }, + ); + }, + metadata: { + scenario: SCENARIO_NAME, + }, + projectNameBase: "e2e-mistral-instrumentation", + rootName: ROOT_NAME, + }); + } finally { + await cleanup(); + } +} + +export async function runWrappedMistralInstrumentation(Mistral) { + await runMistralInstrumentationScenario(Mistral, { + decorateClient: wrapMistral, + }); +} + +export async function runAutoMistralInstrumentation(Mistral) { + await runMistralInstrumentationScenario(Mistral); +} diff --git a/e2e/scenarios/mistral-instrumentation/scenario.mistral-v1-10-0.mjs b/e2e/scenarios/mistral-instrumentation/scenario.mistral-v1-10-0.mjs new file mode 100644 index 000000000..0e5445b21 --- /dev/null +++ b/e2e/scenarios/mistral-instrumentation/scenario.mistral-v1-10-0.mjs @@ -0,0 +1,5 @@ +import { Mistral } from "mistral-sdk-v1-10-0"; +import { runMain } from "../../helpers/provider-runtime.mjs"; +import { runAutoMistralInstrumentation } from "./scenario.impl.mjs"; + +runMain(async () => runAutoMistralInstrumentation(Mistral)); diff --git a/e2e/scenarios/mistral-instrumentation/scenario.mistral-v1-10-0.ts b/e2e/scenarios/mistral-instrumentation/scenario.mistral-v1-10-0.ts new file mode 100644 index 000000000..db6605274 --- /dev/null +++ b/e2e/scenarios/mistral-instrumentation/scenario.mistral-v1-10-0.ts @@ -0,0 +1,5 @@ +import { Mistral } from "mistral-sdk-v1-10-0"; +import { runMain } from "../../helpers/scenario-runtime"; +import { runWrappedMistralInstrumentation } from "./scenario.impl.mjs"; + +runMain(async () => runWrappedMistralInstrumentation(Mistral)); diff --git a/e2e/scenarios/mistral-instrumentation/scenario.mistral-v1-14-1.mjs b/e2e/scenarios/mistral-instrumentation/scenario.mistral-v1-14-1.mjs new file mode 100644 index 000000000..b7ae746f9 --- /dev/null +++ b/e2e/scenarios/mistral-instrumentation/scenario.mistral-v1-14-1.mjs @@ -0,0 +1,5 @@ +import { Mistral } from "mistral-sdk-v1-14-1"; +import { runMain } from "../../helpers/provider-runtime.mjs"; +import { runAutoMistralInstrumentation } from "./scenario.impl.mjs"; + +runMain(async () => runAutoMistralInstrumentation(Mistral)); diff --git a/e2e/scenarios/mistral-instrumentation/scenario.mistral-v1-14-1.ts b/e2e/scenarios/mistral-instrumentation/scenario.mistral-v1-14-1.ts new file mode 100644 index 000000000..5b882da41 --- /dev/null +++ b/e2e/scenarios/mistral-instrumentation/scenario.mistral-v1-14-1.ts @@ -0,0 +1,5 @@ +import { Mistral } from "mistral-sdk-v1-14-1"; +import { runMain } from "../../helpers/scenario-runtime"; +import { runWrappedMistralInstrumentation } from "./scenario.impl.mjs"; + +runMain(async () => runWrappedMistralInstrumentation(Mistral)); diff --git a/e2e/scenarios/mistral-instrumentation/scenario.mistral-v1-15-1.mjs b/e2e/scenarios/mistral-instrumentation/scenario.mistral-v1-15-1.mjs new file mode 100644 index 000000000..57938f2b6 --- /dev/null +++ b/e2e/scenarios/mistral-instrumentation/scenario.mistral-v1-15-1.mjs @@ -0,0 +1,5 @@ +import { Mistral } from "mistral-sdk-v1-15-1"; +import { runMain } from "../../helpers/provider-runtime.mjs"; +import { runAutoMistralInstrumentation } from "./scenario.impl.mjs"; + +runMain(async () => runAutoMistralInstrumentation(Mistral)); diff --git a/e2e/scenarios/mistral-instrumentation/scenario.mistral-v1-15-1.ts b/e2e/scenarios/mistral-instrumentation/scenario.mistral-v1-15-1.ts new file mode 100644 index 000000000..ca52f313f --- /dev/null +++ b/e2e/scenarios/mistral-instrumentation/scenario.mistral-v1-15-1.ts @@ -0,0 +1,5 @@ +import { Mistral } from "mistral-sdk-v1-15-1"; +import { runMain } from "../../helpers/scenario-runtime"; +import { runWrappedMistralInstrumentation } from "./scenario.impl.mjs"; + +runMain(async () => runWrappedMistralInstrumentation(Mistral)); diff --git a/e2e/scenarios/mistral-instrumentation/scenario.mistral-v1-3-4.mjs b/e2e/scenarios/mistral-instrumentation/scenario.mistral-v1-3-4.mjs new file mode 100644 index 000000000..4f772e22d --- /dev/null +++ b/e2e/scenarios/mistral-instrumentation/scenario.mistral-v1-3-4.mjs @@ -0,0 +1,5 @@ +import { Mistral } from "mistral-sdk-v1-3-4"; +import { runMain } from "../../helpers/provider-runtime.mjs"; +import { runAutoMistralInstrumentation } from "./scenario.impl.mjs"; + +runMain(async () => runAutoMistralInstrumentation(Mistral)); diff --git a/e2e/scenarios/mistral-instrumentation/scenario.mistral-v1-3-4.ts b/e2e/scenarios/mistral-instrumentation/scenario.mistral-v1-3-4.ts new file mode 100644 index 000000000..1db41c17f --- /dev/null +++ b/e2e/scenarios/mistral-instrumentation/scenario.mistral-v1-3-4.ts @@ -0,0 +1,5 @@ +import { Mistral } from "mistral-sdk-v1-3-4"; +import { runMain } from "../../helpers/scenario-runtime"; +import { runWrappedMistralInstrumentation } from "./scenario.impl.mjs"; + +runMain(async () => runWrappedMistralInstrumentation(Mistral)); diff --git a/e2e/scenarios/mistral-instrumentation/scenario.mistral-v1.mjs b/e2e/scenarios/mistral-instrumentation/scenario.mistral-v1.mjs new file mode 100644 index 000000000..53db8ddb8 --- /dev/null +++ b/e2e/scenarios/mistral-instrumentation/scenario.mistral-v1.mjs @@ -0,0 +1,5 @@ +import { Mistral } from "mistral-sdk-v1"; +import { runMain } from "../../helpers/provider-runtime.mjs"; +import { runAutoMistralInstrumentation } from "./scenario.impl.mjs"; + +runMain(async () => runAutoMistralInstrumentation(Mistral)); diff --git a/e2e/scenarios/mistral-instrumentation/scenario.mistral-v1.ts b/e2e/scenarios/mistral-instrumentation/scenario.mistral-v1.ts new file mode 100644 index 000000000..994487e1e --- /dev/null +++ b/e2e/scenarios/mistral-instrumentation/scenario.mistral-v1.ts @@ -0,0 +1,5 @@ +import { Mistral } from "mistral-sdk-v1"; +import { runMain } from "../../helpers/scenario-runtime"; +import { runWrappedMistralInstrumentation } from "./scenario.impl.mjs"; + +runMain(async () => runWrappedMistralInstrumentation(Mistral)); diff --git a/e2e/scenarios/mistral-instrumentation/scenario.mjs b/e2e/scenarios/mistral-instrumentation/scenario.mjs new file mode 100644 index 000000000..d4eb85870 --- /dev/null +++ b/e2e/scenarios/mistral-instrumentation/scenario.mjs @@ -0,0 +1,5 @@ +import { Mistral } from "mistral-sdk-v2"; +import { runMain } from "../../helpers/provider-runtime.mjs"; +import { runAutoMistralInstrumentation } from "./scenario.impl.mjs"; + +runMain(async () => runAutoMistralInstrumentation(Mistral)); diff --git a/e2e/scenarios/mistral-instrumentation/scenario.test.ts b/e2e/scenarios/mistral-instrumentation/scenario.test.ts new file mode 100644 index 000000000..77b1da4f6 --- /dev/null +++ b/e2e/scenarios/mistral-instrumentation/scenario.test.ts @@ -0,0 +1,60 @@ +import { describe } from "vitest"; +import { + prepareScenarioDir, + readInstalledPackageVersion, + resolveScenarioDir, +} from "../../helpers/scenario-harness"; +import { defineMistralInstrumentationAssertions } from "./assertions"; +import { + MISTRAL_SCENARIO_SPECS, + MISTRAL_SCENARIO_TIMEOUT_MS, +} from "./scenario.impl.mjs"; + +const scenarioDir = await prepareScenarioDir({ + scenarioDir: resolveScenarioDir(import.meta.url), +}); + +const mistralScenarios = await Promise.all( + MISTRAL_SCENARIO_SPECS.map(async (scenario) => ({ + ...scenario, + version: await readInstalledPackageVersion( + scenarioDir, + scenario.dependencyName, + ), + })), +); + +for (const scenario of mistralScenarios) { + describe(`mistral sdk ${scenario.version}`, () => { + defineMistralInstrumentationAssertions({ + name: "wrapped instrumentation", + runScenario: async ({ runScenarioDir }) => { + await runScenarioDir({ + entry: scenario.wrapperEntry, + runContext: { variantKey: scenario.snapshotName }, + scenarioDir, + timeoutMs: MISTRAL_SCENARIO_TIMEOUT_MS, + }); + }, + snapshotName: scenario.snapshotName, + testFileUrl: import.meta.url, + timeoutMs: MISTRAL_SCENARIO_TIMEOUT_MS, + }); + + defineMistralInstrumentationAssertions({ + name: "auto-hook instrumentation", + runScenario: async ({ runNodeScenarioDir }) => { + await runNodeScenarioDir({ + entry: scenario.autoEntry, + nodeArgs: ["--import", "braintrust/hook.mjs"], + runContext: { variantKey: scenario.snapshotName }, + scenarioDir, + timeoutMs: MISTRAL_SCENARIO_TIMEOUT_MS, + }); + }, + snapshotName: scenario.snapshotName, + testFileUrl: import.meta.url, + timeoutMs: MISTRAL_SCENARIO_TIMEOUT_MS, + }); + }); +} diff --git a/e2e/scenarios/mistral-instrumentation/scenario.ts b/e2e/scenarios/mistral-instrumentation/scenario.ts new file mode 100644 index 000000000..62f148ed9 --- /dev/null +++ b/e2e/scenarios/mistral-instrumentation/scenario.ts @@ -0,0 +1,5 @@ +import { Mistral } from "mistral-sdk-v2"; +import { runMain } from "../../helpers/scenario-runtime"; +import { runWrappedMistralInstrumentation } from "./scenario.impl.mjs"; + +runMain(async () => runWrappedMistralInstrumentation(Mistral)); diff --git a/e2e/scripts/run-canary-tests-docker.mjs b/e2e/scripts/run-canary-tests-docker.mjs index cc96cc0fd..130980697 100644 --- a/e2e/scripts/run-canary-tests-docker.mjs +++ b/e2e/scripts/run-canary-tests-docker.mjs @@ -21,6 +21,7 @@ const ALLOWED_ENV_KEYS = [ "OPENAI_API_KEY", "OPENAI_BASE_URL", "OPENROUTER_API_KEY", + "MISTRAL_API_KEY", ]; function getAllowedEnv() { diff --git a/js/src/auto-instrumentations/bundler/plugin.ts b/js/src/auto-instrumentations/bundler/plugin.ts index 29a25eec0..dd2f1efe2 100644 --- a/js/src/auto-instrumentations/bundler/plugin.ts +++ b/js/src/auto-instrumentations/bundler/plugin.ts @@ -26,6 +26,7 @@ import { aiSDKConfigs } from "../configs/ai-sdk"; import { claudeAgentSDKConfigs } from "../configs/claude-agent-sdk"; import { googleGenAIConfigs } from "../configs/google-genai"; import { openRouterConfigs } from "../configs/openrouter"; +import { mistralConfigs } from "../configs/mistral"; export interface BundlerPluginOptions { /** @@ -73,6 +74,7 @@ export const unplugin = createUnplugin((options = {}) => { ...claudeAgentSDKConfigs, ...googleGenAIConfigs, ...openRouterConfigs, + ...mistralConfigs, ...(options.instrumentations || []), ]; diff --git a/js/src/auto-instrumentations/bundler/webpack-loader.ts b/js/src/auto-instrumentations/bundler/webpack-loader.ts index 3c2f8caec..29f3fc7ac 100644 --- a/js/src/auto-instrumentations/bundler/webpack-loader.ts +++ b/js/src/auto-instrumentations/bundler/webpack-loader.ts @@ -35,6 +35,7 @@ import { aiSDKConfigs } from "../configs/ai-sdk"; import { claudeAgentSDKConfigs } from "../configs/claude-agent-sdk"; import { googleGenAIConfigs } from "../configs/google-genai"; import { openRouterConfigs } from "../configs/openrouter"; +import { mistralConfigs } from "../configs/mistral"; import { type BundlerPluginOptions } from "./plugin"; /** @@ -68,6 +69,7 @@ function getMatcher(options: BundlerPluginOptions): InstrumentationMatcher { ...claudeAgentSDKConfigs, ...googleGenAIConfigs, ...openRouterConfigs, + ...mistralConfigs, ...(options.instrumentations ?? []), ]; const dcModule = options.browser ? "dc-browser" : undefined; diff --git a/js/src/auto-instrumentations/configs/claude-agent-sdk.test.ts b/js/src/auto-instrumentations/configs/claude-agent-sdk.test.ts deleted file mode 100644 index 5c71ea78b..000000000 --- a/js/src/auto-instrumentations/configs/claude-agent-sdk.test.ts +++ /dev/null @@ -1,36 +0,0 @@ -import { describe, expect, it } from "vitest"; -import { claudeAgentSDKConfigs } from "./claude-agent-sdk"; -import { claudeAgentSDKChannels } from "../../instrumentation/plugins/claude-agent-sdk-channels"; - -describe("claudeAgentSDKConfigs", () => { - it("registers sync query instrumentation for 0.1.x", () => { - expect(claudeAgentSDKConfigs).toContainEqual({ - channelName: claudeAgentSDKChannels.query.channelName, - module: { - name: "@anthropic-ai/claude-agent-sdk", - versionRange: ">=0.1.0 <0.2.0", - filePath: "sdk.mjs", - }, - functionQuery: { - functionName: "query", - kind: "Sync", - }, - }); - }); - - it("registers export-alias query instrumentation for 0.2.x", () => { - expect(claudeAgentSDKConfigs).toContainEqual({ - channelName: claudeAgentSDKChannels.query.channelName, - module: { - name: "@anthropic-ai/claude-agent-sdk", - versionRange: ">=0.2.0", - filePath: "sdk.mjs", - }, - functionQuery: { - functionName: "query", - kind: "Sync", - isExportAlias: true, - }, - }); - }); -}); diff --git a/js/src/auto-instrumentations/configs/mistral.ts b/js/src/auto-instrumentations/configs/mistral.ts new file mode 100644 index 000000000..896f54db1 --- /dev/null +++ b/js/src/auto-instrumentations/configs/mistral.ts @@ -0,0 +1,200 @@ +import type { InstrumentationConfig } from "@apm-js-collab/code-transformer"; +import { mistralChannels } from "../../instrumentation/plugins/mistral-channels"; + +export const mistralConfigs: InstrumentationConfig[] = [ + { + channelName: mistralChannels.chatComplete.channelName, + module: { + name: "@mistralai/mistralai", + versionRange: ">=1.0.0 <2.0.0", + filePath: "sdk/chat.js", + }, + functionQuery: { + className: "Chat", + methodName: "complete", + kind: "Async", + }, + }, + + { + channelName: mistralChannels.chatComplete.channelName, + module: { + name: "@mistralai/mistralai", + versionRange: ">=2.0.0 <3.0.0", + filePath: "esm/sdk/chat.js", + }, + functionQuery: { + className: "Chat", + methodName: "complete", + kind: "Async", + }, + }, + + { + channelName: mistralChannels.chatStream.channelName, + module: { + name: "@mistralai/mistralai", + versionRange: ">=1.0.0 <2.0.0", + filePath: "sdk/chat.js", + }, + functionQuery: { + className: "Chat", + methodName: "stream", + kind: "Async", + }, + }, + + { + channelName: mistralChannels.chatStream.channelName, + module: { + name: "@mistralai/mistralai", + versionRange: ">=2.0.0 <3.0.0", + filePath: "esm/sdk/chat.js", + }, + functionQuery: { + className: "Chat", + methodName: "stream", + kind: "Async", + }, + }, + + { + channelName: mistralChannels.embeddingsCreate.channelName, + module: { + name: "@mistralai/mistralai", + versionRange: ">=1.0.0 <2.0.0", + filePath: "sdk/embeddings.js", + }, + functionQuery: { + className: "Embeddings", + methodName: "create", + kind: "Async", + }, + }, + + { + channelName: mistralChannels.embeddingsCreate.channelName, + module: { + name: "@mistralai/mistralai", + versionRange: ">=2.0.0 <3.0.0", + filePath: "esm/sdk/embeddings.js", + }, + functionQuery: { + className: "Embeddings", + methodName: "create", + kind: "Async", + }, + }, + + { + channelName: mistralChannels.fimComplete.channelName, + module: { + name: "@mistralai/mistralai", + versionRange: ">=1.0.0 <2.0.0", + filePath: "sdk/fim.js", + }, + functionQuery: { + className: "Fim", + methodName: "complete", + kind: "Async", + }, + }, + + { + channelName: mistralChannels.fimComplete.channelName, + module: { + name: "@mistralai/mistralai", + versionRange: ">=2.0.0 <3.0.0", + filePath: "esm/sdk/fim.js", + }, + functionQuery: { + className: "Fim", + methodName: "complete", + kind: "Async", + }, + }, + + { + channelName: mistralChannels.fimStream.channelName, + module: { + name: "@mistralai/mistralai", + versionRange: ">=1.0.0 <2.0.0", + filePath: "sdk/fim.js", + }, + functionQuery: { + className: "Fim", + methodName: "stream", + kind: "Async", + }, + }, + + { + channelName: mistralChannels.fimStream.channelName, + module: { + name: "@mistralai/mistralai", + versionRange: ">=2.0.0 <3.0.0", + filePath: "esm/sdk/fim.js", + }, + functionQuery: { + className: "Fim", + methodName: "stream", + kind: "Async", + }, + }, + + { + channelName: mistralChannels.agentsComplete.channelName, + module: { + name: "@mistralai/mistralai", + versionRange: ">=1.0.0 <2.0.0", + filePath: "sdk/agents.js", + }, + functionQuery: { + className: "Agents", + methodName: "complete", + kind: "Async", + }, + }, + + { + channelName: mistralChannels.agentsComplete.channelName, + module: { + name: "@mistralai/mistralai", + versionRange: ">=2.0.0 <3.0.0", + filePath: "esm/sdk/agents.js", + }, + functionQuery: { + className: "Agents", + methodName: "complete", + kind: "Async", + }, + }, + + { + channelName: mistralChannels.agentsStream.channelName, + module: { + name: "@mistralai/mistralai", + versionRange: ">=1.0.0 <2.0.0", + filePath: "sdk/agents.js", + }, + functionQuery: { + className: "Agents", + methodName: "stream", + kind: "Async", + }, + }, + + { + channelName: mistralChannels.agentsStream.channelName, + module: { + name: "@mistralai/mistralai", + versionRange: ">=2.0.0 <3.0.0", + filePath: "esm/sdk/agents.js", + }, + functionQuery: { + className: "Agents", + methodName: "stream", + kind: "Async", + }, + }, +]; diff --git a/js/src/auto-instrumentations/configs/openrouter.test.ts b/js/src/auto-instrumentations/configs/openrouter.test.ts deleted file mode 100644 index aef665be9..000000000 --- a/js/src/auto-instrumentations/configs/openrouter.test.ts +++ /dev/null @@ -1,21 +0,0 @@ -import { describe, expect, it } from "vitest"; -import { openRouterConfigs } from "./openrouter"; -import { openRouterChannels } from "../../instrumentation/plugins/openrouter-channels"; - -describe("openRouterConfigs", () => { - it("registers auto-instrumentation for OpenRouter.callModel", () => { - expect(openRouterConfigs).toContainEqual({ - channelName: openRouterChannels.callModel.channelName, - module: { - name: "@openrouter/sdk", - versionRange: ">=0.9.11 <1.0.0", - filePath: "esm/sdk/sdk.js", - }, - functionQuery: { - className: "OpenRouter", - methodName: "callModel", - kind: "Sync", - }, - }); - }); -}); diff --git a/js/src/auto-instrumentations/hook.mts b/js/src/auto-instrumentations/hook.mts index d7f656baa..db202a0a6 100644 --- a/js/src/auto-instrumentations/hook.mts +++ b/js/src/auto-instrumentations/hook.mts @@ -20,6 +20,7 @@ import { aiSDKConfigs } from "./configs/ai-sdk.js"; import { claudeAgentSDKConfigs } from "./configs/claude-agent-sdk.js"; import { googleGenAIConfigs } from "./configs/google-genai.js"; import { openRouterConfigs } from "./configs/openrouter.js"; +import { mistralConfigs } from "./configs/mistral.js"; import { ModulePatch } from "./loader/cjs-patch.js"; import { patchTracingChannel } from "./patch-tracing-channel.js"; @@ -38,6 +39,7 @@ const allConfigs = [ ...claudeAgentSDKConfigs, ...googleGenAIConfigs, ...openRouterConfigs, + ...mistralConfigs, ]; // 1. Register ESM loader for ESM modules diff --git a/js/src/auto-instrumentations/index.ts b/js/src/auto-instrumentations/index.ts index 2be27701d..87040081e 100644 --- a/js/src/auto-instrumentations/index.ts +++ b/js/src/auto-instrumentations/index.ts @@ -34,6 +34,7 @@ export { aiSDKConfigs } from "./configs/ai-sdk"; export { claudeAgentSDKConfigs } from "./configs/claude-agent-sdk"; export { googleGenAIConfigs } from "./configs/google-genai"; export { openRouterConfigs } from "./configs/openrouter"; +export { mistralConfigs } from "./configs/mistral"; // Re-export orchestrion configuration types // Note: ModuleMetadata and FunctionQuery are properties of InstrumentationConfig, diff --git a/js/src/auto-instrumentations/loader/cjs-patch.ts b/js/src/auto-instrumentations/loader/cjs-patch.ts index a29014e8e..127b5ef5b 100644 --- a/js/src/auto-instrumentations/loader/cjs-patch.ts +++ b/js/src/auto-instrumentations/loader/cjs-patch.ts @@ -7,7 +7,7 @@ import { create, type InstrumentationConfig, } from "@apm-js-collab/code-transformer"; -import * as Module from "node:module"; +import * as NodeModule from "node:module"; import { sep } from "node:path"; import moduleDetailsFromPath from "module-details-from-path"; import { getPackageName, getPackageVersion } from "./get-package-version.js"; @@ -15,14 +15,18 @@ import { getPackageName, getPackageVersion } from "./get-package-version.js"; export class ModulePatch { private packages: Set; private instrumentator: any; + private modulePrototype: { _compile?: (...args: any[]) => unknown }; private originalCompile: any; constructor({ instrumentations = [], }: { instrumentations?: InstrumentationConfig[] } = {}) { + const modulePrototype = resolveModulePrototype() as any; + this.packages = new Set(instrumentations.map((i) => i.module.name)); this.instrumentator = create(instrumentations); - this.originalCompile = (Module.prototype as any)._compile; + this.modulePrototype = modulePrototype; + this.originalCompile = modulePrototype._compile; } /** @@ -32,9 +36,7 @@ export class ModulePatch { */ patch() { const self = this; - (Module.prototype as any)._compile = function wrappedCompile( - ...args: any[] - ) { + this.modulePrototype._compile = function wrappedCompile(...args: any[]) { const [content, filename] = args; // Normalize path to platform-specific separator for module-details-from-path @@ -82,6 +84,19 @@ export class ModulePatch { * **Note**: This is intended to be used in testing only. */ unpatch() { - (Module.prototype as any)._compile = this.originalCompile; + this.modulePrototype._compile = this.originalCompile; } } + +function resolveModulePrototype(): + | { _compile?: (...args: any[]) => unknown } + | undefined { + const moduleCtor = (NodeModule as any).Module; + if (moduleCtor && typeof moduleCtor === "function") { + return moduleCtor.prototype as { _compile?: (...args: any[]) => unknown }; + } + + return (NodeModule as any).prototype as + | { _compile?: (...args: any[]) => unknown } + | undefined; +} diff --git a/js/src/auto-instrumentations/loader/esm-hook.mts b/js/src/auto-instrumentations/loader/esm-hook.mts index fceeff60d..4f6d16a28 100644 --- a/js/src/auto-instrumentations/loader/esm-hook.mts +++ b/js/src/auto-instrumentations/loader/esm-hook.mts @@ -11,7 +11,7 @@ import { type InstrumentationConfig, } from "@apm-js-collab/code-transformer"; import moduleDetailsFromPath from "module-details-from-path"; -import { getPackageVersion } from "./get-package-version.js"; +import { getPackageName, getPackageVersion } from "./get-package-version.js"; let instrumentator: any; let packages: Set; @@ -44,14 +44,19 @@ export async function resolve( const resolvedModule = moduleDetailsFromPath(normalizedForPlatform); - if (resolvedModule && packages?.has(resolvedModule.name)) { - const version = getPackageVersion(resolvedModule.basedir); + if (resolvedModule) { + const packageName = + getPackageName(resolvedModule.basedir) ?? resolvedModule.name; + if (!packages?.has(packageName)) { + return url; + } + const version = getPackageVersion(resolvedModule.basedir); // Normalize module path for WASM transformer (expects forward slashes) const normalizedModulePath = resolvedModule.path.replace(/\\/g, "/"); const transformer = instrumentator.getTransformer( - resolvedModule.name, + packageName, version, normalizedModulePath, ); diff --git a/js/src/auto-instrumentations/loader/get-package-version.ts b/js/src/auto-instrumentations/loader/get-package-version.ts index 25effb8ad..d2470060b 100644 --- a/js/src/auto-instrumentations/loader/get-package-version.ts +++ b/js/src/auto-instrumentations/loader/get-package-version.ts @@ -3,7 +3,7 @@ * If the package.json file cannot be read, it defaults to the Node.js version. */ -import { readFileSync } from "node:fs"; +import { readFileSync, realpathSync } from "node:fs"; import { join } from "node:path"; const packageVersions = new Map(); @@ -19,12 +19,36 @@ function readPackageJson(baseDir: string): Record | undefined { } } +function resolvePackageBaseDir(baseDir: string): string { + try { + return realpathSync(baseDir); + } catch { + return baseDir; + } +} + +function readPackageJsonWithFallback( + baseDir: string, +): Record | undefined { + const packageJson = readPackageJson(baseDir); + if (packageJson) { + return packageJson; + } + + const resolvedBaseDir = resolvePackageBaseDir(baseDir); + if (resolvedBaseDir === baseDir) { + return undefined; + } + + return readPackageJson(resolvedBaseDir); +} + export function getPackageVersion(baseDir: string): string { if (packageVersions.has(baseDir)) { return packageVersions.get(baseDir)!; } - const packageJson = readPackageJson(baseDir); + const packageJson = readPackageJsonWithFallback(baseDir); if (typeof packageJson?.version === "string") { packageVersions.set(baseDir, packageJson.version); return packageJson.version; @@ -38,7 +62,7 @@ export function getPackageName(baseDir: string): string | undefined { return packageNames.get(baseDir); } - const packageJson = readPackageJson(baseDir); + const packageJson = readPackageJsonWithFallback(baseDir); if (typeof packageJson?.name === "string") { packageNames.set(baseDir, packageJson.name); return packageJson.name; diff --git a/js/src/exports.ts b/js/src/exports.ts index c01c1e3cb..7f705238e 100644 --- a/js/src/exports.ts +++ b/js/src/exports.ts @@ -174,6 +174,7 @@ export { wrapMastraAgent } from "./wrappers/mastra"; export { wrapClaudeAgentSDK } from "./wrappers/claude-agent-sdk/claude-agent-sdk"; export { wrapGoogleGenAI } from "./wrappers/google-genai"; export { wrapOpenRouter } from "./wrappers/openrouter"; +export { wrapMistral } from "./wrappers/mistral"; export { wrapVitest } from "./wrappers/vitest"; export { initNodeTestSuite } from "./wrappers/node-test"; diff --git a/js/src/instrumentation/braintrust-plugin.test.ts b/js/src/instrumentation/braintrust-plugin.test.ts index 2e499a4c3..d8f6a5904 100644 --- a/js/src/instrumentation/braintrust-plugin.test.ts +++ b/js/src/instrumentation/braintrust-plugin.test.ts @@ -6,6 +6,7 @@ import { AISDKPlugin } from "./plugins/ai-sdk-plugin"; import { ClaudeAgentSDKPlugin } from "./plugins/claude-agent-sdk-plugin"; import { GoogleGenAIPlugin } from "./plugins/google-genai-plugin"; import { OpenRouterPlugin } from "./plugins/openrouter-plugin"; +import { MistralPlugin } from "./plugins/mistral-plugin"; function createPluginClassMock() { return vi.fn(function MockPlugin(this: { @@ -48,6 +49,10 @@ vi.mock("./plugins/openrouter-plugin", () => ({ OpenRouterPlugin: createPluginClassMock(), })); +vi.mock("./plugins/mistral-plugin", () => ({ + MistralPlugin: createPluginClassMock(), +})); + describe("BraintrustPlugin", () => { beforeEach(() => { vi.clearAllMocks(); @@ -109,6 +114,15 @@ describe("BraintrustPlugin", () => { expect(mockInstance.enable).toHaveBeenCalledTimes(1); }); + it("should create and enable Mistral plugin by default", () => { + const plugin = new BraintrustPlugin(); + plugin.enable(); + + expect(MistralPlugin).toHaveBeenCalledTimes(1); + const mockInstance = vi.mocked(MistralPlugin).mock.results[0].value; + expect(mockInstance.enable).toHaveBeenCalledTimes(1); + }); + it("should create all plugins when enabled with no config", () => { const plugin = new BraintrustPlugin(); plugin.enable(); @@ -119,6 +133,7 @@ describe("BraintrustPlugin", () => { expect(ClaudeAgentSDKPlugin).toHaveBeenCalledTimes(1); expect(GoogleGenAIPlugin).toHaveBeenCalledTimes(1); expect(OpenRouterPlugin).toHaveBeenCalledTimes(1); + expect(MistralPlugin).toHaveBeenCalledTimes(1); }); it("should create all plugins when enabled with empty config", () => { @@ -131,6 +146,7 @@ describe("BraintrustPlugin", () => { expect(ClaudeAgentSDKPlugin).toHaveBeenCalledTimes(1); expect(GoogleGenAIPlugin).toHaveBeenCalledTimes(1); expect(OpenRouterPlugin).toHaveBeenCalledTimes(1); + expect(MistralPlugin).toHaveBeenCalledTimes(1); }); it("should create all plugins when enabled with empty integrations config", () => { @@ -143,6 +159,7 @@ describe("BraintrustPlugin", () => { expect(ClaudeAgentSDKPlugin).toHaveBeenCalledTimes(1); expect(GoogleGenAIPlugin).toHaveBeenCalledTimes(1); expect(OpenRouterPlugin).toHaveBeenCalledTimes(1); + expect(MistralPlugin).toHaveBeenCalledTimes(1); }); }); @@ -190,6 +207,7 @@ describe("BraintrustPlugin", () => { expect(ClaudeAgentSDKPlugin).toHaveBeenCalledTimes(1); expect(GoogleGenAIPlugin).toHaveBeenCalledTimes(1); expect(OpenRouterPlugin).toHaveBeenCalledTimes(1); + expect(MistralPlugin).toHaveBeenCalledTimes(1); }); it("should not create Claude Agent SDK plugin when claudeAgentSDK: false", () => { @@ -205,6 +223,7 @@ describe("BraintrustPlugin", () => { expect(AISDKPlugin).toHaveBeenCalledTimes(1); expect(GoogleGenAIPlugin).toHaveBeenCalledTimes(1); expect(OpenRouterPlugin).toHaveBeenCalledTimes(1); + expect(MistralPlugin).toHaveBeenCalledTimes(1); }); it("should not create Google GenAI plugin when googleGenAI: false", () => { @@ -220,6 +239,7 @@ describe("BraintrustPlugin", () => { expect(AISDKPlugin).toHaveBeenCalledTimes(1); expect(ClaudeAgentSDKPlugin).toHaveBeenCalledTimes(1); expect(OpenRouterPlugin).toHaveBeenCalledTimes(1); + expect(MistralPlugin).toHaveBeenCalledTimes(1); }); it("should not create OpenRouter plugin when openrouter: false", () => { @@ -234,6 +254,22 @@ describe("BraintrustPlugin", () => { expect(AISDKPlugin).toHaveBeenCalledTimes(1); expect(ClaudeAgentSDKPlugin).toHaveBeenCalledTimes(1); expect(GoogleGenAIPlugin).toHaveBeenCalledTimes(1); + expect(MistralPlugin).toHaveBeenCalledTimes(1); + }); + + it("should not create Mistral plugin when mistral: false", () => { + const plugin = new BraintrustPlugin({ + integrations: { mistral: false }, + }); + plugin.enable(); + + expect(MistralPlugin).not.toHaveBeenCalled(); + expect(OpenAIPlugin).toHaveBeenCalledTimes(1); + expect(AnthropicPlugin).toHaveBeenCalledTimes(1); + expect(AISDKPlugin).toHaveBeenCalledTimes(1); + expect(ClaudeAgentSDKPlugin).toHaveBeenCalledTimes(1); + expect(GoogleGenAIPlugin).toHaveBeenCalledTimes(1); + expect(OpenRouterPlugin).toHaveBeenCalledTimes(1); }); it("should not create any plugins when all are disabled", () => { @@ -245,6 +281,7 @@ describe("BraintrustPlugin", () => { claudeAgentSDK: false, googleGenAI: false, openrouter: false, + mistral: false, }, }); plugin.enable(); @@ -255,6 +292,7 @@ describe("BraintrustPlugin", () => { expect(ClaudeAgentSDKPlugin).not.toHaveBeenCalled(); expect(GoogleGenAIPlugin).not.toHaveBeenCalled(); expect(OpenRouterPlugin).not.toHaveBeenCalled(); + expect(MistralPlugin).not.toHaveBeenCalled(); }); it("should allow selective enabling of plugins", () => { @@ -266,6 +304,7 @@ describe("BraintrustPlugin", () => { claudeAgentSDK: true, googleGenAI: false, openrouter: true, + mistral: false, }, }); plugin.enable(); @@ -276,6 +315,7 @@ describe("BraintrustPlugin", () => { expect(AnthropicPlugin).not.toHaveBeenCalled(); expect(AISDKPlugin).not.toHaveBeenCalled(); expect(GoogleGenAIPlugin).not.toHaveBeenCalled(); + expect(MistralPlugin).not.toHaveBeenCalled(); }); }); @@ -292,6 +332,8 @@ describe("BraintrustPlugin", () => { expect(AnthropicPlugin).toHaveBeenCalledTimes(1); expect(ClaudeAgentSDKPlugin).toHaveBeenCalledTimes(1); expect(GoogleGenAIPlugin).toHaveBeenCalledTimes(1); + expect(OpenRouterPlugin).toHaveBeenCalledTimes(1); + expect(MistralPlugin).toHaveBeenCalledTimes(1); }); it("should not create Google GenAI plugin when google: false (legacy)", () => { @@ -306,6 +348,8 @@ describe("BraintrustPlugin", () => { expect(AnthropicPlugin).toHaveBeenCalledTimes(1); expect(AISDKPlugin).toHaveBeenCalledTimes(1); expect(ClaudeAgentSDKPlugin).toHaveBeenCalledTimes(1); + expect(OpenRouterPlugin).toHaveBeenCalledTimes(1); + expect(MistralPlugin).toHaveBeenCalledTimes(1); }); it("should not create AI SDK plugin when both aisdk and vercel are false", () => { @@ -376,6 +420,7 @@ describe("BraintrustPlugin", () => { const googleGenAIMock = vi.mocked(GoogleGenAIPlugin).mock.results[0].value; const openRouterMock = vi.mocked(OpenRouterPlugin).mock.results[0].value; + const mistralMock = vi.mocked(MistralPlugin).mock.results[0].value; expect(openaiMock.enable).toHaveBeenCalledTimes(1); expect(anthropicMock.enable).toHaveBeenCalledTimes(1); @@ -383,6 +428,7 @@ describe("BraintrustPlugin", () => { expect(claudeAgentSDKMock.enable).toHaveBeenCalledTimes(1); expect(googleGenAIMock.enable).toHaveBeenCalledTimes(1); expect(openRouterMock.enable).toHaveBeenCalledTimes(1); + expect(mistralMock.enable).toHaveBeenCalledTimes(1); }); it("should disable and nullify all sub-plugins when disabled", () => { @@ -397,6 +443,7 @@ describe("BraintrustPlugin", () => { const googleGenAIMock = vi.mocked(GoogleGenAIPlugin).mock.results[0].value; const openRouterMock = vi.mocked(OpenRouterPlugin).mock.results[0].value; + const mistralMock = vi.mocked(MistralPlugin).mock.results[0].value; plugin.disable(); @@ -406,6 +453,7 @@ describe("BraintrustPlugin", () => { expect(claudeAgentSDKMock.disable).toHaveBeenCalledTimes(1); expect(googleGenAIMock.disable).toHaveBeenCalledTimes(1); expect(openRouterMock.disable).toHaveBeenCalledTimes(1); + expect(mistralMock.disable).toHaveBeenCalledTimes(1); }); it("should be idempotent on multiple enable calls", () => { @@ -445,6 +493,7 @@ describe("BraintrustPlugin", () => { expect(ClaudeAgentSDKPlugin).not.toHaveBeenCalled(); expect(GoogleGenAIPlugin).not.toHaveBeenCalled(); expect(OpenRouterPlugin).not.toHaveBeenCalled(); + expect(MistralPlugin).not.toHaveBeenCalled(); }); it("should allow re-enabling after disable", () => { @@ -462,6 +511,7 @@ describe("BraintrustPlugin", () => { expect(ClaudeAgentSDKPlugin).toHaveBeenCalledTimes(1); expect(GoogleGenAIPlugin).toHaveBeenCalledTimes(1); expect(OpenRouterPlugin).toHaveBeenCalledTimes(1); + expect(MistralPlugin).toHaveBeenCalledTimes(1); }); it("should only disable plugins that were enabled", () => { @@ -473,6 +523,7 @@ describe("BraintrustPlugin", () => { claudeAgentSDK: false, googleGenAI: true, openrouter: true, + mistral: false, }, }); plugin.enable(); @@ -489,6 +540,7 @@ describe("BraintrustPlugin", () => { expect(aiSDKMock.disable).toHaveBeenCalledTimes(1); expect(googleGenAIMock.disable).toHaveBeenCalledTimes(1); expect(openRouterMock.disable).toHaveBeenCalledTimes(1); + expect(MistralPlugin).not.toHaveBeenCalled(); }); }); }); diff --git a/js/src/instrumentation/braintrust-plugin.ts b/js/src/instrumentation/braintrust-plugin.ts index 89d2b141e..726be1da2 100644 --- a/js/src/instrumentation/braintrust-plugin.ts +++ b/js/src/instrumentation/braintrust-plugin.ts @@ -5,6 +5,7 @@ import { AISDKPlugin } from "./plugins/ai-sdk-plugin"; import { ClaudeAgentSDKPlugin } from "./plugins/claude-agent-sdk-plugin"; import { GoogleGenAIPlugin } from "./plugins/google-genai-plugin"; import { OpenRouterPlugin } from "./plugins/openrouter-plugin"; +import { MistralPlugin } from "./plugins/mistral-plugin"; export interface BraintrustPluginConfig { integrations?: { @@ -16,6 +17,7 @@ export interface BraintrustPluginConfig { googleGenAI?: boolean; claudeAgentSDK?: boolean; openrouter?: boolean; + mistral?: boolean; }; } @@ -28,6 +30,7 @@ export interface BraintrustPluginConfig { * - Claude Agent SDK (agent interactions) * - Vercel AI SDK (generateText, streamText, etc.) * - Google GenAI SDK + * - Mistral SDK * * The plugin is automatically enabled when the Braintrust library is loaded. * Individual integrations can be disabled via configuration. @@ -40,6 +43,7 @@ export class BraintrustPlugin extends BasePlugin { private claudeAgentSDKPlugin: ClaudeAgentSDKPlugin | null = null; private googleGenAIPlugin: GoogleGenAIPlugin | null = null; private openRouterPlugin: OpenRouterPlugin | null = null; + private mistralPlugin: MistralPlugin | null = null; constructor(config: BraintrustPluginConfig = {}) { super(); @@ -85,6 +89,11 @@ export class BraintrustPlugin extends BasePlugin { this.openRouterPlugin = new OpenRouterPlugin(); this.openRouterPlugin.enable(); } + + if (integrations.mistral !== false) { + this.mistralPlugin = new MistralPlugin(); + this.mistralPlugin.enable(); + } } protected onDisable(): void { @@ -117,6 +126,11 @@ export class BraintrustPlugin extends BasePlugin { this.openRouterPlugin.disable(); this.openRouterPlugin = null; } + + if (this.mistralPlugin) { + this.mistralPlugin.disable(); + this.mistralPlugin = null; + } } } diff --git a/js/src/instrumentation/core/channel-tracing.ts b/js/src/instrumentation/core/channel-tracing.ts index 9abb30c36..87e04109f 100644 --- a/js/src/instrumentation/core/channel-tracing.ts +++ b/js/src/instrumentation/core/channel-tracing.ts @@ -104,6 +104,17 @@ export type StreamingChannelSpanConfig = span: Span; startTime: number; }) => boolean; + onComplete?: (args: { + channelName: string; + chunks?: ChunkOf[]; + endEvent: AsyncEndOf; + metadata?: Record; + metrics: Record; + output: unknown; + result: StreamingResult; + span: Span; + startTime: number; + }) => void; }; export type SyncStreamChannelSpanConfig = @@ -295,6 +306,40 @@ function logErrorAndEnd< states.delete(event as object); } +function runStreamingCompletionHook(args: { + channelName: string; + config: StreamingChannelSpanConfig; + chunks?: ChunkOf[]; + endEvent: AsyncEndOf; + metadata?: Record; + metrics: Record; + output: unknown; + result: StreamingResult; + span: Span; + startTime: number; +}): void { + if (!args.config.onComplete) { + return; + } + + try { + args.config.onComplete({ + channelName: args.channelName, + ...(args.chunks ? { chunks: args.chunks } : {}), + endEvent: args.endEvent, + ...(args.metadata !== undefined ? { metadata: args.metadata } : {}), + metrics: args.metrics, + output: args.output, + result: args.result, + span: args.span, + startTime: args.startTime, + }); + } catch (error) { + // eslint-disable-next-line no-restricted-properties -- preserving intentional console usage. + console.error(`Error in onComplete hook for ${args.channelName}:`, error); + } +} + export function traceAsyncChannel( channel: TChannel, config: AsyncChannelSpanConfig, @@ -456,6 +501,19 @@ export function traceStreamingChannel( getCurrentUnixTimestamp() - startTime; } + runStreamingCompletionHook({ + channelName, + chunks, + config, + endEvent: asyncEndEvent, + ...(metadata !== undefined ? { metadata } : {}), + metrics, + output, + result: asyncEndEvent.result as StreamingResult, + span, + startTime, + }); + span.log({ output, ...(metadata !== undefined ? { metadata } : {}), @@ -511,6 +569,20 @@ export function traceStreamingChannel( asyncEndEvent, ); + runStreamingCompletionHook({ + channelName, + config, + endEvent: asyncEndEvent, + ...(normalizeMetadata(metadata) !== undefined + ? { metadata: normalizeMetadata(metadata) } + : {}), + metrics, + output, + result: asyncEndEvent.result as StreamingResult, + span, + startTime, + }); + span.log({ output, ...(normalizeMetadata(metadata) !== undefined diff --git a/js/src/instrumentation/plugins/mistral-channels.ts b/js/src/instrumentation/plugins/mistral-channels.ts new file mode 100644 index 000000000..6328d3872 --- /dev/null +++ b/js/src/instrumentation/plugins/mistral-channels.ts @@ -0,0 +1,78 @@ +import { channel, defineChannels } from "../core/channel-definitions"; +import type { + MistralAgentsCompletionEvent, + MistralAgentsCompletionResponse, + MistralAgentsCreateParams, + MistralAgentsResult, + MistralChatCompletionEvent, + MistralChatCompletionResponse, + MistralChatCreateParams, + MistralChatResult, + MistralEmbeddingCreateParams, + MistralEmbeddingResponse, + MistralFimCompletionEvent, + MistralFimCompletionResponse, + MistralFimCreateParams, + MistralFimResult, +} from "../../vendor-sdk-types/mistral"; + +export const mistralChannels = defineChannels("@mistralai/mistralai", { + chatComplete: channel< + [MistralChatCreateParams], + MistralChatCompletionResponse + >({ + channelName: "chat.complete", + kind: "async", + }), + + chatStream: channel< + [MistralChatCreateParams], + MistralChatResult, + Record, + MistralChatCompletionEvent + >({ + channelName: "chat.stream", + kind: "async", + }), + + embeddingsCreate: channel< + [MistralEmbeddingCreateParams], + MistralEmbeddingResponse + >({ + channelName: "embeddings.create", + kind: "async", + }), + + fimComplete: channel<[MistralFimCreateParams], MistralFimCompletionResponse>({ + channelName: "fim.complete", + kind: "async", + }), + + fimStream: channel< + [MistralFimCreateParams], + MistralFimResult, + Record, + MistralFimCompletionEvent + >({ + channelName: "fim.stream", + kind: "async", + }), + + agentsComplete: channel< + [MistralAgentsCreateParams], + MistralAgentsCompletionResponse + >({ + channelName: "agents.complete", + kind: "async", + }), + + agentsStream: channel< + [MistralAgentsCreateParams], + MistralAgentsResult, + Record, + MistralAgentsCompletionEvent + >({ + channelName: "agents.stream", + kind: "async", + }), +}); diff --git a/js/src/instrumentation/plugins/mistral-plugin.test.ts b/js/src/instrumentation/plugins/mistral-plugin.test.ts new file mode 100644 index 000000000..e73d63076 --- /dev/null +++ b/js/src/instrumentation/plugins/mistral-plugin.test.ts @@ -0,0 +1,370 @@ +import { describe, expect, it } from "vitest"; +import { + aggregateMistralStreamChunks, + extractMistralRequestMetadata, + extractMistralResponseMetadata, + parseMistralMetricsFromUsage, +} from "./mistral-plugin"; + +describe("extractMistralRequestMetadata", () => { + it("keeps only allowlisted request metadata", () => { + expect( + extractMistralRequestMetadata({ + model: "mistral-large-latest", + maxTokens: 128, + temperature: 0.4, + n: 2, + safe_prompt: true, + toolChoice: "auto", + messages: [{ role: "user", content: "hi" }], + tools: [{ type: "function" }], + suffix: "ignored", + arbitrary: "ignored", + }), + ).toEqual({ + model: "mistral-large-latest", + maxTokens: 128, + temperature: 0.4, + n: 2, + safe_prompt: true, + toolChoice: "auto", + }); + }); + + it("returns empty metadata for missing input", () => { + expect(extractMistralRequestMetadata(undefined)).toEqual({}); + }); +}); + +describe("extractMistralResponseMetadata", () => { + it("keeps only allowlisted response metadata", () => { + expect( + extractMistralResponseMetadata({ + id: "cmpl_123", + created: 1234, + object: "chat.completion", + model: "mistral-large-latest", + agentId: "agent_123", + usage: { total_tokens: 20 }, + choices: [{ index: 0 }], + data: [{ embedding: [0.1] }], + arbitrary: "ignored", + }), + ).toEqual({ + id: "cmpl_123", + created: 1234, + object: "chat.completion", + model: "mistral-large-latest", + agentId: "agent_123", + }); + }); + + it("returns undefined when no allowlisted keys are present", () => { + expect( + extractMistralResponseMetadata({ + usage: { total_tokens: 20 }, + choices: [{ index: 0 }], + }), + ).toBeUndefined(); + }); +}); + +describe("parseMistralMetricsFromUsage", () => { + it("returns empty metrics for missing usage", () => { + expect(parseMistralMetricsFromUsage(undefined)).toEqual({}); + expect(parseMistralMetricsFromUsage(null)).toEqual({}); + }); + + it("normalizes common token counters", () => { + expect( + parseMistralMetricsFromUsage({ + promptTokens: 10, + completion_tokens: 6, + totalTokens: 16, + promptAudioSeconds: 3, + }), + ).toEqual({ + prompt_tokens: 10, + completion_tokens: 6, + tokens: 16, + prompt_audio_seconds: 3, + }); + }); + + it("normalizes token detail counters", () => { + expect( + parseMistralMetricsFromUsage({ + inputTokensDetails: { + cachedTokens: 7, + }, + output_tokens_details: { + reasoning_tokens: 2, + }, + }), + ).toEqual({ + prompt_cached_tokens: 7, + completion_reasoning_tokens: 2, + }); + }); +}); + +describe("aggregateMistralStreamChunks", () => { + it("aggregates stream text and usage into a single output row", () => { + const aggregated = aggregateMistralStreamChunks([ + { + data: { + id: "cmpl_1", + model: "mistral-small-latest", + object: "chat.completion.chunk", + created: 1, + choices: [ + { + delta: { + role: "assistant", + content: "Hello", + }, + }, + ], + }, + }, + { + data: { + id: "cmpl_1", + model: "mistral-small-latest", + object: "chat.completion.chunk", + created: 1, + usage: { + prompt_tokens: 12, + completion_tokens: 3, + total_tokens: 15, + }, + choices: [ + { + delta: { + content: " world", + }, + finish_reason: "stop", + }, + ], + }, + }, + ]); + + expect(aggregated.metrics).toMatchObject({ + prompt_tokens: 12, + completion_tokens: 3, + tokens: 15, + }); + + expect(aggregated.output?.[0]).toMatchObject({ + index: 0, + message: { + role: "assistant", + content: "Hello world", + }, + finishReason: "stop", + }); + + expect(aggregated.metadata).toMatchObject({ + id: "cmpl_1", + model: "mistral-small-latest", + object: "chat.completion.chunk", + created: 1, + }); + }); + + it("merges tool call argument deltas", () => { + const aggregated = aggregateMistralStreamChunks([ + { + data: { + choices: [ + { + delta: { + toolCalls: [ + { + id: "tool_1", + function: { + name: "lookup_weather", + arguments: '{"city":"Vie', + }, + }, + ], + }, + }, + ], + }, + }, + { + data: { + choices: [ + { + delta: { + tool_calls: [ + { + id: "tool_1", + function: { + arguments: 'nna"}', + }, + }, + ], + }, + finishReason: "tool_calls", + }, + ], + }, + }, + ]); + + expect(aggregated.output?.[0]).toMatchObject({ + message: { + content: null, + toolCalls: [ + { + id: "tool_1", + function: { + name: "lookup_weather", + arguments: '{"city":"Vienna"}', + }, + }, + ], + }, + finishReason: "tool_calls", + }); + }); + + it("merges interleaved tool call deltas by index", () => { + const aggregated = aggregateMistralStreamChunks([ + { + data: { + choices: [ + { + delta: { + toolCalls: [ + { + index: 0, + id: "tool_0", + function: { + name: "first_tool", + arguments: '{"city":"Vie', + }, + }, + { + index: 1, + id: "tool_1", + function: { + name: "second_tool", + arguments: '{"unit":"c"}', + }, + }, + ], + }, + }, + ], + }, + }, + { + data: { + choices: [ + { + delta: { + tool_calls: [ + { + index: 0, + id: "tool_0", + function: { + arguments: 'nna"}', + }, + }, + ], + }, + finishReason: "tool_calls", + }, + ], + }, + }, + ]); + + expect(aggregated.output?.[0]).toMatchObject({ + message: { + content: null, + toolCalls: [ + { + index: 0, + id: "tool_0", + function: { + name: "first_tool", + arguments: '{"city":"Vienna"}', + }, + }, + { + index: 1, + id: "tool_1", + function: { + name: "second_tool", + arguments: '{"unit":"c"}', + }, + }, + ], + }, + finishReason: "tool_calls", + }); + }); + + it("keeps streamed choices separated when multiple choices are returned", () => { + const aggregated = aggregateMistralStreamChunks([ + { + data: { + choices: [ + { + index: 0, + delta: { + role: "assistant", + content: "a", + }, + }, + { + index: 1, + delta: { + role: "assistant", + content: "b", + }, + }, + ], + }, + }, + { + data: { + choices: [ + { + index: 0, + finishReason: "stop", + }, + { + index: 1, + finishReason: "length", + }, + ], + }, + }, + ]); + + expect(aggregated.output).toEqual([ + { + index: 0, + message: { + role: "assistant", + content: "a", + }, + finishReason: "stop", + }, + { + index: 1, + message: { + role: "assistant", + content: "b", + }, + finishReason: "length", + }, + ]); + }); +}); diff --git a/js/src/instrumentation/plugins/mistral-plugin.ts b/js/src/instrumentation/plugins/mistral-plugin.ts new file mode 100644 index 000000000..88ae25f46 --- /dev/null +++ b/js/src/instrumentation/plugins/mistral-plugin.ts @@ -0,0 +1,686 @@ +import { BasePlugin } from "../core"; +import { + traceAsyncChannel, + traceStreamingChannel, + unsubscribeAll, +} from "../core/channel-tracing"; +import { SpanTypeAttribute, isObject } from "../../../util/index"; +import { processInputAttachments } from "../../wrappers/attachment-utils"; +import { getCurrentUnixTimestamp } from "../../util"; +import { mistralChannels } from "./mistral-channels"; +import type { + MistralChatCompletionChunk, + MistralChatCompletionChunkChoice, + MistralChatCompletionEvent, + MistralChatCompletionResponse, + MistralToolCallDelta, +} from "../../vendor-sdk-types/mistral"; + +export class MistralPlugin extends BasePlugin { + protected onEnable(): void { + this.subscribeToMistralChannels(); + } + + protected onDisable(): void { + this.unsubscribers = unsubscribeAll(this.unsubscribers); + } + + private subscribeToMistralChannels(): void { + this.unsubscribers.push( + traceStreamingChannel(mistralChannels.chatComplete, { + name: "mistral.chat.complete", + type: SpanTypeAttribute.LLM, + extractInput: extractMessagesInputWithMetadata, + extractOutput: (result) => { + return result?.choices; + }, + extractMetadata: (result) => extractMistralResponseMetadata(result), + extractMetrics: (result, startTime) => + extractMistralMetrics(result?.usage, startTime), + }), + ); + + this.unsubscribers.push( + traceStreamingChannel(mistralChannels.chatStream, { + name: "mistral.chat.stream", + type: SpanTypeAttribute.LLM, + extractInput: extractMessagesInputWithMetadata, + extractOutput: extractMistralStreamOutput, + extractMetadata: (result) => extractMistralResponseMetadata(result), + extractMetrics: (result, startTime) => + extractMistralStreamingMetrics(result, startTime), + aggregateChunks: aggregateMistralStreamChunks, + }), + ); + + this.unsubscribers.push( + traceAsyncChannel(mistralChannels.embeddingsCreate, { + name: "mistral.embeddings.create", + type: SpanTypeAttribute.LLM, + extractInput: extractEmbeddingInputWithMetadata, + extractOutput: (result) => { + const embedding = result?.data?.[0]?.embedding; + return Array.isArray(embedding) + ? { embedding_length: embedding.length } + : undefined; + }, + extractMetadata: (result) => extractMistralResponseMetadata(result), + extractMetrics: (result) => parseMistralMetricsFromUsage(result?.usage), + }), + ); + + this.unsubscribers.push( + traceStreamingChannel(mistralChannels.fimComplete, { + name: "mistral.fim.complete", + type: SpanTypeAttribute.LLM, + extractInput: extractPromptInputWithMetadata, + extractOutput: (result) => { + return result?.choices; + }, + extractMetadata: (result) => extractMistralResponseMetadata(result), + extractMetrics: (result, startTime) => + extractMistralMetrics(result?.usage, startTime), + }), + ); + + this.unsubscribers.push( + traceStreamingChannel(mistralChannels.fimStream, { + name: "mistral.fim.stream", + type: SpanTypeAttribute.LLM, + extractInput: extractPromptInputWithMetadata, + extractOutput: extractMistralStreamOutput, + extractMetadata: (result) => extractMistralResponseMetadata(result), + extractMetrics: (result, startTime) => + extractMistralStreamingMetrics(result, startTime), + aggregateChunks: aggregateMistralStreamChunks, + }), + ); + + this.unsubscribers.push( + traceStreamingChannel(mistralChannels.agentsComplete, { + name: "mistral.agents.complete", + type: SpanTypeAttribute.LLM, + extractInput: extractMessagesInputWithMetadata, + extractOutput: (result) => { + return result?.choices; + }, + extractMetadata: (result) => extractMistralResponseMetadata(result), + extractMetrics: (result, startTime) => + extractMistralMetrics(result?.usage, startTime), + }), + ); + + this.unsubscribers.push( + traceStreamingChannel(mistralChannels.agentsStream, { + name: "mistral.agents.stream", + type: SpanTypeAttribute.LLM, + extractInput: extractMessagesInputWithMetadata, + extractOutput: extractMistralStreamOutput, + extractMetadata: (result) => extractMistralResponseMetadata(result), + extractMetrics: (result, startTime) => + extractMistralStreamingMetrics(result, startTime), + aggregateChunks: aggregateMistralStreamChunks, + }), + ); + } +} + +const TOKEN_NAME_MAP: Record = { + promptTokens: "prompt_tokens", + inputTokens: "prompt_tokens", + completionTokens: "completion_tokens", + outputTokens: "completion_tokens", + totalTokens: "tokens", + prompt_tokens: "prompt_tokens", + input_tokens: "prompt_tokens", + completion_tokens: "completion_tokens", + output_tokens: "completion_tokens", + total_tokens: "tokens", + promptAudioSeconds: "prompt_audio_seconds", + prompt_audio_seconds: "prompt_audio_seconds", +}; + +const TOKEN_DETAIL_PREFIX_MAP: Record = { + promptTokensDetails: "prompt", + inputTokensDetails: "prompt", + completionTokensDetails: "completion", + outputTokensDetails: "completion", + prompt_tokens_details: "prompt", + input_tokens_details: "prompt", + completion_tokens_details: "completion", + output_tokens_details: "completion", +}; + +const MISTRAL_REQUEST_METADATA_ALLOWLIST = new Set([ + "agentId", + "agent_id", + "encodingFormat", + "encoding_format", + "frequencyPenalty", + "frequency_penalty", + "maxTokens", + "max_tokens", + "model", + "n", + "presencePenalty", + "presence_penalty", + "randomSeed", + "random_seed", + "responseFormat", + "response_format", + "safePrompt", + "safe_prompt", + "stream", + "stop", + "temperature", + "toolChoice", + "tool_choice", + "topP", + "top_p", +]); + +const MISTRAL_RESPONSE_METADATA_ALLOWLIST = new Set([ + "agentId", + "agent_id", + "created", + "id", + "model", + "object", +]); + +function camelToSnake(value: string): string { + return value.replace(/[A-Z]/g, (match) => `_${match.toLowerCase()}`); +} + +function normalizeArgs(args: unknown[] | unknown): unknown[] { + if (Array.isArray(args)) { + return args; + } + + if (isArrayLike(args)) { + return Array.from(args); + } + + return [args]; +} + +function isArrayLike(value: unknown): value is ArrayLike { + return ( + isObject(value) && + "length" in value && + typeof value.length === "number" && + Number.isInteger(value.length) && + value.length >= 0 + ); +} + +function getMistralRequestArg( + args: unknown[] | unknown, +): Record | undefined { + const firstObjectArg = normalizeArgs(args).find((arg) => isObject(arg)); + return isObject(firstObjectArg) ? firstObjectArg : undefined; +} + +function addMistralProviderMetadata( + metadata: Record, +): Record { + return { + ...metadata, + provider: "mistral", + }; +} + +function pickAllowedMetadata( + metadata: Record | undefined, + allowlist: ReadonlySet, +): Record { + if (!metadata) { + return {}; + } + + const picked: Record = {}; + for (const key of allowlist) { + const value = metadata[key]; + if (value !== undefined) { + picked[key] = value; + } + } + return picked; +} + +export function extractMistralRequestMetadata( + metadata: Record | undefined, +): Record { + return pickAllowedMetadata(metadata, MISTRAL_REQUEST_METADATA_ALLOWLIST); +} + +function isMistralChatCompletionChunk( + value: unknown, +): value is MistralChatCompletionChunk { + return isObject(value); +} + +function isMistralChunkChoice( + value: unknown, +): value is MistralChatCompletionChunkChoice { + return isObject(value); +} + +function extractMessagesInputWithMetadata(args: unknown[] | unknown): { + input: unknown; + metadata: Record; +} { + const params = getMistralRequestArg(args); + const { messages, ...rawMetadata } = params || {}; + + return { + input: processInputAttachments(messages), + metadata: addMistralProviderMetadata( + extractMistralRequestMetadata(rawMetadata), + ), + }; +} + +function extractEmbeddingInputWithMetadata(args: unknown[] | unknown): { + input: unknown; + metadata: Record; +} { + const params = getMistralRequestArg(args); + const { inputs, ...rawMetadata } = params || {}; + + return { + input: inputs, + metadata: addMistralProviderMetadata( + extractMistralRequestMetadata(rawMetadata), + ), + }; +} + +function extractPromptInputWithMetadata(args: unknown[] | unknown): { + input: unknown; + metadata: Record; +} { + const params = getMistralRequestArg(args); + const { prompt, ...rawMetadata } = params || {}; + + return { + input: prompt, + metadata: addMistralProviderMetadata( + extractMistralRequestMetadata(rawMetadata), + ), + }; +} + +export function extractMistralResponseMetadata( + result: unknown, +): Record | undefined { + if (!isObject(result)) { + return undefined; + } + + const { choices: _choices, usage: _usage, data: _data, ...metadata } = result; + const picked = pickAllowedMetadata( + metadata, + MISTRAL_RESPONSE_METADATA_ALLOWLIST, + ); + + return Object.keys(picked).length > 0 ? picked : undefined; +} + +function extractMistralMetrics( + usage: unknown, + startTime?: number, +): Record { + const metrics = parseMistralMetricsFromUsage(usage); + if (startTime) { + metrics.time_to_first_token = getCurrentUnixTimestamp() - startTime; + } + return metrics; +} + +function extractMistralStreamOutput(result: unknown): unknown { + return isObject(result) ? result.choices : undefined; +} + +function extractMistralStreamingMetrics( + result: unknown, + startTime?: number, +): Record { + const metrics = isObject(result) + ? parseMistralMetricsFromUsage(result.usage) + : {}; + if (startTime) { + metrics.time_to_first_token = getCurrentUnixTimestamp() - startTime; + } + return metrics; +} + +function extractDeltaText(content: unknown): string | undefined { + if (typeof content === "string") { + return content; + } + + if (!Array.isArray(content)) { + return undefined; + } + + const textParts = content + .map((part) => { + if (!isObject(part) || part.type !== "text") { + return ""; + } + + return typeof part.text === "string" ? part.text : ""; + }) + .filter((part) => part.length > 0); + + return textParts.length > 0 ? textParts.join("") : undefined; +} + +function getDeltaToolCalls( + delta: Record, +): MistralToolCallDelta[] { + const toolCalls = + (Array.isArray(delta.toolCalls) && delta.toolCalls) || + (Array.isArray(delta.tool_calls) && delta.tool_calls) || + []; + + return toolCalls.filter((toolCall) => isObject(toolCall)); +} + +function getToolCallIndex(toolCall: MistralToolCallDelta): number | undefined { + return typeof toolCall.index === "number" && toolCall.index >= 0 + ? toolCall.index + : undefined; +} + +function createMergedToolCallDelta( + delta: MistralToolCallDelta, +): MistralToolCallDelta { + return { + ...delta, + function: { + ...delta.function, + arguments: + typeof delta.function?.arguments === "string" + ? delta.function.arguments + : "", + }, + }; +} + +function mergeToolCallDeltaPair( + current: MistralToolCallDelta, + delta: MistralToolCallDelta, +): MistralToolCallDelta { + const currentArguments = + typeof current.function?.arguments === "string" + ? current.function.arguments + : ""; + const deltaArguments = + typeof delta.function?.arguments === "string" + ? delta.function.arguments + : ""; + + return { + ...current, + ...delta, + function: { + ...(current.function || {}), + ...(delta.function || {}), + arguments: `${currentArguments}${deltaArguments}`, + }, + }; +} + +function mergeToolCallDeltas( + toolCalls: MistralToolCallDelta[] | undefined, + deltas: MistralToolCallDelta[], +): MistralToolCallDelta[] | undefined { + if (deltas.length === 0) { + return toolCalls; + } + + const merged = toolCalls ? [...toolCalls] : []; + const indexToPosition = new Map(); + const idToPosition = new Map(); + + for (const [position, toolCall] of merged.entries()) { + const index = getToolCallIndex(toolCall); + if (index !== undefined && !indexToPosition.has(index)) { + indexToPosition.set(index, position); + } + + if (typeof toolCall.id === "string" && !idToPosition.has(toolCall.id)) { + idToPosition.set(toolCall.id, position); + } + } + + for (const delta of deltas) { + const deltaIndex = getToolCallIndex(delta); + const existingByIndex = + deltaIndex !== undefined ? indexToPosition.get(deltaIndex) : undefined; + const existingById = + typeof delta.id === "string" ? idToPosition.get(delta.id) : undefined; + const existingPosition = existingByIndex ?? existingById; + + if (existingPosition === undefined) { + const newToolCall = createMergedToolCallDelta(delta); + merged.push(newToolCall); + + const newPosition = merged.length - 1; + const newIndex = getToolCallIndex(newToolCall); + if (newIndex !== undefined && !indexToPosition.has(newIndex)) { + indexToPosition.set(newIndex, newPosition); + } + if ( + typeof newToolCall.id === "string" && + !idToPosition.has(newToolCall.id) + ) { + idToPosition.set(newToolCall.id, newPosition); + } + continue; + } + + const mergedToolCall = mergeToolCallDeltaPair( + merged[existingPosition], + delta, + ); + merged[existingPosition] = mergedToolCall; + + const mergedIndex = getToolCallIndex(mergedToolCall); + if (mergedIndex !== undefined && !indexToPosition.has(mergedIndex)) { + indexToPosition.set(mergedIndex, existingPosition); + } + if ( + typeof mergedToolCall.id === "string" && + !idToPosition.has(mergedToolCall.id) + ) { + idToPosition.set(mergedToolCall.id, existingPosition); + } + } + + return merged.length > 0 ? merged : undefined; +} + +function getChoiceFinishReason( + choice: MistralChatCompletionChunkChoice, +): string | null | undefined { + if (typeof choice.finishReason === "string" || choice.finishReason === null) { + return choice.finishReason; + } + + if ( + typeof choice.finish_reason === "string" || + choice.finish_reason === null + ) { + return choice.finish_reason; + } + + return undefined; +} + +type MistralChoiceAccumulator = { + content?: string; + finishReason?: string | null; + index: number; + order: number; + role?: string; + toolCalls?: MistralToolCallDelta[]; +}; + +export function parseMistralMetricsFromUsage( + usage: unknown, +): Record { + if (!isObject(usage)) { + return {}; + } + + const metrics: Record = {}; + + for (const [name, value] of Object.entries(usage)) { + if (typeof value === "number") { + metrics[TOKEN_NAME_MAP[name] || camelToSnake(name)] = value; + continue; + } + + if (!isObject(value)) { + continue; + } + + const prefix = TOKEN_DETAIL_PREFIX_MAP[name]; + if (!prefix) { + continue; + } + + for (const [nestedName, nestedValue] of Object.entries(value)) { + if (typeof nestedValue !== "number") { + continue; + } + + metrics[`${prefix}_${camelToSnake(nestedName)}`] = nestedValue; + } + } + + return metrics; +} + +export function aggregateMistralStreamChunks( + chunks: MistralChatCompletionEvent[], +): { + output: MistralChatCompletionResponse["choices"]; + metrics: Record; + metadata?: Record; +} { + const choiceAccumulators = new Map(); + const indexToAccumulatorKey = new Map(); + const positionToAccumulatorKey = new Map(); + let nextAccumulatorOrder = 0; + let metrics: Record = {}; + let metadata: Record | undefined; + + for (const event of chunks) { + const chunk = isMistralChatCompletionChunk(event?.data) + ? event.data + : undefined; + if (!chunk) { + continue; + } + + if (isObject(chunk.usage)) { + metrics = { + ...metrics, + ...parseMistralMetricsFromUsage(chunk.usage), + }; + } + + const chunkMetadata = extractMistralResponseMetadata(chunk); + if (chunkMetadata) { + metadata = { ...(metadata || {}), ...chunkMetadata }; + } + + for (const [choicePosition, rawChoice] of (chunk.choices || []).entries()) { + if (!isMistralChunkChoice(rawChoice)) { + continue; + } + const choice = rawChoice; + const choiceIndex = + typeof choice.index === "number" && choice.index >= 0 + ? choice.index + : undefined; + let accumulatorKey = + choiceIndex !== undefined + ? indexToAccumulatorKey.get(choiceIndex) + : undefined; + if (!accumulatorKey) { + accumulatorKey = positionToAccumulatorKey.get(choicePosition); + } + if (!accumulatorKey) { + const initialIndex = choiceIndex ?? choicePosition; + const keyPrefix = choiceIndex !== undefined ? "index" : "position"; + accumulatorKey = `${keyPrefix}:${initialIndex}`; + choiceAccumulators.set(accumulatorKey, { + index: initialIndex, + order: nextAccumulatorOrder++, + }); + } + + const accumulator = choiceAccumulators.get(accumulatorKey); + if (!accumulator) { + continue; + } + + if (choiceIndex !== undefined) { + accumulator.index = choiceIndex; + indexToAccumulatorKey.set(choiceIndex, accumulatorKey); + } + positionToAccumulatorKey.set(choicePosition, accumulatorKey); + + const delta = isObject(choice.delta) ? choice.delta : undefined; + if (delta) { + if (!accumulator.role && typeof delta.role === "string") { + accumulator.role = delta.role; + } + + const deltaText = extractDeltaText(delta.content); + if (deltaText) { + accumulator.content = `${accumulator.content || ""}${deltaText}`; + } + + accumulator.toolCalls = mergeToolCallDeltas( + accumulator.toolCalls, + getDeltaToolCalls(delta), + ); + } + + const choiceFinishReason = getChoiceFinishReason(choice); + if (choiceFinishReason !== undefined) { + accumulator.finishReason = choiceFinishReason; + } + } + } + + const output = Array.from(choiceAccumulators.values()) + .sort((left, right) => + left.index === right.index + ? left.order - right.order + : left.index - right.index, + ) + .map((choice) => ({ + index: choice.index, + message: { + ...(choice.role ? { role: choice.role } : {}), + content: choice.content ?? null, + ...(choice.toolCalls ? { toolCalls: choice.toolCalls } : {}), + }, + ...(choice.finishReason !== undefined + ? { finishReason: choice.finishReason } + : {}), + })); + + return { + output, + metrics, + ...(metadata ? { metadata } : {}), + }; +} diff --git a/js/src/instrumentation/registry.test.ts b/js/src/instrumentation/registry.test.ts index c3b62db7e..fc2c36bd3 100644 --- a/js/src/instrumentation/registry.test.ts +++ b/js/src/instrumentation/registry.test.ts @@ -119,6 +119,7 @@ describe("configureInstrumentation API", () => { openai: false, anthropic: true, openrouter: false, + mistral: false, }, }); }); diff --git a/js/src/instrumentation/registry.ts b/js/src/instrumentation/registry.ts index 79825abeb..e3521bdc3 100644 --- a/js/src/instrumentation/registry.ts +++ b/js/src/instrumentation/registry.ts @@ -21,6 +21,7 @@ export interface InstrumentationConfig { google?: boolean; claudeAgentSDK?: boolean; openrouter?: boolean; + mistral?: boolean; }; } @@ -107,6 +108,7 @@ class PluginRegistry { google: true, claudeAgentSDK: true, openrouter: true, + mistral: true, }; } diff --git a/js/src/vendor-sdk-types/mistral.ts b/js/src/vendor-sdk-types/mistral.ts new file mode 100644 index 000000000..90fd390fd --- /dev/null +++ b/js/src/vendor-sdk-types/mistral.ts @@ -0,0 +1,171 @@ +export type MistralToolCallDelta = { + id?: string; + type?: string; + function?: { + name?: string; + arguments?: string; + }; + [key: string]: unknown; +}; + +export type MistralChatMessageDelta = { + role?: string; + content?: string | null; + toolCalls?: MistralToolCallDelta[] | null; + tool_calls?: MistralToolCallDelta[] | null; + [key: string]: unknown; +}; + +export type MistralChatCompletionChoice = { + index?: number; + message?: { + role?: string; + content?: string | null; + toolCalls?: unknown; + tool_calls?: unknown; + }; + finishReason?: string | null; + finish_reason?: string | null; + [key: string]: unknown; +}; + +export type MistralChatCompletionChunkChoice = { + index?: number; + delta?: MistralChatMessageDelta; + finishReason?: string | null; + finish_reason?: string | null; + [key: string]: unknown; +}; + +export type MistralChatCompletionChunk = { + id?: string; + object?: string; + created?: number; + model?: string; + usage?: unknown; + choices?: MistralChatCompletionChunkChoice[]; + [key: string]: unknown; +}; + +export type MistralChatCompletionEvent = { + data?: MistralChatCompletionChunk; + [key: string]: unknown; +}; + +export type MistralChatCompletionResponse = { + id?: string; + object?: string; + model?: string; + created?: number; + usage?: unknown; + choices?: MistralChatCompletionChoice[]; + [key: string]: unknown; +}; + +export type MistralEmbeddingCreateParams = { + inputs?: unknown; + [key: string]: unknown; +}; + +export type MistralFimCreateParams = { + prompt?: unknown; + suffix?: unknown; + stream?: boolean; + [key: string]: unknown; +}; + +export type MistralAgentsCreateParams = { + messages?: unknown; + agentId?: string; + agent_id?: string; + stream?: boolean; + [key: string]: unknown; +}; + +export type MistralEmbeddingResponse = { + id?: string; + object?: string; + model?: string; + usage?: unknown; + data?: Array<{ + embedding?: number[] | string; + [key: string]: unknown; + }>; + [key: string]: unknown; +}; + +export type MistralChatCreateParams = { + messages?: unknown; + stream?: boolean; + [key: string]: unknown; +}; + +export type MistralChatStreamingResult = + AsyncIterable; +export type MistralFimStreamingResult = + AsyncIterable; +export type MistralAgentsStreamingResult = + AsyncIterable; + +export type MistralFimCompletionResponse = MistralChatCompletionResponse; +export type MistralAgentsCompletionResponse = MistralChatCompletionResponse; +export type MistralFimCompletionEvent = MistralChatCompletionEvent; +export type MistralAgentsCompletionEvent = MistralChatCompletionEvent; + +export type MistralChatResult = + | MistralChatCompletionResponse + | MistralChatStreamingResult; +export type MistralFimResult = + | MistralFimCompletionResponse + | MistralFimStreamingResult; +export type MistralAgentsResult = + | MistralAgentsCompletionResponse + | MistralAgentsStreamingResult; + +export type MistralChat = { + complete: ( + request: MistralChatCreateParams, + options?: unknown, + ) => Promise; + stream: ( + request: MistralChatCreateParams, + options?: unknown, + ) => Promise; +}; + +export type MistralEmbeddings = { + create: ( + request: MistralEmbeddingCreateParams, + options?: unknown, + ) => Promise; +}; + +export type MistralFim = { + complete: ( + request: MistralFimCreateParams, + options?: unknown, + ) => Promise; + stream: ( + request: MistralFimCreateParams, + options?: unknown, + ) => Promise; +}; + +export type MistralAgents = { + complete: ( + request: MistralAgentsCreateParams, + options?: unknown, + ) => Promise; + stream: ( + request: MistralAgentsCreateParams, + options?: unknown, + ) => Promise; +}; + +export type MistralClient = { + chat?: MistralChat; + fim?: MistralFim; + agents?: MistralAgents; + embeddings?: MistralEmbeddings; + [key: string]: unknown; +}; diff --git a/js/src/wrappers/mistral.ts b/js/src/wrappers/mistral.ts new file mode 100644 index 000000000..b2ce06aff --- /dev/null +++ b/js/src/wrappers/mistral.ts @@ -0,0 +1,245 @@ +import { mistralChannels } from "../instrumentation/plugins/mistral-channels"; +import type { + MistralAgents, + MistralAgentsCompletionResponse, + MistralAgentsCreateParams, + MistralAgentsStreamingResult, + MistralChat, + MistralChatCompletionResponse, + MistralChatCreateParams, + MistralChatStreamingResult, + MistralClient, + MistralEmbeddingCreateParams, + MistralEmbeddingResponse, + MistralEmbeddings, + MistralFim, + MistralFimCompletionResponse, + MistralFimCreateParams, + MistralFimStreamingResult, +} from "../vendor-sdk-types/mistral"; + +/** + * Wrap a Mistral client (created with `new Mistral(...)`) with Braintrust tracing. + */ +export function wrapMistral(mistral: T): T { + if (isSupportedMistralClient(mistral)) { + return mistralProxy(mistral) as T; + } + + // eslint-disable-next-line no-restricted-properties -- preserving intentional console usage. + console.warn("Unsupported Mistral library. Not wrapping."); + return mistral; +} + +function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null; +} + +function hasFunction(value: unknown, methodName: string): boolean { + return ( + isRecord(value) && + methodName in value && + typeof value[methodName] === "function" + ); +} + +function isSupportedMistralClient(value: unknown): value is MistralClient { + if (!isRecord(value)) { + return false; + } + + return ( + (value.chat !== undefined && hasChat(value.chat)) || + (value.embeddings !== undefined && hasEmbeddings(value.embeddings)) || + (value.fim !== undefined && hasFim(value.fim)) || + (value.agents !== undefined && hasAgents(value.agents)) + ); +} + +function hasChat(value: unknown): value is MistralChat { + return hasFunction(value, "complete") && hasFunction(value, "stream"); +} + +function hasEmbeddings(value: unknown): value is MistralEmbeddings { + return hasFunction(value, "create"); +} + +function hasFim(value: unknown): value is MistralFim { + return hasFunction(value, "complete") && hasFunction(value, "stream"); +} + +function hasAgents(value: unknown): value is MistralAgents { + return hasFunction(value, "complete") && hasFunction(value, "stream"); +} + +function mistralProxy(mistral: MistralClient): MistralClient { + return new Proxy(mistral, { + get(target, prop, receiver) { + switch (prop) { + case "chat": + return target.chat ? chatProxy(target.chat) : target.chat; + case "fim": + return target.fim ? fimProxy(target.fim) : target.fim; + case "agents": + return target.agents ? agentsProxy(target.agents) : target.agents; + case "embeddings": + return target.embeddings + ? embeddingsProxy(target.embeddings) + : target.embeddings; + default: + return Reflect.get(target, prop, receiver); + } + }, + }); +} + +function chatProxy(chat: MistralChat): MistralChat { + return new Proxy(chat, { + get(target, prop, receiver) { + if (prop === "complete") { + return wrapChatComplete(target.complete.bind(target)); + } + + if (prop === "stream") { + return wrapChatStream(target.stream.bind(target)); + } + + return Reflect.get(target, prop, receiver); + }, + }); +} + +function embeddingsProxy(embeddings: MistralEmbeddings): MistralEmbeddings { + return new Proxy(embeddings, { + get(target, prop, receiver) { + if (prop === "create") { + return wrapEmbeddingsCreate(target.create.bind(target)); + } + + return Reflect.get(target, prop, receiver); + }, + }); +} + +function fimProxy(fim: MistralFim): MistralFim { + return new Proxy(fim, { + get(target, prop, receiver) { + if (prop === "complete") { + return wrapFimComplete(target.complete.bind(target)); + } + + if (prop === "stream") { + return wrapFimStream(target.stream.bind(target)); + } + + return Reflect.get(target, prop, receiver); + }, + }); +} + +function agentsProxy(agents: MistralAgents): MistralAgents { + return new Proxy(agents, { + get(target, prop, receiver) { + if (prop === "complete") { + return wrapAgentsComplete(target.complete.bind(target)); + } + + if (prop === "stream") { + return wrapAgentsStream(target.stream.bind(target)); + } + + return Reflect.get(target, prop, receiver); + }, + }); +} + +function wrapChatComplete( + complete: ( + request: MistralChatCreateParams, + options?: unknown, + ) => Promise, +): MistralChat["complete"] { + return (request, options) => + mistralChannels.chatComplete.tracePromise( + () => complete(request, options), + { + arguments: [request], + } as Parameters[1], + ); +} + +function wrapChatStream( + stream: ( + request: MistralChatCreateParams, + options?: unknown, + ) => Promise, +): MistralChat["stream"] { + return (request, options) => + mistralChannels.chatStream.tracePromise(() => stream(request, options), { + arguments: [request], + } as Parameters[1]); +} + +function wrapEmbeddingsCreate( + create: ( + request: MistralEmbeddingCreateParams, + options?: unknown, + ) => Promise, +): MistralEmbeddings["create"] { + return (request, options) => + mistralChannels.embeddingsCreate.tracePromise( + () => create(request, options), + { arguments: [request] }, + ); +} + +function wrapFimComplete( + complete: ( + request: MistralFimCreateParams, + options?: unknown, + ) => Promise, +): MistralFim["complete"] { + return (request, options) => + mistralChannels.fimComplete.tracePromise(() => complete(request, options), { + arguments: [request], + } as Parameters[1]); +} + +function wrapFimStream( + stream: ( + request: MistralFimCreateParams, + options?: unknown, + ) => Promise, +): MistralFim["stream"] { + return (request, options) => + mistralChannels.fimStream.tracePromise(() => stream(request, options), { + arguments: [request], + } as Parameters[1]); +} + +function wrapAgentsComplete( + complete: ( + request: MistralAgentsCreateParams, + options?: unknown, + ) => Promise, +): MistralAgents["complete"] { + return (request, options) => + mistralChannels.agentsComplete.tracePromise( + () => complete(request, options), + { + arguments: [request], + } as Parameters[1], + ); +} + +function wrapAgentsStream( + stream: ( + request: MistralAgentsCreateParams, + options?: unknown, + ) => Promise, +): MistralAgents["stream"] { + return (request, options) => + mistralChannels.agentsStream.tracePromise(() => stream(request, options), { + arguments: [request], + } as Parameters[1]); +} diff --git a/turbo.json b/turbo.json index 08809361d..42046bc5d 100644 --- a/turbo.json +++ b/turbo.json @@ -5,7 +5,8 @@ "OPENAI_BASE_URL", "ANTHROPIC_API_KEY", "GEMINI_API_KEY", - "OPENROUTER_API_KEY" + "OPENROUTER_API_KEY", + "MISTRAL_API_KEY" ], "tasks": { "build": { @@ -24,7 +25,8 @@ "GEMINI_API_KEY", "OPENAI_API_KEY", "OPENAI_BASE_URL", - "OPENROUTER_API_KEY" + "OPENROUTER_API_KEY", + "MISTRAL_API_KEY" ], "dependsOn": ["^build"], "outputs": [] @@ -38,7 +40,8 @@ "GEMINI_API_KEY", "OPENAI_API_KEY", "OPENAI_BASE_URL", - "OPENROUTER_API_KEY" + "OPENROUTER_API_KEY", + "MISTRAL_API_KEY" ], "dependsOn": ["^build"], "outputs": [] @@ -57,7 +60,8 @@ "GEMINI_API_KEY", "OPENAI_API_KEY", "OPENAI_BASE_URL", - "OPENROUTER_API_KEY" + "OPENROUTER_API_KEY", + "MISTRAL_API_KEY" ], "dependsOn": [], "outputs": [] @@ -71,7 +75,8 @@ "GEMINI_API_KEY", "OPENAI_API_KEY", "OPENAI_BASE_URL", - "OPENROUTER_API_KEY" + "OPENROUTER_API_KEY", + "MISTRAL_API_KEY" ], "dependsOn": ["^build"], "outputs": [] @@ -99,7 +104,8 @@ "BRAINTRUST_API_KEY", "GEMINI_API_KEY", "OPENAI_API_KEY", - "OPENROUTER_API_KEY" + "OPENROUTER_API_KEY", + "MISTRAL_API_KEY" ], "dependsOn": ["^build", "build"] }, @@ -110,7 +116,8 @@ "BRAINTRUST_API_KEY", "GEMINI_API_KEY", "OPENAI_API_KEY", - "OPENROUTER_API_KEY" + "OPENROUTER_API_KEY", + "MISTRAL_API_KEY" ], "dependsOn": ["^build", "build"] }, @@ -121,7 +128,8 @@ "BRAINTRUST_API_KEY", "GEMINI_API_KEY", "OPENAI_API_KEY", - "OPENROUTER_API_KEY" + "OPENROUTER_API_KEY", + "MISTRAL_API_KEY" ], "dependsOn": ["^build", "build"] }