From 76196cca2c5106094acc77ad2063b28cf6f5d528 Mon Sep 17 00:00:00 2001 From: Luca Forstner Date: Thu, 2 Apr 2026 14:25:20 +0200 Subject: [PATCH 01/12] feat: Add instrumentation for `@mistralai/mistralai` --- .agents/skills/instrumentation/SKILL.md | 1 + .env.example | 10 +- .github/workflows/e2e-canary.yaml | 1 + .github/workflows/integration-tests.yaml | 3 + e2e/README.md | 1 + e2e/helpers/scenario-installer.ts | 1 + .../mistral-v1-10-0.log-payloads.json | 272 +++++++++ .../mistral-v1-10-0.span-events.json | 266 +++++++++ .../mistral-v1-14-1.log-payloads.json | 272 +++++++++ .../mistral-v1-14-1.span-events.json | 266 +++++++++ .../mistral-v1-15-1.log-payloads.json | 272 +++++++++ .../mistral-v1-15-1.span-events.json | 266 +++++++++ .../mistral-v1-3-4.log-payloads.json | 272 +++++++++ .../mistral-v1-3-4.span-events.json | 266 +++++++++ .../mistral-v1.log-payloads.json | 272 +++++++++ .../__snapshots__/mistral-v1.span-events.json | 266 +++++++++ .../mistral-v2.log-payloads.json | 278 +++++++++ .../__snapshots__/mistral-v2.span-events.json | 272 +++++++++ .../mistral-instrumentation/assertions.ts | 546 ++++++++++++++++++ .../mistral-instrumentation/constants.mjs | 15 + .../mistral-instrumentation/package.json | 20 + .../mistral-instrumentation/pnpm-lock.yaml | 122 ++++ .../mistral-instrumentation/scenario.impl.mjs | 446 ++++++++++++++ .../scenario.mistral-v1-10-0.mjs | 5 + .../scenario.mistral-v1-10-0.ts | 5 + .../scenario.mistral-v1-14-1.mjs | 5 + .../scenario.mistral-v1-14-1.ts | 5 + .../scenario.mistral-v1-15-1.mjs | 5 + .../scenario.mistral-v1-15-1.ts | 5 + .../scenario.mistral-v1-3-4.mjs | 5 + .../scenario.mistral-v1-3-4.ts | 5 + .../scenario.mistral-v1.mjs | 5 + .../scenario.mistral-v1.ts | 5 + .../mistral-instrumentation/scenario.mjs | 5 + .../mistral-instrumentation/scenario.test.ts | 62 ++ .../mistral-instrumentation/scenario.ts | 5 + e2e/scripts/run-canary-tests-docker.mjs | 1 + .../auto-instrumentations/bundler/plugin.ts | 2 + .../bundler/webpack-loader.ts | 2 + .../configs/claude-agent-sdk.test.ts | 36 -- .../auto-instrumentations/configs/mistral.ts | 200 +++++++ .../configs/openrouter.test.ts | 21 - js/src/auto-instrumentations/hook.mts | 2 + js/src/auto-instrumentations/index.ts | 1 + js/src/exports.ts | 1 + .../instrumentation/braintrust-plugin.test.ts | 52 ++ js/src/instrumentation/braintrust-plugin.ts | 14 + .../plugins/mistral-channels.ts | 78 +++ .../plugins/mistral-plugin.test.ts | 169 ++++++ .../instrumentation/plugins/mistral-plugin.ts | 499 ++++++++++++++++ js/src/instrumentation/registry.test.ts | 1 + js/src/instrumentation/registry.ts | 2 + js/src/vendor-sdk-types/mistral.ts | 171 ++++++ js/src/wrappers/mistral.ts | 245 ++++++++ turbo.json | 24 +- 55 files changed, 5980 insertions(+), 69 deletions(-) create mode 100644 e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-10-0.log-payloads.json create mode 100644 e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-10-0.span-events.json create mode 100644 e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-14-1.log-payloads.json create mode 100644 e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-14-1.span-events.json create mode 100644 e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-15-1.log-payloads.json create mode 100644 e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-15-1.span-events.json create mode 100644 e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-3-4.log-payloads.json create mode 100644 e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-3-4.span-events.json create mode 100644 e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1.log-payloads.json create mode 100644 e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1.span-events.json create mode 100644 e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v2.log-payloads.json create mode 100644 e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v2.span-events.json create mode 100644 e2e/scenarios/mistral-instrumentation/assertions.ts create mode 100644 e2e/scenarios/mistral-instrumentation/constants.mjs create mode 100644 e2e/scenarios/mistral-instrumentation/package.json create mode 100644 e2e/scenarios/mistral-instrumentation/pnpm-lock.yaml create mode 100644 e2e/scenarios/mistral-instrumentation/scenario.impl.mjs create mode 100644 e2e/scenarios/mistral-instrumentation/scenario.mistral-v1-10-0.mjs create mode 100644 e2e/scenarios/mistral-instrumentation/scenario.mistral-v1-10-0.ts create mode 100644 e2e/scenarios/mistral-instrumentation/scenario.mistral-v1-14-1.mjs create mode 100644 e2e/scenarios/mistral-instrumentation/scenario.mistral-v1-14-1.ts create mode 100644 e2e/scenarios/mistral-instrumentation/scenario.mistral-v1-15-1.mjs create mode 100644 e2e/scenarios/mistral-instrumentation/scenario.mistral-v1-15-1.ts create mode 100644 e2e/scenarios/mistral-instrumentation/scenario.mistral-v1-3-4.mjs create mode 100644 e2e/scenarios/mistral-instrumentation/scenario.mistral-v1-3-4.ts create mode 100644 e2e/scenarios/mistral-instrumentation/scenario.mistral-v1.mjs create mode 100644 e2e/scenarios/mistral-instrumentation/scenario.mistral-v1.ts create mode 100644 e2e/scenarios/mistral-instrumentation/scenario.mjs create mode 100644 e2e/scenarios/mistral-instrumentation/scenario.test.ts create mode 100644 e2e/scenarios/mistral-instrumentation/scenario.ts delete mode 100644 js/src/auto-instrumentations/configs/claude-agent-sdk.test.ts create mode 100644 js/src/auto-instrumentations/configs/mistral.ts delete mode 100644 js/src/auto-instrumentations/configs/openrouter.test.ts create mode 100644 js/src/instrumentation/plugins/mistral-channels.ts create mode 100644 js/src/instrumentation/plugins/mistral-plugin.test.ts create mode 100644 js/src/instrumentation/plugins/mistral-plugin.ts create mode 100644 js/src/vendor-sdk-types/mistral.ts create mode 100644 js/src/wrappers/mistral.ts diff --git a/.agents/skills/instrumentation/SKILL.md b/.agents/skills/instrumentation/SKILL.md index eca1287a2..4676b0873 100644 --- a/.agents/skills/instrumentation/SKILL.md +++ b/.agents/skills/instrumentation/SKILL.md @@ -35,6 +35,7 @@ Map the change before editing: - Promise/stream behavior must be preserved. Patches need to keep subclass/helper semantics intact. - Contain instrumentation failures. Extraction/logging bugs should be logged or ignored as appropriate, but must not break the user call path. - Log only the useful surface. Prefer narrow, stable payloads over dumping full request/response objects; exclude redundant or overly large data when possible. +- We want to limit our instrumentation to operations that are relevant for AI generations and operations (LLMs, embeddings, media generation, ...). Things like creating entities on platforms (CRUD for Workflows of Agent entities) is irrelevant to us. ## Process diff --git a/.env.example b/.env.example index 2fc0f5f70..7ade92540 100644 --- a/.env.example +++ b/.env.example @@ -1,4 +1,6 @@ - -BRAINTRUST_API_KEY=your_braintrust_api_key -OPENAI_API_KEY=your_openai_api_key -ANTHROPIC_API_KEY=your_anthropic_key +BRAINTRUST_API_KEY= +OPENAI_API_KEY= +ANTHROPIC_API_KEY= +GEMINI_API_KEY= +OPENROUTER_API_KEY= +MISTRAL_API_KEY= diff --git a/.github/workflows/e2e-canary.yaml b/.github/workflows/e2e-canary.yaml index 71266780e..0aa472263 100644 --- a/.github/workflows/e2e-canary.yaml +++ b/.github/workflows/e2e-canary.yaml @@ -37,6 +37,7 @@ jobs: GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }} OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} + MISTRAL_API_KEY: ${{ secrets.MISTRAL_API_KEY }} run: pnpm test:e2e:canary - name: Create or update nightly canary issue if: ${{ failure() && github.event_name == 'schedule' }} diff --git a/.github/workflows/integration-tests.yaml b/.github/workflows/integration-tests.yaml index 44119c35a..120301dd7 100644 --- a/.github/workflows/integration-tests.yaml +++ b/.github/workflows/integration-tests.yaml @@ -30,6 +30,7 @@ jobs: OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }} OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} + MISTRAL_API_KEY: ${{ secrets.MISTRAL_API_KEY }} steps: - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0 @@ -57,6 +58,7 @@ jobs: GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }} OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} + MISTRAL_API_KEY: ${{ secrets.MISTRAL_API_KEY }} steps: - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 - uses: denoland/setup-deno@ff4860f9d7236f320afa0f82b7e6457384805d05 # v2.0.4 @@ -105,6 +107,7 @@ jobs: GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }} OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} + MISTRAL_API_KEY: ${{ secrets.MISTRAL_API_KEY }} steps: - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0 diff --git a/e2e/README.md b/e2e/README.md index d9d56751d..3ec5a95f7 100644 --- a/e2e/README.md +++ b/e2e/README.md @@ -116,6 +116,7 @@ Non-hermetic scenarios require provider credentials in addition to the mock Brai - `ANTHROPIC_API_KEY` - `GEMINI_API_KEY` or `GOOGLE_API_KEY` - `OPENROUTER_API_KEY` +- `MISTRAL_API_KEY` `claude-agent-sdk-instrumentation` also uses `ANTHROPIC_API_KEY`, because it runs the real Claude Agent SDK against Anthropic in the same style as the existing live Anthropic wrapper coverage. diff --git a/e2e/helpers/scenario-installer.ts b/e2e/helpers/scenario-installer.ts index 1d29fb4ac..bd387f701 100644 --- a/e2e/helpers/scenario-installer.ts +++ b/e2e/helpers/scenario-installer.ts @@ -26,6 +26,7 @@ const INSTALL_SECRET_ENV_VARS = [ "GH_TOKEN", "OPENAI_API_KEY", "OPENROUTER_API_KEY", + "MISTRAL_API_KEY", ] as const; const cleanupDirs = new Set(); diff --git a/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-10-0.log-payloads.json b/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-10-0.log-payloads.json new file mode 100644 index 000000000..2812afcce --- /dev/null +++ b/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-10-0.log-payloads.json @@ -0,0 +1,272 @@ +[ + { + "has_input": false, + "has_output": false, + "input": null, + "metadata": { + "scenario": "mistral-instrumentation" + }, + "metric_keys": [], + "output": null, + "span_id": "" + }, + { + "has_input": false, + "has_output": false, + "input": null, + "metadata": { + "operation": "chat-complete" + }, + "metric_keys": [], + "output": null, + "span_id": "" + }, + { + "has_input": true, + "has_output": true, + "input": { + "item_count": 2, + "roles": [ + "system", + "user" + ], + "type": "messages" + }, + "metadata": { + "model": "mistral-small-latest", + "provider": "mistral" + }, + "metric_keys": [ + "completion_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "output": { + "choice_count": 1, + "has_content": true, + "role": "assistant", + "tool_call_count": 0, + "type": "choices" + }, + "span_id": "" + }, + { + "has_input": false, + "has_output": false, + "input": null, + "metadata": { + "operation": "chat-stream" + }, + "metric_keys": [], + "output": null, + "span_id": "" + }, + { + "has_input": true, + "has_output": true, + "input": { + "item_count": 1, + "roles": [ + "user" + ], + "type": "messages" + }, + "metadata": { + "model": "mistral-small-latest", + "provider": "mistral" + }, + "metric_keys": [ + "completion_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "output": { + "choice_count": 1, + "has_content": true, + "role": "assistant", + "tool_call_count": 0, + "type": "choices" + }, + "span_id": "" + }, + { + "has_input": false, + "has_output": false, + "input": null, + "metadata": { + "operation": "fim-complete" + }, + "metric_keys": [], + "output": null, + "span_id": "" + }, + { + "has_input": true, + "has_output": true, + "input": "", + "metadata": { + "model": "codestral-latest", + "provider": "mistral" + }, + "metric_keys": [ + "completion_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "output": { + "choice_count": 1, + "has_content": true, + "role": "assistant", + "tool_call_count": 0, + "type": "choices" + }, + "span_id": "" + }, + { + "has_input": false, + "has_output": false, + "input": null, + "metadata": { + "operation": "fim-stream" + }, + "metric_keys": [], + "output": null, + "span_id": "" + }, + { + "has_input": true, + "has_output": true, + "input": "", + "metadata": { + "model": "codestral-latest", + "provider": "mistral" + }, + "metric_keys": [ + "completion_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "output": { + "choice_count": 1, + "has_content": true, + "role": "assistant", + "tool_call_count": 0, + "type": "choices" + }, + "span_id": "" + }, + { + "has_input": false, + "has_output": false, + "input": null, + "metadata": { + "operation": "agents-complete" + }, + "metric_keys": [], + "output": null, + "span_id": "" + }, + { + "has_input": true, + "has_output": true, + "input": { + "item_count": 1, + "roles": [ + "user" + ], + "type": "messages" + }, + "metadata": { + "model": "mistral-small-latest", + "provider": "mistral" + }, + "metric_keys": [ + "completion_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "output": { + "choice_count": 1, + "has_content": true, + "role": "assistant", + "tool_call_count": 0, + "type": "choices" + }, + "span_id": "" + }, + { + "has_input": false, + "has_output": false, + "input": null, + "metadata": { + "operation": "agents-stream" + }, + "metric_keys": [], + "output": null, + "span_id": "" + }, + { + "has_input": true, + "has_output": true, + "input": { + "item_count": 1, + "roles": [ + "user" + ], + "type": "messages" + }, + "metadata": { + "model": "mistral-small-latest", + "provider": "mistral" + }, + "metric_keys": [ + "completion_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "output": { + "choice_count": 1, + "has_content": true, + "role": "assistant", + "tool_call_count": 0, + "type": "choices" + }, + "span_id": "" + }, + { + "has_input": false, + "has_output": false, + "input": null, + "metadata": { + "operation": "embeddings-create" + }, + "metric_keys": [], + "output": null, + "span_id": "" + }, + { + "has_input": true, + "has_output": true, + "input": "", + "metadata": { + "model": "mistral-embed", + "provider": "mistral" + }, + "metric_keys": [ + "completion_tokens", + "prompt_tokens", + "tokens" + ], + "output": { + "embedding_length": "", + "type": "embedding" + }, + "span_id": "" + } +] diff --git a/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-10-0.span-events.json b/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-10-0.span-events.json new file mode 100644 index 000000000..eb0733762 --- /dev/null +++ b/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-10-0.span-events.json @@ -0,0 +1,266 @@ +[ + { + "has_input": false, + "has_output": false, + "metadata": { + "scenario": "mistral-instrumentation" + }, + "metric_keys": [], + "name": "mistral-root", + "root_span_id": "", + "span_id": "", + "span_parents": [], + "type": "task" + }, + { + "has_input": false, + "has_output": false, + "metadata": { + "operation": "chat-complete" + }, + "metric_keys": [], + "name": "mistral-chat-complete-operation", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": null + }, + { + "has_input": true, + "has_output": true, + "metadata": { + "model": "mistral-small-latest", + "provider": "mistral" + }, + "metric_keys": [ + "completion_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "name": "mistral.chat.complete", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": "llm" + }, + { + "has_input": false, + "has_output": false, + "metadata": { + "operation": "chat-stream" + }, + "metric_keys": [], + "name": "mistral-chat-stream-operation", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": null + }, + { + "has_input": true, + "has_output": true, + "metadata": { + "model": "mistral-small-latest", + "provider": "mistral" + }, + "metric_keys": [ + "completion_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "name": "mistral.chat.stream", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": "llm" + }, + { + "has_input": false, + "has_output": false, + "metadata": { + "operation": "fim-complete" + }, + "metric_keys": [], + "name": "mistral-fim-complete-operation", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": null + }, + { + "has_input": true, + "has_output": true, + "metadata": { + "model": "codestral-latest", + "provider": "mistral" + }, + "metric_keys": [ + "completion_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "name": "mistral.fim.complete", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": "llm" + }, + { + "has_input": false, + "has_output": false, + "metadata": { + "operation": "fim-stream" + }, + "metric_keys": [], + "name": "mistral-fim-stream-operation", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": null + }, + { + "has_input": true, + "has_output": true, + "metadata": { + "model": "codestral-latest", + "provider": "mistral" + }, + "metric_keys": [ + "completion_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "name": "mistral.fim.stream", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": "llm" + }, + { + "has_input": false, + "has_output": false, + "metadata": { + "operation": "agents-complete" + }, + "metric_keys": [], + "name": "mistral-agents-complete-operation", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": null + }, + { + "has_input": true, + "has_output": true, + "metadata": { + "model": "mistral-small-latest", + "provider": "mistral" + }, + "metric_keys": [ + "completion_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "name": "mistral.agents.complete", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": "llm" + }, + { + "has_input": false, + "has_output": false, + "metadata": { + "operation": "agents-stream" + }, + "metric_keys": [], + "name": "mistral-agents-stream-operation", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": null + }, + { + "has_input": true, + "has_output": true, + "metadata": { + "model": "mistral-small-latest", + "provider": "mistral" + }, + "metric_keys": [ + "completion_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "name": "mistral.agents.stream", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": "llm" + }, + { + "has_input": false, + "has_output": false, + "metadata": { + "operation": "embeddings-create" + }, + "metric_keys": [], + "name": "mistral-embeddings-operation", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": null + }, + { + "has_input": true, + "has_output": true, + "metadata": { + "model": "mistral-embed", + "provider": "mistral" + }, + "metric_keys": [ + "completion_tokens", + "prompt_tokens", + "tokens" + ], + "name": "mistral.embeddings.create", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": "llm" + } +] diff --git a/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-14-1.log-payloads.json b/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-14-1.log-payloads.json new file mode 100644 index 000000000..2812afcce --- /dev/null +++ b/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-14-1.log-payloads.json @@ -0,0 +1,272 @@ +[ + { + "has_input": false, + "has_output": false, + "input": null, + "metadata": { + "scenario": "mistral-instrumentation" + }, + "metric_keys": [], + "output": null, + "span_id": "" + }, + { + "has_input": false, + "has_output": false, + "input": null, + "metadata": { + "operation": "chat-complete" + }, + "metric_keys": [], + "output": null, + "span_id": "" + }, + { + "has_input": true, + "has_output": true, + "input": { + "item_count": 2, + "roles": [ + "system", + "user" + ], + "type": "messages" + }, + "metadata": { + "model": "mistral-small-latest", + "provider": "mistral" + }, + "metric_keys": [ + "completion_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "output": { + "choice_count": 1, + "has_content": true, + "role": "assistant", + "tool_call_count": 0, + "type": "choices" + }, + "span_id": "" + }, + { + "has_input": false, + "has_output": false, + "input": null, + "metadata": { + "operation": "chat-stream" + }, + "metric_keys": [], + "output": null, + "span_id": "" + }, + { + "has_input": true, + "has_output": true, + "input": { + "item_count": 1, + "roles": [ + "user" + ], + "type": "messages" + }, + "metadata": { + "model": "mistral-small-latest", + "provider": "mistral" + }, + "metric_keys": [ + "completion_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "output": { + "choice_count": 1, + "has_content": true, + "role": "assistant", + "tool_call_count": 0, + "type": "choices" + }, + "span_id": "" + }, + { + "has_input": false, + "has_output": false, + "input": null, + "metadata": { + "operation": "fim-complete" + }, + "metric_keys": [], + "output": null, + "span_id": "" + }, + { + "has_input": true, + "has_output": true, + "input": "", + "metadata": { + "model": "codestral-latest", + "provider": "mistral" + }, + "metric_keys": [ + "completion_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "output": { + "choice_count": 1, + "has_content": true, + "role": "assistant", + "tool_call_count": 0, + "type": "choices" + }, + "span_id": "" + }, + { + "has_input": false, + "has_output": false, + "input": null, + "metadata": { + "operation": "fim-stream" + }, + "metric_keys": [], + "output": null, + "span_id": "" + }, + { + "has_input": true, + "has_output": true, + "input": "", + "metadata": { + "model": "codestral-latest", + "provider": "mistral" + }, + "metric_keys": [ + "completion_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "output": { + "choice_count": 1, + "has_content": true, + "role": "assistant", + "tool_call_count": 0, + "type": "choices" + }, + "span_id": "" + }, + { + "has_input": false, + "has_output": false, + "input": null, + "metadata": { + "operation": "agents-complete" + }, + "metric_keys": [], + "output": null, + "span_id": "" + }, + { + "has_input": true, + "has_output": true, + "input": { + "item_count": 1, + "roles": [ + "user" + ], + "type": "messages" + }, + "metadata": { + "model": "mistral-small-latest", + "provider": "mistral" + }, + "metric_keys": [ + "completion_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "output": { + "choice_count": 1, + "has_content": true, + "role": "assistant", + "tool_call_count": 0, + "type": "choices" + }, + "span_id": "" + }, + { + "has_input": false, + "has_output": false, + "input": null, + "metadata": { + "operation": "agents-stream" + }, + "metric_keys": [], + "output": null, + "span_id": "" + }, + { + "has_input": true, + "has_output": true, + "input": { + "item_count": 1, + "roles": [ + "user" + ], + "type": "messages" + }, + "metadata": { + "model": "mistral-small-latest", + "provider": "mistral" + }, + "metric_keys": [ + "completion_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "output": { + "choice_count": 1, + "has_content": true, + "role": "assistant", + "tool_call_count": 0, + "type": "choices" + }, + "span_id": "" + }, + { + "has_input": false, + "has_output": false, + "input": null, + "metadata": { + "operation": "embeddings-create" + }, + "metric_keys": [], + "output": null, + "span_id": "" + }, + { + "has_input": true, + "has_output": true, + "input": "", + "metadata": { + "model": "mistral-embed", + "provider": "mistral" + }, + "metric_keys": [ + "completion_tokens", + "prompt_tokens", + "tokens" + ], + "output": { + "embedding_length": "", + "type": "embedding" + }, + "span_id": "" + } +] diff --git a/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-14-1.span-events.json b/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-14-1.span-events.json new file mode 100644 index 000000000..eb0733762 --- /dev/null +++ b/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-14-1.span-events.json @@ -0,0 +1,266 @@ +[ + { + "has_input": false, + "has_output": false, + "metadata": { + "scenario": "mistral-instrumentation" + }, + "metric_keys": [], + "name": "mistral-root", + "root_span_id": "", + "span_id": "", + "span_parents": [], + "type": "task" + }, + { + "has_input": false, + "has_output": false, + "metadata": { + "operation": "chat-complete" + }, + "metric_keys": [], + "name": "mistral-chat-complete-operation", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": null + }, + { + "has_input": true, + "has_output": true, + "metadata": { + "model": "mistral-small-latest", + "provider": "mistral" + }, + "metric_keys": [ + "completion_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "name": "mistral.chat.complete", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": "llm" + }, + { + "has_input": false, + "has_output": false, + "metadata": { + "operation": "chat-stream" + }, + "metric_keys": [], + "name": "mistral-chat-stream-operation", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": null + }, + { + "has_input": true, + "has_output": true, + "metadata": { + "model": "mistral-small-latest", + "provider": "mistral" + }, + "metric_keys": [ + "completion_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "name": "mistral.chat.stream", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": "llm" + }, + { + "has_input": false, + "has_output": false, + "metadata": { + "operation": "fim-complete" + }, + "metric_keys": [], + "name": "mistral-fim-complete-operation", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": null + }, + { + "has_input": true, + "has_output": true, + "metadata": { + "model": "codestral-latest", + "provider": "mistral" + }, + "metric_keys": [ + "completion_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "name": "mistral.fim.complete", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": "llm" + }, + { + "has_input": false, + "has_output": false, + "metadata": { + "operation": "fim-stream" + }, + "metric_keys": [], + "name": "mistral-fim-stream-operation", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": null + }, + { + "has_input": true, + "has_output": true, + "metadata": { + "model": "codestral-latest", + "provider": "mistral" + }, + "metric_keys": [ + "completion_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "name": "mistral.fim.stream", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": "llm" + }, + { + "has_input": false, + "has_output": false, + "metadata": { + "operation": "agents-complete" + }, + "metric_keys": [], + "name": "mistral-agents-complete-operation", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": null + }, + { + "has_input": true, + "has_output": true, + "metadata": { + "model": "mistral-small-latest", + "provider": "mistral" + }, + "metric_keys": [ + "completion_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "name": "mistral.agents.complete", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": "llm" + }, + { + "has_input": false, + "has_output": false, + "metadata": { + "operation": "agents-stream" + }, + "metric_keys": [], + "name": "mistral-agents-stream-operation", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": null + }, + { + "has_input": true, + "has_output": true, + "metadata": { + "model": "mistral-small-latest", + "provider": "mistral" + }, + "metric_keys": [ + "completion_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "name": "mistral.agents.stream", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": "llm" + }, + { + "has_input": false, + "has_output": false, + "metadata": { + "operation": "embeddings-create" + }, + "metric_keys": [], + "name": "mistral-embeddings-operation", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": null + }, + { + "has_input": true, + "has_output": true, + "metadata": { + "model": "mistral-embed", + "provider": "mistral" + }, + "metric_keys": [ + "completion_tokens", + "prompt_tokens", + "tokens" + ], + "name": "mistral.embeddings.create", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": "llm" + } +] diff --git a/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-15-1.log-payloads.json b/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-15-1.log-payloads.json new file mode 100644 index 000000000..2812afcce --- /dev/null +++ b/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-15-1.log-payloads.json @@ -0,0 +1,272 @@ +[ + { + "has_input": false, + "has_output": false, + "input": null, + "metadata": { + "scenario": "mistral-instrumentation" + }, + "metric_keys": [], + "output": null, + "span_id": "" + }, + { + "has_input": false, + "has_output": false, + "input": null, + "metadata": { + "operation": "chat-complete" + }, + "metric_keys": [], + "output": null, + "span_id": "" + }, + { + "has_input": true, + "has_output": true, + "input": { + "item_count": 2, + "roles": [ + "system", + "user" + ], + "type": "messages" + }, + "metadata": { + "model": "mistral-small-latest", + "provider": "mistral" + }, + "metric_keys": [ + "completion_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "output": { + "choice_count": 1, + "has_content": true, + "role": "assistant", + "tool_call_count": 0, + "type": "choices" + }, + "span_id": "" + }, + { + "has_input": false, + "has_output": false, + "input": null, + "metadata": { + "operation": "chat-stream" + }, + "metric_keys": [], + "output": null, + "span_id": "" + }, + { + "has_input": true, + "has_output": true, + "input": { + "item_count": 1, + "roles": [ + "user" + ], + "type": "messages" + }, + "metadata": { + "model": "mistral-small-latest", + "provider": "mistral" + }, + "metric_keys": [ + "completion_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "output": { + "choice_count": 1, + "has_content": true, + "role": "assistant", + "tool_call_count": 0, + "type": "choices" + }, + "span_id": "" + }, + { + "has_input": false, + "has_output": false, + "input": null, + "metadata": { + "operation": "fim-complete" + }, + "metric_keys": [], + "output": null, + "span_id": "" + }, + { + "has_input": true, + "has_output": true, + "input": "", + "metadata": { + "model": "codestral-latest", + "provider": "mistral" + }, + "metric_keys": [ + "completion_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "output": { + "choice_count": 1, + "has_content": true, + "role": "assistant", + "tool_call_count": 0, + "type": "choices" + }, + "span_id": "" + }, + { + "has_input": false, + "has_output": false, + "input": null, + "metadata": { + "operation": "fim-stream" + }, + "metric_keys": [], + "output": null, + "span_id": "" + }, + { + "has_input": true, + "has_output": true, + "input": "", + "metadata": { + "model": "codestral-latest", + "provider": "mistral" + }, + "metric_keys": [ + "completion_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "output": { + "choice_count": 1, + "has_content": true, + "role": "assistant", + "tool_call_count": 0, + "type": "choices" + }, + "span_id": "" + }, + { + "has_input": false, + "has_output": false, + "input": null, + "metadata": { + "operation": "agents-complete" + }, + "metric_keys": [], + "output": null, + "span_id": "" + }, + { + "has_input": true, + "has_output": true, + "input": { + "item_count": 1, + "roles": [ + "user" + ], + "type": "messages" + }, + "metadata": { + "model": "mistral-small-latest", + "provider": "mistral" + }, + "metric_keys": [ + "completion_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "output": { + "choice_count": 1, + "has_content": true, + "role": "assistant", + "tool_call_count": 0, + "type": "choices" + }, + "span_id": "" + }, + { + "has_input": false, + "has_output": false, + "input": null, + "metadata": { + "operation": "agents-stream" + }, + "metric_keys": [], + "output": null, + "span_id": "" + }, + { + "has_input": true, + "has_output": true, + "input": { + "item_count": 1, + "roles": [ + "user" + ], + "type": "messages" + }, + "metadata": { + "model": "mistral-small-latest", + "provider": "mistral" + }, + "metric_keys": [ + "completion_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "output": { + "choice_count": 1, + "has_content": true, + "role": "assistant", + "tool_call_count": 0, + "type": "choices" + }, + "span_id": "" + }, + { + "has_input": false, + "has_output": false, + "input": null, + "metadata": { + "operation": "embeddings-create" + }, + "metric_keys": [], + "output": null, + "span_id": "" + }, + { + "has_input": true, + "has_output": true, + "input": "", + "metadata": { + "model": "mistral-embed", + "provider": "mistral" + }, + "metric_keys": [ + "completion_tokens", + "prompt_tokens", + "tokens" + ], + "output": { + "embedding_length": "", + "type": "embedding" + }, + "span_id": "" + } +] diff --git a/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-15-1.span-events.json b/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-15-1.span-events.json new file mode 100644 index 000000000..eb0733762 --- /dev/null +++ b/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-15-1.span-events.json @@ -0,0 +1,266 @@ +[ + { + "has_input": false, + "has_output": false, + "metadata": { + "scenario": "mistral-instrumentation" + }, + "metric_keys": [], + "name": "mistral-root", + "root_span_id": "", + "span_id": "", + "span_parents": [], + "type": "task" + }, + { + "has_input": false, + "has_output": false, + "metadata": { + "operation": "chat-complete" + }, + "metric_keys": [], + "name": "mistral-chat-complete-operation", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": null + }, + { + "has_input": true, + "has_output": true, + "metadata": { + "model": "mistral-small-latest", + "provider": "mistral" + }, + "metric_keys": [ + "completion_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "name": "mistral.chat.complete", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": "llm" + }, + { + "has_input": false, + "has_output": false, + "metadata": { + "operation": "chat-stream" + }, + "metric_keys": [], + "name": "mistral-chat-stream-operation", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": null + }, + { + "has_input": true, + "has_output": true, + "metadata": { + "model": "mistral-small-latest", + "provider": "mistral" + }, + "metric_keys": [ + "completion_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "name": "mistral.chat.stream", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": "llm" + }, + { + "has_input": false, + "has_output": false, + "metadata": { + "operation": "fim-complete" + }, + "metric_keys": [], + "name": "mistral-fim-complete-operation", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": null + }, + { + "has_input": true, + "has_output": true, + "metadata": { + "model": "codestral-latest", + "provider": "mistral" + }, + "metric_keys": [ + "completion_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "name": "mistral.fim.complete", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": "llm" + }, + { + "has_input": false, + "has_output": false, + "metadata": { + "operation": "fim-stream" + }, + "metric_keys": [], + "name": "mistral-fim-stream-operation", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": null + }, + { + "has_input": true, + "has_output": true, + "metadata": { + "model": "codestral-latest", + "provider": "mistral" + }, + "metric_keys": [ + "completion_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "name": "mistral.fim.stream", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": "llm" + }, + { + "has_input": false, + "has_output": false, + "metadata": { + "operation": "agents-complete" + }, + "metric_keys": [], + "name": "mistral-agents-complete-operation", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": null + }, + { + "has_input": true, + "has_output": true, + "metadata": { + "model": "mistral-small-latest", + "provider": "mistral" + }, + "metric_keys": [ + "completion_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "name": "mistral.agents.complete", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": "llm" + }, + { + "has_input": false, + "has_output": false, + "metadata": { + "operation": "agents-stream" + }, + "metric_keys": [], + "name": "mistral-agents-stream-operation", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": null + }, + { + "has_input": true, + "has_output": true, + "metadata": { + "model": "mistral-small-latest", + "provider": "mistral" + }, + "metric_keys": [ + "completion_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "name": "mistral.agents.stream", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": "llm" + }, + { + "has_input": false, + "has_output": false, + "metadata": { + "operation": "embeddings-create" + }, + "metric_keys": [], + "name": "mistral-embeddings-operation", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": null + }, + { + "has_input": true, + "has_output": true, + "metadata": { + "model": "mistral-embed", + "provider": "mistral" + }, + "metric_keys": [ + "completion_tokens", + "prompt_tokens", + "tokens" + ], + "name": "mistral.embeddings.create", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": "llm" + } +] diff --git a/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-3-4.log-payloads.json b/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-3-4.log-payloads.json new file mode 100644 index 000000000..2812afcce --- /dev/null +++ b/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-3-4.log-payloads.json @@ -0,0 +1,272 @@ +[ + { + "has_input": false, + "has_output": false, + "input": null, + "metadata": { + "scenario": "mistral-instrumentation" + }, + "metric_keys": [], + "output": null, + "span_id": "" + }, + { + "has_input": false, + "has_output": false, + "input": null, + "metadata": { + "operation": "chat-complete" + }, + "metric_keys": [], + "output": null, + "span_id": "" + }, + { + "has_input": true, + "has_output": true, + "input": { + "item_count": 2, + "roles": [ + "system", + "user" + ], + "type": "messages" + }, + "metadata": { + "model": "mistral-small-latest", + "provider": "mistral" + }, + "metric_keys": [ + "completion_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "output": { + "choice_count": 1, + "has_content": true, + "role": "assistant", + "tool_call_count": 0, + "type": "choices" + }, + "span_id": "" + }, + { + "has_input": false, + "has_output": false, + "input": null, + "metadata": { + "operation": "chat-stream" + }, + "metric_keys": [], + "output": null, + "span_id": "" + }, + { + "has_input": true, + "has_output": true, + "input": { + "item_count": 1, + "roles": [ + "user" + ], + "type": "messages" + }, + "metadata": { + "model": "mistral-small-latest", + "provider": "mistral" + }, + "metric_keys": [ + "completion_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "output": { + "choice_count": 1, + "has_content": true, + "role": "assistant", + "tool_call_count": 0, + "type": "choices" + }, + "span_id": "" + }, + { + "has_input": false, + "has_output": false, + "input": null, + "metadata": { + "operation": "fim-complete" + }, + "metric_keys": [], + "output": null, + "span_id": "" + }, + { + "has_input": true, + "has_output": true, + "input": "", + "metadata": { + "model": "codestral-latest", + "provider": "mistral" + }, + "metric_keys": [ + "completion_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "output": { + "choice_count": 1, + "has_content": true, + "role": "assistant", + "tool_call_count": 0, + "type": "choices" + }, + "span_id": "" + }, + { + "has_input": false, + "has_output": false, + "input": null, + "metadata": { + "operation": "fim-stream" + }, + "metric_keys": [], + "output": null, + "span_id": "" + }, + { + "has_input": true, + "has_output": true, + "input": "", + "metadata": { + "model": "codestral-latest", + "provider": "mistral" + }, + "metric_keys": [ + "completion_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "output": { + "choice_count": 1, + "has_content": true, + "role": "assistant", + "tool_call_count": 0, + "type": "choices" + }, + "span_id": "" + }, + { + "has_input": false, + "has_output": false, + "input": null, + "metadata": { + "operation": "agents-complete" + }, + "metric_keys": [], + "output": null, + "span_id": "" + }, + { + "has_input": true, + "has_output": true, + "input": { + "item_count": 1, + "roles": [ + "user" + ], + "type": "messages" + }, + "metadata": { + "model": "mistral-small-latest", + "provider": "mistral" + }, + "metric_keys": [ + "completion_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "output": { + "choice_count": 1, + "has_content": true, + "role": "assistant", + "tool_call_count": 0, + "type": "choices" + }, + "span_id": "" + }, + { + "has_input": false, + "has_output": false, + "input": null, + "metadata": { + "operation": "agents-stream" + }, + "metric_keys": [], + "output": null, + "span_id": "" + }, + { + "has_input": true, + "has_output": true, + "input": { + "item_count": 1, + "roles": [ + "user" + ], + "type": "messages" + }, + "metadata": { + "model": "mistral-small-latest", + "provider": "mistral" + }, + "metric_keys": [ + "completion_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "output": { + "choice_count": 1, + "has_content": true, + "role": "assistant", + "tool_call_count": 0, + "type": "choices" + }, + "span_id": "" + }, + { + "has_input": false, + "has_output": false, + "input": null, + "metadata": { + "operation": "embeddings-create" + }, + "metric_keys": [], + "output": null, + "span_id": "" + }, + { + "has_input": true, + "has_output": true, + "input": "", + "metadata": { + "model": "mistral-embed", + "provider": "mistral" + }, + "metric_keys": [ + "completion_tokens", + "prompt_tokens", + "tokens" + ], + "output": { + "embedding_length": "", + "type": "embedding" + }, + "span_id": "" + } +] diff --git a/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-3-4.span-events.json b/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-3-4.span-events.json new file mode 100644 index 000000000..eb0733762 --- /dev/null +++ b/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-3-4.span-events.json @@ -0,0 +1,266 @@ +[ + { + "has_input": false, + "has_output": false, + "metadata": { + "scenario": "mistral-instrumentation" + }, + "metric_keys": [], + "name": "mistral-root", + "root_span_id": "", + "span_id": "", + "span_parents": [], + "type": "task" + }, + { + "has_input": false, + "has_output": false, + "metadata": { + "operation": "chat-complete" + }, + "metric_keys": [], + "name": "mistral-chat-complete-operation", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": null + }, + { + "has_input": true, + "has_output": true, + "metadata": { + "model": "mistral-small-latest", + "provider": "mistral" + }, + "metric_keys": [ + "completion_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "name": "mistral.chat.complete", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": "llm" + }, + { + "has_input": false, + "has_output": false, + "metadata": { + "operation": "chat-stream" + }, + "metric_keys": [], + "name": "mistral-chat-stream-operation", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": null + }, + { + "has_input": true, + "has_output": true, + "metadata": { + "model": "mistral-small-latest", + "provider": "mistral" + }, + "metric_keys": [ + "completion_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "name": "mistral.chat.stream", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": "llm" + }, + { + "has_input": false, + "has_output": false, + "metadata": { + "operation": "fim-complete" + }, + "metric_keys": [], + "name": "mistral-fim-complete-operation", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": null + }, + { + "has_input": true, + "has_output": true, + "metadata": { + "model": "codestral-latest", + "provider": "mistral" + }, + "metric_keys": [ + "completion_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "name": "mistral.fim.complete", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": "llm" + }, + { + "has_input": false, + "has_output": false, + "metadata": { + "operation": "fim-stream" + }, + "metric_keys": [], + "name": "mistral-fim-stream-operation", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": null + }, + { + "has_input": true, + "has_output": true, + "metadata": { + "model": "codestral-latest", + "provider": "mistral" + }, + "metric_keys": [ + "completion_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "name": "mistral.fim.stream", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": "llm" + }, + { + "has_input": false, + "has_output": false, + "metadata": { + "operation": "agents-complete" + }, + "metric_keys": [], + "name": "mistral-agents-complete-operation", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": null + }, + { + "has_input": true, + "has_output": true, + "metadata": { + "model": "mistral-small-latest", + "provider": "mistral" + }, + "metric_keys": [ + "completion_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "name": "mistral.agents.complete", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": "llm" + }, + { + "has_input": false, + "has_output": false, + "metadata": { + "operation": "agents-stream" + }, + "metric_keys": [], + "name": "mistral-agents-stream-operation", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": null + }, + { + "has_input": true, + "has_output": true, + "metadata": { + "model": "mistral-small-latest", + "provider": "mistral" + }, + "metric_keys": [ + "completion_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "name": "mistral.agents.stream", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": "llm" + }, + { + "has_input": false, + "has_output": false, + "metadata": { + "operation": "embeddings-create" + }, + "metric_keys": [], + "name": "mistral-embeddings-operation", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": null + }, + { + "has_input": true, + "has_output": true, + "metadata": { + "model": "mistral-embed", + "provider": "mistral" + }, + "metric_keys": [ + "completion_tokens", + "prompt_tokens", + "tokens" + ], + "name": "mistral.embeddings.create", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": "llm" + } +] diff --git a/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1.log-payloads.json b/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1.log-payloads.json new file mode 100644 index 000000000..2812afcce --- /dev/null +++ b/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1.log-payloads.json @@ -0,0 +1,272 @@ +[ + { + "has_input": false, + "has_output": false, + "input": null, + "metadata": { + "scenario": "mistral-instrumentation" + }, + "metric_keys": [], + "output": null, + "span_id": "" + }, + { + "has_input": false, + "has_output": false, + "input": null, + "metadata": { + "operation": "chat-complete" + }, + "metric_keys": [], + "output": null, + "span_id": "" + }, + { + "has_input": true, + "has_output": true, + "input": { + "item_count": 2, + "roles": [ + "system", + "user" + ], + "type": "messages" + }, + "metadata": { + "model": "mistral-small-latest", + "provider": "mistral" + }, + "metric_keys": [ + "completion_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "output": { + "choice_count": 1, + "has_content": true, + "role": "assistant", + "tool_call_count": 0, + "type": "choices" + }, + "span_id": "" + }, + { + "has_input": false, + "has_output": false, + "input": null, + "metadata": { + "operation": "chat-stream" + }, + "metric_keys": [], + "output": null, + "span_id": "" + }, + { + "has_input": true, + "has_output": true, + "input": { + "item_count": 1, + "roles": [ + "user" + ], + "type": "messages" + }, + "metadata": { + "model": "mistral-small-latest", + "provider": "mistral" + }, + "metric_keys": [ + "completion_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "output": { + "choice_count": 1, + "has_content": true, + "role": "assistant", + "tool_call_count": 0, + "type": "choices" + }, + "span_id": "" + }, + { + "has_input": false, + "has_output": false, + "input": null, + "metadata": { + "operation": "fim-complete" + }, + "metric_keys": [], + "output": null, + "span_id": "" + }, + { + "has_input": true, + "has_output": true, + "input": "", + "metadata": { + "model": "codestral-latest", + "provider": "mistral" + }, + "metric_keys": [ + "completion_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "output": { + "choice_count": 1, + "has_content": true, + "role": "assistant", + "tool_call_count": 0, + "type": "choices" + }, + "span_id": "" + }, + { + "has_input": false, + "has_output": false, + "input": null, + "metadata": { + "operation": "fim-stream" + }, + "metric_keys": [], + "output": null, + "span_id": "" + }, + { + "has_input": true, + "has_output": true, + "input": "", + "metadata": { + "model": "codestral-latest", + "provider": "mistral" + }, + "metric_keys": [ + "completion_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "output": { + "choice_count": 1, + "has_content": true, + "role": "assistant", + "tool_call_count": 0, + "type": "choices" + }, + "span_id": "" + }, + { + "has_input": false, + "has_output": false, + "input": null, + "metadata": { + "operation": "agents-complete" + }, + "metric_keys": [], + "output": null, + "span_id": "" + }, + { + "has_input": true, + "has_output": true, + "input": { + "item_count": 1, + "roles": [ + "user" + ], + "type": "messages" + }, + "metadata": { + "model": "mistral-small-latest", + "provider": "mistral" + }, + "metric_keys": [ + "completion_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "output": { + "choice_count": 1, + "has_content": true, + "role": "assistant", + "tool_call_count": 0, + "type": "choices" + }, + "span_id": "" + }, + { + "has_input": false, + "has_output": false, + "input": null, + "metadata": { + "operation": "agents-stream" + }, + "metric_keys": [], + "output": null, + "span_id": "" + }, + { + "has_input": true, + "has_output": true, + "input": { + "item_count": 1, + "roles": [ + "user" + ], + "type": "messages" + }, + "metadata": { + "model": "mistral-small-latest", + "provider": "mistral" + }, + "metric_keys": [ + "completion_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "output": { + "choice_count": 1, + "has_content": true, + "role": "assistant", + "tool_call_count": 0, + "type": "choices" + }, + "span_id": "" + }, + { + "has_input": false, + "has_output": false, + "input": null, + "metadata": { + "operation": "embeddings-create" + }, + "metric_keys": [], + "output": null, + "span_id": "" + }, + { + "has_input": true, + "has_output": true, + "input": "", + "metadata": { + "model": "mistral-embed", + "provider": "mistral" + }, + "metric_keys": [ + "completion_tokens", + "prompt_tokens", + "tokens" + ], + "output": { + "embedding_length": "", + "type": "embedding" + }, + "span_id": "" + } +] diff --git a/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1.span-events.json b/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1.span-events.json new file mode 100644 index 000000000..eb0733762 --- /dev/null +++ b/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1.span-events.json @@ -0,0 +1,266 @@ +[ + { + "has_input": false, + "has_output": false, + "metadata": { + "scenario": "mistral-instrumentation" + }, + "metric_keys": [], + "name": "mistral-root", + "root_span_id": "", + "span_id": "", + "span_parents": [], + "type": "task" + }, + { + "has_input": false, + "has_output": false, + "metadata": { + "operation": "chat-complete" + }, + "metric_keys": [], + "name": "mistral-chat-complete-operation", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": null + }, + { + "has_input": true, + "has_output": true, + "metadata": { + "model": "mistral-small-latest", + "provider": "mistral" + }, + "metric_keys": [ + "completion_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "name": "mistral.chat.complete", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": "llm" + }, + { + "has_input": false, + "has_output": false, + "metadata": { + "operation": "chat-stream" + }, + "metric_keys": [], + "name": "mistral-chat-stream-operation", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": null + }, + { + "has_input": true, + "has_output": true, + "metadata": { + "model": "mistral-small-latest", + "provider": "mistral" + }, + "metric_keys": [ + "completion_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "name": "mistral.chat.stream", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": "llm" + }, + { + "has_input": false, + "has_output": false, + "metadata": { + "operation": "fim-complete" + }, + "metric_keys": [], + "name": "mistral-fim-complete-operation", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": null + }, + { + "has_input": true, + "has_output": true, + "metadata": { + "model": "codestral-latest", + "provider": "mistral" + }, + "metric_keys": [ + "completion_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "name": "mistral.fim.complete", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": "llm" + }, + { + "has_input": false, + "has_output": false, + "metadata": { + "operation": "fim-stream" + }, + "metric_keys": [], + "name": "mistral-fim-stream-operation", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": null + }, + { + "has_input": true, + "has_output": true, + "metadata": { + "model": "codestral-latest", + "provider": "mistral" + }, + "metric_keys": [ + "completion_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "name": "mistral.fim.stream", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": "llm" + }, + { + "has_input": false, + "has_output": false, + "metadata": { + "operation": "agents-complete" + }, + "metric_keys": [], + "name": "mistral-agents-complete-operation", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": null + }, + { + "has_input": true, + "has_output": true, + "metadata": { + "model": "mistral-small-latest", + "provider": "mistral" + }, + "metric_keys": [ + "completion_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "name": "mistral.agents.complete", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": "llm" + }, + { + "has_input": false, + "has_output": false, + "metadata": { + "operation": "agents-stream" + }, + "metric_keys": [], + "name": "mistral-agents-stream-operation", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": null + }, + { + "has_input": true, + "has_output": true, + "metadata": { + "model": "mistral-small-latest", + "provider": "mistral" + }, + "metric_keys": [ + "completion_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "name": "mistral.agents.stream", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": "llm" + }, + { + "has_input": false, + "has_output": false, + "metadata": { + "operation": "embeddings-create" + }, + "metric_keys": [], + "name": "mistral-embeddings-operation", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": null + }, + { + "has_input": true, + "has_output": true, + "metadata": { + "model": "mistral-embed", + "provider": "mistral" + }, + "metric_keys": [ + "completion_tokens", + "prompt_tokens", + "tokens" + ], + "name": "mistral.embeddings.create", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": "llm" + } +] diff --git a/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v2.log-payloads.json b/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v2.log-payloads.json new file mode 100644 index 000000000..abb4d8a80 --- /dev/null +++ b/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v2.log-payloads.json @@ -0,0 +1,278 @@ +[ + { + "has_input": false, + "has_output": false, + "input": null, + "metadata": { + "scenario": "mistral-instrumentation" + }, + "metric_keys": [], + "output": null, + "span_id": "" + }, + { + "has_input": false, + "has_output": false, + "input": null, + "metadata": { + "operation": "chat-complete" + }, + "metric_keys": [], + "output": null, + "span_id": "" + }, + { + "has_input": true, + "has_output": true, + "input": { + "item_count": 2, + "roles": [ + "system", + "user" + ], + "type": "messages" + }, + "metadata": { + "model": "mistral-small-latest", + "provider": "mistral" + }, + "metric_keys": [ + "completion_tokens", + "prompt_cached_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "output": { + "choice_count": 1, + "has_content": true, + "role": "assistant", + "tool_call_count": 0, + "type": "choices" + }, + "span_id": "" + }, + { + "has_input": false, + "has_output": false, + "input": null, + "metadata": { + "operation": "chat-stream" + }, + "metric_keys": [], + "output": null, + "span_id": "" + }, + { + "has_input": true, + "has_output": true, + "input": { + "item_count": 1, + "roles": [ + "user" + ], + "type": "messages" + }, + "metadata": { + "model": "mistral-small-latest", + "provider": "mistral" + }, + "metric_keys": [ + "completion_tokens", + "prompt_cached_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "output": { + "choice_count": 1, + "has_content": true, + "role": "assistant", + "tool_call_count": 0, + "type": "choices" + }, + "span_id": "" + }, + { + "has_input": false, + "has_output": false, + "input": null, + "metadata": { + "operation": "fim-complete" + }, + "metric_keys": [], + "output": null, + "span_id": "" + }, + { + "has_input": true, + "has_output": true, + "input": "", + "metadata": { + "model": "codestral-latest", + "provider": "mistral" + }, + "metric_keys": [ + "completion_tokens", + "prompt_cached_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "output": { + "choice_count": 1, + "has_content": true, + "role": "assistant", + "tool_call_count": 0, + "type": "choices" + }, + "span_id": "" + }, + { + "has_input": false, + "has_output": false, + "input": null, + "metadata": { + "operation": "fim-stream" + }, + "metric_keys": [], + "output": null, + "span_id": "" + }, + { + "has_input": true, + "has_output": true, + "input": "", + "metadata": { + "model": "codestral-latest", + "provider": "mistral" + }, + "metric_keys": [ + "completion_tokens", + "prompt_cached_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "output": { + "choice_count": 1, + "has_content": true, + "role": "assistant", + "tool_call_count": 0, + "type": "choices" + }, + "span_id": "" + }, + { + "has_input": false, + "has_output": false, + "input": null, + "metadata": { + "operation": "agents-complete" + }, + "metric_keys": [], + "output": null, + "span_id": "" + }, + { + "has_input": true, + "has_output": true, + "input": { + "item_count": 1, + "roles": [ + "user" + ], + "type": "messages" + }, + "metadata": { + "model": "mistral-small-latest", + "provider": "mistral" + }, + "metric_keys": [ + "completion_tokens", + "prompt_cached_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "output": { + "choice_count": 1, + "has_content": true, + "role": "assistant", + "tool_call_count": 0, + "type": "choices" + }, + "span_id": "" + }, + { + "has_input": false, + "has_output": false, + "input": null, + "metadata": { + "operation": "agents-stream" + }, + "metric_keys": [], + "output": null, + "span_id": "" + }, + { + "has_input": true, + "has_output": true, + "input": { + "item_count": 1, + "roles": [ + "user" + ], + "type": "messages" + }, + "metadata": { + "model": "mistral-small-latest", + "provider": "mistral" + }, + "metric_keys": [ + "completion_tokens", + "prompt_cached_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "output": { + "choice_count": 1, + "has_content": true, + "role": "assistant", + "tool_call_count": 0, + "type": "choices" + }, + "span_id": "" + }, + { + "has_input": false, + "has_output": false, + "input": null, + "metadata": { + "operation": "embeddings-create" + }, + "metric_keys": [], + "output": null, + "span_id": "" + }, + { + "has_input": true, + "has_output": true, + "input": "", + "metadata": { + "model": "mistral-embed", + "provider": "mistral" + }, + "metric_keys": [ + "completion_tokens", + "prompt_tokens", + "tokens" + ], + "output": { + "embedding_length": "", + "type": "embedding" + }, + "span_id": "" + } +] diff --git a/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v2.span-events.json b/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v2.span-events.json new file mode 100644 index 000000000..a55bc5909 --- /dev/null +++ b/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v2.span-events.json @@ -0,0 +1,272 @@ +[ + { + "has_input": false, + "has_output": false, + "metadata": { + "scenario": "mistral-instrumentation" + }, + "metric_keys": [], + "name": "mistral-root", + "root_span_id": "", + "span_id": "", + "span_parents": [], + "type": "task" + }, + { + "has_input": false, + "has_output": false, + "metadata": { + "operation": "chat-complete" + }, + "metric_keys": [], + "name": "mistral-chat-complete-operation", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": null + }, + { + "has_input": true, + "has_output": true, + "metadata": { + "model": "mistral-small-latest", + "provider": "mistral" + }, + "metric_keys": [ + "completion_tokens", + "prompt_cached_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "name": "mistral.chat.complete", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": "llm" + }, + { + "has_input": false, + "has_output": false, + "metadata": { + "operation": "chat-stream" + }, + "metric_keys": [], + "name": "mistral-chat-stream-operation", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": null + }, + { + "has_input": true, + "has_output": true, + "metadata": { + "model": "mistral-small-latest", + "provider": "mistral" + }, + "metric_keys": [ + "completion_tokens", + "prompt_cached_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "name": "mistral.chat.stream", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": "llm" + }, + { + "has_input": false, + "has_output": false, + "metadata": { + "operation": "fim-complete" + }, + "metric_keys": [], + "name": "mistral-fim-complete-operation", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": null + }, + { + "has_input": true, + "has_output": true, + "metadata": { + "model": "codestral-latest", + "provider": "mistral" + }, + "metric_keys": [ + "completion_tokens", + "prompt_cached_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "name": "mistral.fim.complete", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": "llm" + }, + { + "has_input": false, + "has_output": false, + "metadata": { + "operation": "fim-stream" + }, + "metric_keys": [], + "name": "mistral-fim-stream-operation", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": null + }, + { + "has_input": true, + "has_output": true, + "metadata": { + "model": "codestral-latest", + "provider": "mistral" + }, + "metric_keys": [ + "completion_tokens", + "prompt_cached_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "name": "mistral.fim.stream", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": "llm" + }, + { + "has_input": false, + "has_output": false, + "metadata": { + "operation": "agents-complete" + }, + "metric_keys": [], + "name": "mistral-agents-complete-operation", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": null + }, + { + "has_input": true, + "has_output": true, + "metadata": { + "model": "mistral-small-latest", + "provider": "mistral" + }, + "metric_keys": [ + "completion_tokens", + "prompt_cached_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "name": "mistral.agents.complete", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": "llm" + }, + { + "has_input": false, + "has_output": false, + "metadata": { + "operation": "agents-stream" + }, + "metric_keys": [], + "name": "mistral-agents-stream-operation", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": null + }, + { + "has_input": true, + "has_output": true, + "metadata": { + "model": "mistral-small-latest", + "provider": "mistral" + }, + "metric_keys": [ + "completion_tokens", + "prompt_cached_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "name": "mistral.agents.stream", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": "llm" + }, + { + "has_input": false, + "has_output": false, + "metadata": { + "operation": "embeddings-create" + }, + "metric_keys": [], + "name": "mistral-embeddings-operation", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": null + }, + { + "has_input": true, + "has_output": true, + "metadata": { + "model": "mistral-embed", + "provider": "mistral" + }, + "metric_keys": [ + "completion_tokens", + "prompt_tokens", + "tokens" + ], + "name": "mistral.embeddings.create", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": "llm" + } +] diff --git a/e2e/scenarios/mistral-instrumentation/assertions.ts b/e2e/scenarios/mistral-instrumentation/assertions.ts new file mode 100644 index 000000000..2dccabdaa --- /dev/null +++ b/e2e/scenarios/mistral-instrumentation/assertions.ts @@ -0,0 +1,546 @@ +import { beforeAll, describe, expect, test } from "vitest"; +import type { + CapturedLogEvent, + CapturedLogPayload, + CapturedLogRow, +} from "../../helpers/mock-braintrust-server"; +import type { Json } from "../../helpers/normalize"; +import { normalizeForSnapshot } from "../../helpers/normalize"; +import { + formatJsonFileSnapshot, + resolveFileSnapshotPath, +} from "../../helpers/file-snapshot"; +import { withScenarioHarness } from "../../helpers/scenario-harness"; +import { findChildSpans, findLatestSpan } from "../../helpers/trace-selectors"; +import { + payloadRowsForRootSpan, + summarizeWrapperContract, +} from "../../helpers/wrapper-contract"; +import { + FIM_MODEL, + CHAT_MODEL, + AGENT_MODEL, + EMBEDDING_MODEL, + ROOT_NAME, + SCENARIO_NAME, +} from "./constants.mjs"; + +type RunMistralScenario = (harness: { + runNodeScenarioDir: (options: { + entry: string; + nodeArgs: string[]; + runContext?: { variantKey: string }; + scenarioDir: string; + timeoutMs: number; + }) => Promise; + runScenarioDir: (options: { + entry: string; + runContext?: { variantKey: string }; + scenarioDir: string; + timeoutMs: number; + }) => Promise; +}) => Promise; + +function findMistralSpan( + events: CapturedLogEvent[], + parentId: string | undefined, + names: string[], +) { + for (const name of names) { + const spans = findChildSpans(events, name, parentId); + const spanWithOutput = spans.find((candidate) => candidate.output != null); + if (spanWithOutput) { + return spanWithOutput; + } + if (spans[0]) { + return spans[0]; + } + } + + return undefined; +} + +function isRecord(value: Json | undefined): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value); +} + +function pickMetadata( + metadata: Record | undefined, + keys: string[], +): Json { + if (!metadata) { + return null; + } + + const picked = Object.fromEntries( + keys.flatMap((key) => + key in metadata ? [[key, metadata[key] as Json]] : [], + ), + ); + + return Object.keys(picked).length > 0 ? (picked as Json) : null; +} + +function summarizeInput(input: unknown): Json { + if (Array.isArray(input)) { + const roles = input + .map((message) => + isRecord(message as Json) && typeof message.role === "string" + ? message.role + : null, + ) + .filter((role): role is string => role !== null); + + return { + item_count: input.length, + roles, + type: "messages", + }; + } + + if (typeof input === "string") { + return ""; + } + + if (!isRecord(input as Json)) { + return null; + } + + return { + keys: Object.keys(input).sort(), + type: "object", + }; +} + +function summarizeOutput(output: unknown): Json { + if (Array.isArray(output)) { + const firstChoice = output[0] as Json; + if (!isRecord(firstChoice) || !isRecord(firstChoice.message)) { + return { + choice_count: output.length, + type: "array", + }; + } + + const message = firstChoice.message; + const toolCalls = + (Array.isArray(message.tool_calls) && message.tool_calls) || + (Array.isArray(message.toolCalls) && message.toolCalls) || + []; + + return { + choice_count: output.length, + has_content: + typeof message.content === "string" + ? message.content.length > 0 + : false, + role: typeof message.role === "string" ? message.role : null, + tool_call_count: toolCalls.length, + type: "choices", + }; + } + + if (!isRecord(output as Json)) { + return output === undefined ? null : ""; + } + + if (typeof output.embedding_length === "number") { + return { + embedding_length: "", + type: "embedding", + }; + } + + return { + keys: Object.keys(output).sort(), + type: "object", + }; +} + +function summarizePayloadRow(row: CapturedLogRow): Json { + const metrics = + typeof row.metrics === "object" && + row.metrics !== null && + !Array.isArray(row.metrics) + ? (row.metrics as Record) + : {}; + + return { + has_input: row.input !== undefined && row.input !== null, + has_output: row.output !== undefined && row.output !== null, + input: summarizeInput(row.input), + metadata: pickMetadata( + row.metadata as Record | undefined, + ["model", "operation", "provider", "scenario"], + ), + metric_keys: Object.keys(metrics) + .filter((key) => key !== "start" && key !== "end") + .sort(), + output: summarizeOutput(row.output), + span_id: typeof row.span_id === "string" ? row.span_id : null, + } satisfies Json; +} + +function mergeRecordValues( + left: Record | undefined, + right: unknown, +): Record | undefined { + if (!right || typeof right !== "object" || Array.isArray(right)) { + return left; + } + + return { + ...(left || {}), + ...(right as Record), + }; +} + +function mergePayloadRows(rows: CapturedLogRow[]): CapturedLogRow[] { + const mergedBySpan = new Map(); + const spanOrder: string[] = []; + + for (const row of rows) { + const spanId = + typeof row.span_id === "string" + ? row.span_id + : `unknown-${spanOrder.length}`; + const existing = mergedBySpan.get(spanId); + + if (!existing) { + mergedBySpan.set(spanId, { + ...row, + }); + spanOrder.push(spanId); + continue; + } + + mergedBySpan.set(spanId, { + ...existing, + ...(row.input !== undefined && row.input !== null + ? { + input: row.input, + } + : {}), + ...(row.output !== undefined && row.output !== null + ? { + output: row.output, + } + : {}), + metadata: mergeRecordValues( + existing.metadata as Record | undefined, + row.metadata, + ), + metrics: mergeRecordValues( + existing.metrics as Record | undefined, + row.metrics, + ), + }); + } + + return spanOrder + .map((spanId) => mergedBySpan.get(spanId)) + .filter((row): row is CapturedLogRow => row !== undefined); +} + +function buildSpanSummary(events: CapturedLogEvent[]): Json { + const chatCompleteOperation = findLatestSpan( + events, + "mistral-chat-complete-operation", + ); + const chatStreamOperation = findLatestSpan( + events, + "mistral-chat-stream-operation", + ); + const fimCompleteOperation = findLatestSpan( + events, + "mistral-fim-complete-operation", + ); + const fimStreamOperation = findLatestSpan( + events, + "mistral-fim-stream-operation", + ); + const agentsCompleteOperation = findLatestSpan( + events, + "mistral-agents-complete-operation", + ); + const agentsStreamOperation = findLatestSpan( + events, + "mistral-agents-stream-operation", + ); + const embeddingsOperation = findLatestSpan( + events, + "mistral-embeddings-operation", + ); + + return normalizeForSnapshot( + [ + findLatestSpan(events, ROOT_NAME), + chatCompleteOperation, + findMistralSpan(events, chatCompleteOperation?.span.id, [ + "mistral.chat.complete", + ]), + chatStreamOperation, + findMistralSpan(events, chatStreamOperation?.span.id, [ + "mistral.chat.stream", + ]), + fimCompleteOperation, + findMistralSpan(events, fimCompleteOperation?.span.id, [ + "mistral.fim.complete", + ]), + fimStreamOperation, + findMistralSpan(events, fimStreamOperation?.span.id, [ + "mistral.fim.stream", + ]), + agentsCompleteOperation, + findMistralSpan(events, agentsCompleteOperation?.span.id, [ + "mistral.agents.complete", + ]), + agentsStreamOperation, + findMistralSpan(events, agentsStreamOperation?.span.id, [ + "mistral.agents.stream", + ]), + embeddingsOperation, + findMistralSpan(events, embeddingsOperation?.span.id, [ + "mistral.embeddings.create", + ]), + ].map((event) => + summarizeWrapperContract(event!, [ + "model", + "operation", + "provider", + "scenario", + ]), + ) as Json, + ); +} + +function buildPayloadSummary( + events: CapturedLogEvent[], + payloads: CapturedLogPayload[], +): Json { + const root = findLatestSpan(events, ROOT_NAME); + const payloadRows = payloadRowsForRootSpan(payloads, root?.span.id); + const mergedRows = mergePayloadRows(payloadRows); + return normalizeForSnapshot( + mergedRows.map((row) => summarizePayloadRow(row)), + ); +} + +export function defineMistralInstrumentationAssertions(options: { + name: string; + runScenario: RunMistralScenario; + snapshotName: string; + testFileUrl: string; + timeoutMs: number; +}): void { + const spanSnapshotPath = resolveFileSnapshotPath( + options.testFileUrl, + `${options.snapshotName}.span-events.json`, + ); + const payloadSnapshotPath = resolveFileSnapshotPath( + options.testFileUrl, + `${options.snapshotName}.log-payloads.json`, + ); + const testConfig = { + timeout: options.timeoutMs, + }; + + describe(options.name, () => { + let events: CapturedLogEvent[] = []; + let payloads: CapturedLogPayload[] = []; + + beforeAll(async () => { + await withScenarioHarness(async (harness) => { + await options.runScenario(harness); + events = harness.events(); + payloads = harness.payloads(); + }); + }, options.timeoutMs); + + test("captures the root trace for the scenario", testConfig, () => { + const root = findLatestSpan(events, ROOT_NAME); + + expect(root).toBeDefined(); + expect(root?.row.metadata).toMatchObject({ + scenario: SCENARIO_NAME, + }); + }); + + test("captures trace for chat.complete()", testConfig, () => { + const root = findLatestSpan(events, ROOT_NAME); + const operation = findLatestSpan( + events, + "mistral-chat-complete-operation", + ); + const span = findMistralSpan(events, operation?.span.id, [ + "mistral.chat.complete", + ]); + + expect(operation).toBeDefined(); + expect(span).toBeDefined(); + expect(operation?.span.parentIds).toEqual([root?.span.id ?? ""]); + expect(span?.span.type).toBe("llm"); + expect(span?.row.metadata).toMatchObject({ + model: CHAT_MODEL, + provider: "mistral", + }); + expect(span?.output).toBeDefined(); + }); + + test("captures trace for chat.stream()", testConfig, () => { + const root = findLatestSpan(events, ROOT_NAME); + const operation = findLatestSpan(events, "mistral-chat-stream-operation"); + const span = findMistralSpan(events, operation?.span.id, [ + "mistral.chat.stream", + ]); + + expect(operation).toBeDefined(); + expect(span).toBeDefined(); + expect(operation?.span.parentIds).toEqual([root?.span.id ?? ""]); + expect(span?.span.type).toBe("llm"); + expect(span?.row.metadata).toMatchObject({ + model: CHAT_MODEL, + provider: "mistral", + }); + expect(span?.metrics?.time_to_first_token).toEqual(expect.any(Number)); + expect(span?.output).toBeDefined(); + }); + + test("captures trace for fim.complete()", testConfig, () => { + const root = findLatestSpan(events, ROOT_NAME); + const operation = findLatestSpan( + events, + "mistral-fim-complete-operation", + ); + const span = findMistralSpan(events, operation?.span.id, [ + "mistral.fim.complete", + ]); + + expect(operation).toBeDefined(); + expect(span).toBeDefined(); + expect(operation?.span.parentIds).toEqual([root?.span.id ?? ""]); + expect(span?.span.type).toBe("llm"); + expect(span?.row.metadata).toMatchObject({ + model: FIM_MODEL, + provider: "mistral", + }); + expect(span?.input).toEqual(expect.any(String)); + expect(span?.metrics?.time_to_first_token).toEqual(expect.any(Number)); + expect(span?.output).toBeDefined(); + }); + + test("captures trace for fim.stream()", testConfig, () => { + const root = findLatestSpan(events, ROOT_NAME); + const operation = findLatestSpan(events, "mistral-fim-stream-operation"); + const span = findMistralSpan(events, operation?.span.id, [ + "mistral.fim.stream", + ]); + + expect(operation).toBeDefined(); + expect(span).toBeDefined(); + expect(operation?.span.parentIds).toEqual([root?.span.id ?? ""]); + expect(span?.span.type).toBe("llm"); + expect(span?.row.metadata).toMatchObject({ + model: FIM_MODEL, + provider: "mistral", + }); + expect(span?.input).toEqual(expect.any(String)); + expect(span?.metrics?.time_to_first_token).toEqual(expect.any(Number)); + expect(span?.output).toBeDefined(); + }); + + test("captures trace for agents.complete()", testConfig, () => { + const root = findLatestSpan(events, ROOT_NAME); + const operation = findLatestSpan( + events, + "mistral-agents-complete-operation", + ); + const span = findMistralSpan(events, operation?.span.id, [ + "mistral.agents.complete", + ]); + const metadata = span?.row.metadata as + | Record + | undefined; + const input = span?.input as Array<{ role?: string }> | undefined; + + expect(operation).toBeDefined(); + expect(span).toBeDefined(); + expect(operation?.span.parentIds).toEqual([root?.span.id ?? ""]); + expect(span?.span.type).toBe("llm"); + expect(metadata).toMatchObject({ + provider: "mistral", + agentId: expect.any(String), + }); + if (typeof metadata?.model === "string") { + expect(metadata.model).toBe(AGENT_MODEL); + } + expect(input).toEqual(expect.any(Array)); + expect(input?.[0]?.role).toBe("user"); + expect(span?.metrics?.time_to_first_token).toEqual(expect.any(Number)); + expect(span?.output).toBeDefined(); + }); + + test("captures trace for agents.stream()", testConfig, () => { + const root = findLatestSpan(events, ROOT_NAME); + const operation = findLatestSpan( + events, + "mistral-agents-stream-operation", + ); + const span = findMistralSpan(events, operation?.span.id, [ + "mistral.agents.stream", + ]); + const metadata = span?.row.metadata as + | Record + | undefined; + const input = span?.input as Array<{ role?: string }> | undefined; + + expect(operation).toBeDefined(); + expect(span).toBeDefined(); + expect(operation?.span.parentIds).toEqual([root?.span.id ?? ""]); + expect(span?.span.type).toBe("llm"); + expect(metadata).toMatchObject({ + provider: "mistral", + agentId: expect.any(String), + }); + if (typeof metadata?.model === "string") { + expect(metadata.model).toBe(AGENT_MODEL); + } + expect(input).toEqual(expect.any(Array)); + expect(input?.[0]?.role).toBe("user"); + expect(span?.metrics?.time_to_first_token).toEqual(expect.any(Number)); + expect(span?.output).toBeDefined(); + }); + + test("captures trace for embeddings.create()", testConfig, () => { + const root = findLatestSpan(events, ROOT_NAME); + const operation = findLatestSpan(events, "mistral-embeddings-operation"); + const span = findMistralSpan(events, operation?.span.id, [ + "mistral.embeddings.create", + ]); + const output = span?.output as { embedding_length?: number } | undefined; + + expect(operation).toBeDefined(); + expect(span).toBeDefined(); + expect(operation?.span.parentIds).toEqual([root?.span.id ?? ""]); + expect(span?.span.type).toBe("llm"); + expect(span?.row.metadata).toMatchObject({ + model: EMBEDDING_MODEL, + provider: "mistral", + }); + expect(output?.embedding_length).toEqual(expect.any(Number)); + expect(output?.embedding_length).toBeGreaterThan(0); + }); + + test("matches the shared span snapshot", testConfig, async () => { + await expect( + formatJsonFileSnapshot(buildSpanSummary(events)), + ).toMatchFileSnapshot(spanSnapshotPath); + }); + + test("matches the shared payload snapshot", testConfig, async () => { + await expect( + formatJsonFileSnapshot(buildPayloadSummary(events, payloads)), + ).toMatchFileSnapshot(payloadSnapshotPath); + }); + }); +} diff --git a/e2e/scenarios/mistral-instrumentation/constants.mjs b/e2e/scenarios/mistral-instrumentation/constants.mjs new file mode 100644 index 000000000..b4eb0ab83 --- /dev/null +++ b/e2e/scenarios/mistral-instrumentation/constants.mjs @@ -0,0 +1,15 @@ +const CHAT_MODEL = "mistral-small-latest"; +const EMBEDDING_MODEL = "mistral-embed"; +const FIM_MODEL = "codestral-latest"; +const AGENT_MODEL = CHAT_MODEL; +const ROOT_NAME = "mistral-root"; +const SCENARIO_NAME = "mistral-instrumentation"; + +export { + AGENT_MODEL, + CHAT_MODEL, + EMBEDDING_MODEL, + FIM_MODEL, + ROOT_NAME, + SCENARIO_NAME, +}; diff --git a/e2e/scenarios/mistral-instrumentation/package.json b/e2e/scenarios/mistral-instrumentation/package.json new file mode 100644 index 000000000..031cccf46 --- /dev/null +++ b/e2e/scenarios/mistral-instrumentation/package.json @@ -0,0 +1,20 @@ +{ + "name": "@braintrust/e2e-mistral-instrumentation", + "private": true, + "braintrustScenario": { + "canary": { + "dependencies": { + "mistral-sdk-v1": "@mistralai/mistralai@1", + "mistral-sdk-v2": "@mistralai/mistralai@2" + } + } + }, + "dependencies": { + "mistral-sdk-v1": "npm:@mistralai/mistralai@1.15.1", + "mistral-sdk-v1-10-0": "npm:@mistralai/mistralai@1.10.0", + "mistral-sdk-v1-14-1": "npm:@mistralai/mistralai@1.14.1", + "mistral-sdk-v1-15-1": "npm:@mistralai/mistralai@1.15.1", + "mistral-sdk-v1-3-4": "npm:@mistralai/mistralai@1.3.4", + "mistral-sdk-v2": "npm:@mistralai/mistralai@2.1.2" + } +} diff --git a/e2e/scenarios/mistral-instrumentation/pnpm-lock.yaml b/e2e/scenarios/mistral-instrumentation/pnpm-lock.yaml new file mode 100644 index 000000000..e94ea4b92 --- /dev/null +++ b/e2e/scenarios/mistral-instrumentation/pnpm-lock.yaml @@ -0,0 +1,122 @@ +lockfileVersion: '9.0' + +settings: + autoInstallPeers: true + excludeLinksFromLockfile: false + +importers: + + .: + dependencies: + mistral-sdk-v1: + specifier: npm:@mistralai/mistralai@1.15.1 + version: '@mistralai/mistralai@1.15.1' + mistral-sdk-v1-10-0: + specifier: npm:@mistralai/mistralai@1.10.0 + version: '@mistralai/mistralai@1.10.0' + mistral-sdk-v1-14-1: + specifier: npm:@mistralai/mistralai@1.14.1 + version: '@mistralai/mistralai@1.14.1' + mistral-sdk-v1-15-1: + specifier: npm:@mistralai/mistralai@1.15.1 + version: '@mistralai/mistralai@1.15.1' + mistral-sdk-v1-3-4: + specifier: npm:@mistralai/mistralai@1.3.4 + version: '@mistralai/mistralai@1.3.4(zod@4.3.6)' + mistral-sdk-v2: + specifier: npm:@mistralai/mistralai@2.1.2 + version: '@mistralai/mistralai@2.1.2' + +packages: + + '@mistralai/mistralai@1.10.0': + resolution: {integrity: sha512-tdIgWs4Le8vpvPiUEWne6tK0qbVc+jMenujnvTqOjogrJUsCSQhus0tHTU1avDDh5//Rq2dFgP9mWRAdIEoBqg==} + + '@mistralai/mistralai@1.14.1': + resolution: {integrity: sha512-IiLmmZFCCTReQgPAT33r7KQ1nYo5JPdvGkrkZqA8qQ2qB1GHgs5LoP5K2ICyrjnpw2n8oSxMM/VP+liiKcGNlQ==} + + '@mistralai/mistralai@1.15.1': + resolution: {integrity: sha512-fb995eiz3r0KsBGtRjFV+/iLbX+UpfalxpF+YitT3R6ukrPD4PN+FGwwmYcRFhNAzVzDUtTVxQYnjQWEnwV5nw==} + + '@mistralai/mistralai@1.3.4': + resolution: {integrity: sha512-db5UhCXqH0N05XbXMR/2bSiGKIFUzS6p0sI9Nl2XDmJuDZIm+WRGTlsq60ALwhvKpHcQKzN5L58HIneksRrn9g==} + peerDependencies: + zod: '>= 3' + + '@mistralai/mistralai@2.1.2': + resolution: {integrity: sha512-0NlXBNdTMmi4z0oSUWa8KseBHlli+/FkaUTaAyLYTJzkKh3mYM/D3DnJzRil8VD6QC7W3IteARjd3LnxAZGPUw==} + + ws@8.20.0: + resolution: {integrity: sha512-sAt8BhgNbzCtgGbt2OxmpuryO63ZoDk/sqaB/znQm94T4fCEsy/yV+7CdC1kJhOU9lboAEU7R3kquuycDoibVA==} + engines: {node: '>=10.0.0'} + peerDependencies: + bufferutil: ^4.0.1 + utf-8-validate: '>=5.0.2' + peerDependenciesMeta: + bufferutil: + optional: true + utf-8-validate: + optional: true + + zod-to-json-schema@3.25.2: + resolution: {integrity: sha512-O/PgfnpT1xKSDeQYSCfRI5Gy3hPf91mKVDuYLUHZJMiDFptvP41MSnWofm8dnCm0256ZNfZIM7DSzuSMAFnjHA==} + peerDependencies: + zod: ^3.25.28 || ^4 + + zod@3.25.76: + resolution: {integrity: sha512-gzUt/qt81nXsFGKIFcC3YnfEAx5NkunCfnDlvuBSSFS02bcXu4Lmea0AFIUwbLWxWPx3d9p8S5QoaujKcNQxcQ==} + + zod@4.3.6: + resolution: {integrity: sha512-rftlrkhHZOcjDwkGlnUtZZkvaPHCsDATp4pGpuOOMDaTdDDXF91wuVDJoWoPsKX/3YPQ5fHuF3STjcYyKr+Qhg==} + +snapshots: + + '@mistralai/mistralai@1.10.0': + dependencies: + zod: 3.25.76 + zod-to-json-schema: 3.25.2(zod@3.25.76) + + '@mistralai/mistralai@1.14.1': + dependencies: + ws: 8.20.0 + zod: 4.3.6 + zod-to-json-schema: 3.25.2(zod@4.3.6) + transitivePeerDependencies: + - bufferutil + - utf-8-validate + + '@mistralai/mistralai@1.15.1': + dependencies: + ws: 8.20.0 + zod: 4.3.6 + zod-to-json-schema: 3.25.2(zod@4.3.6) + transitivePeerDependencies: + - bufferutil + - utf-8-validate + + '@mistralai/mistralai@1.3.4(zod@4.3.6)': + dependencies: + zod: 4.3.6 + + '@mistralai/mistralai@2.1.2': + dependencies: + ws: 8.20.0 + zod: 4.3.6 + zod-to-json-schema: 3.25.2(zod@4.3.6) + transitivePeerDependencies: + - bufferutil + - utf-8-validate + + ws@8.20.0: {} + + zod-to-json-schema@3.25.2(zod@3.25.76): + dependencies: + zod: 3.25.76 + + zod-to-json-schema@3.25.2(zod@4.3.6): + dependencies: + zod: 4.3.6 + + zod@3.25.76: {} + + zod@4.3.6: {} diff --git a/e2e/scenarios/mistral-instrumentation/scenario.impl.mjs b/e2e/scenarios/mistral-instrumentation/scenario.impl.mjs new file mode 100644 index 000000000..e4d4bc30b --- /dev/null +++ b/e2e/scenarios/mistral-instrumentation/scenario.impl.mjs @@ -0,0 +1,446 @@ +import { wrapMistral } from "braintrust"; +import { + collectAsync, + runOperation, + runTracedScenario, +} from "../../helpers/provider-runtime.mjs"; +import { + AGENT_MODEL, + CHAT_MODEL, + EMBEDDING_MODEL, + FIM_MODEL, + ROOT_NAME, + SCENARIO_NAME, +} from "./constants.mjs"; + +export const MISTRAL_SCENARIO_TIMEOUT_MS = 240_000; +export const MISTRAL_SCENARIO_SPECS = [ + { + autoEntry: "scenario.mistral-v1-3-4.mjs", + dependencyName: "mistral-sdk-v1-3-4", + snapshotName: "mistral-v1-3-4", + supportsAutoHook: false, + wrapperEntry: "scenario.mistral-v1-3-4.ts", + }, + { + autoEntry: "scenario.mistral-v1-10-0.mjs", + dependencyName: "mistral-sdk-v1-10-0", + snapshotName: "mistral-v1-10-0", + supportsAutoHook: false, + wrapperEntry: "scenario.mistral-v1-10-0.ts", + }, + { + autoEntry: "scenario.mistral-v1-14-1.mjs", + dependencyName: "mistral-sdk-v1-14-1", + snapshotName: "mistral-v1-14-1", + supportsAutoHook: false, + wrapperEntry: "scenario.mistral-v1-14-1.ts", + }, + { + autoEntry: "scenario.mistral-v1-15-1.mjs", + dependencyName: "mistral-sdk-v1-15-1", + snapshotName: "mistral-v1-15-1", + supportsAutoHook: false, + wrapperEntry: "scenario.mistral-v1-15-1.ts", + }, + { + autoEntry: "scenario.mistral-v1.mjs", + dependencyName: "mistral-sdk-v1", + snapshotName: "mistral-v1", + supportsAutoHook: false, + wrapperEntry: "scenario.mistral-v1.ts", + }, + { + autoEntry: "scenario.mjs", + dependencyName: "mistral-sdk-v2", + snapshotName: "mistral-v2", + supportsAutoHook: true, + wrapperEntry: "scenario.ts", + }, +]; + +function nonEmptyString(value) { + return typeof value === "string" && value.trim().length > 0 + ? value.trim() + : null; +} + +function isRecord(value) { + return typeof value === "object" && value !== null; +} + +function getAgentId(agent) { + if (!isRecord(agent)) { + return null; + } + + if (nonEmptyString(agent.id)) { + return agent.id.trim(); + } + + if (nonEmptyString(agent.agentId)) { + return agent.agentId.trim(); + } + + if (nonEmptyString(agent.agent_id)) { + return agent.agent_id.trim(); + } + + return null; +} + +function getMistralApiBaseUrl(client) { + const envBaseUrl = + nonEmptyString(process.env.MISTRAL_API_URL) || + nonEmptyString(process.env.MISTRAL_BASE_URL); + if (envBaseUrl) { + return envBaseUrl.replace(/\/+$/g, ""); + } + + const options = + isRecord(client) && isRecord(client._options) ? client._options : undefined; + const optionBaseUrl = + (isRecord(options) && nonEmptyString(options.serverURL)) || + (isRecord(options) && nonEmptyString(options.serverUrl)) || + (isRecord(options) && nonEmptyString(options.baseURL)) || + (isRecord(options) && nonEmptyString(options.baseUrl)); + + return (optionBaseUrl || "https://api.mistral.ai").replace(/\/+$/g, ""); +} + +function getAgentCreatePayload() { + return { + model: AGENT_MODEL, + name: `braintrust-e2e-${Date.now().toString(36)}`, + instructions: "You are concise. Keep responses under five words.", + }; +} + +async function withRetry(callback, { attempts = 3, delayMs = 1_000 } = {}) { + let lastError; + for (let attempt = 1; attempt <= attempts; attempt++) { + try { + return await callback(); + } catch (error) { + lastError = error; + if (attempt === attempts) { + throw error; + } + await new Promise((resolve) => setTimeout(resolve, delayMs * attempt)); + } + } + + throw lastError; +} + +async function createAgentViaHttp(client, apiKey) { + const baseUrl = getMistralApiBaseUrl(client); + const response = await withRetry( + async () => + fetch(`${baseUrl}/v1/agents`, { + body: JSON.stringify(getAgentCreatePayload()), + headers: { + Authorization: `Bearer ${apiKey}`, + "Content-Type": "application/json", + }, + method: "POST", + }), + { attempts: 3, delayMs: 1_000 }, + ); + const responseBody = await response.text(); + if (!response.ok) { + throw new Error( + `Failed to create temporary Mistral agent (${response.status}): ${responseBody}`, + ); + } + + const parsed = JSON.parse(responseBody); + const createdAgentId = getAgentId(parsed); + if (!createdAgentId) { + throw new Error("Mistral agent creation response did not include an id."); + } + + return { + agentId: createdAgentId, + cleanup: async () => { + try { + await fetch(`${baseUrl}/v1/agents/${createdAgentId}`, { + headers: { + Authorization: `Bearer ${apiKey}`, + }, + method: "DELETE", + }); + } catch { + // Ignore cleanup failures for temporary e2e agents. + } + }, + }; +} + +async function createAgentViaSdk(client, apiKey) { + const beta = isRecord(client) && isRecord(client.beta) ? client.beta : null; + const agentManager = beta && isRecord(beta.agents) ? beta.agents : null; + const createAgent = agentManager?.create; + if (typeof createAgent !== "function") { + return null; + } + + const created = await withRetry( + async () => createAgent.call(agentManager, getAgentCreatePayload()), + { attempts: 3, delayMs: 1_000 }, + ); + const createdAgentId = getAgentId(created); + if (!createdAgentId) { + throw new Error("beta.agents.create() did not return an agent id."); + } + + const deleteAgent = agentManager?.delete; + if (typeof deleteAgent === "function") { + return { + agentId: createdAgentId, + cleanup: async () => { + try { + await deleteAgent.call(agentManager, { agentId: createdAgentId }); + } catch { + // Ignore cleanup failures for temporary e2e agents. + } + }, + }; + } + + const baseUrl = getMistralApiBaseUrl(client); + return { + agentId: createdAgentId, + cleanup: async () => { + try { + await fetch(`${baseUrl}/v1/agents/${createdAgentId}`, { + headers: { + Authorization: `Bearer ${apiKey}`, + }, + method: "DELETE", + }); + } catch { + // Ignore cleanup failures for temporary e2e agents. + } + }, + }; +} + +async function resolveAgentRuntime(client) { + const configuredAgentId = nonEmptyString(process.env.MISTRAL_AGENT_ID); + if (configuredAgentId) { + return { + agentId: configuredAgentId, + cleanup: async () => {}, + }; + } + + const apiKey = nonEmptyString(process.env.MISTRAL_API_KEY); + if (!apiKey) { + throw new Error("MISTRAL_API_KEY is required for Mistral e2e scenarios."); + } + + try { + const sdkRuntime = await createAgentViaSdk(client, apiKey); + if (sdkRuntime) { + return sdkRuntime; + } + } catch { + // Fall back to direct API provisioning when SDK beta management is unavailable. + } + + return await createAgentViaHttp(client, apiKey); +} + +async function runMistralInstrumentationScenario( + Mistral, + { decorateClient } = {}, +) { + const baseClient = new Mistral({ + apiKey: process.env.MISTRAL_API_KEY, + }); + const client = decorateClient ? decorateClient(baseClient) : baseClient; + const { agentId, cleanup } = await resolveAgentRuntime(baseClient); + + try { + await runTracedScenario({ + callback: async () => { + await runOperation( + "mistral-chat-complete-operation", + "chat-complete", + async () => { + await withRetry( + async () => + client.chat.complete({ + model: CHAT_MODEL, + messages: [ + { + role: "system", + content: + "You are concise. Keep responses under five words.", + }, + { + role: "user", + content: "Reply with exactly: observability", + }, + ], + maxTokens: 16, + temperature: 0, + }), + { attempts: 3, delayMs: 1_000 }, + ); + }, + ); + + await runOperation( + "mistral-chat-stream-operation", + "chat-stream", + async () => { + await withRetry( + async () => { + const stream = await client.chat.stream({ + model: CHAT_MODEL, + messages: [ + { + role: "user", + content: "Reply with exactly: streamed output", + }, + ], + maxTokens: 24, + stream: true, + temperature: 0, + }); + await collectAsync(stream); + }, + { attempts: 3, delayMs: 1_000 }, + ); + }, + ); + + await runOperation( + "mistral-fim-complete-operation", + "fim-complete", + async () => { + await withRetry( + async () => + client.fim.complete({ + model: FIM_MODEL, + prompt: "function add(a, b) {", + suffix: "}", + maxTokens: 24, + temperature: 0, + }), + { attempts: 3, delayMs: 1_000 }, + ); + }, + ); + + await runOperation( + "mistral-fim-stream-operation", + "fim-stream", + async () => { + await withRetry( + async () => { + const stream = await client.fim.stream({ + model: FIM_MODEL, + prompt: "const project = ", + suffix: ";", + maxTokens: 16, + stream: true, + temperature: 0, + }); + await collectAsync(stream); + }, + { attempts: 3, delayMs: 1_000 }, + ); + }, + ); + + await runOperation( + "mistral-agents-complete-operation", + "agents-complete", + async () => { + await withRetry( + async () => + client.agents.complete({ + agentId, + messages: [ + { + role: "user", + content: "Reply with exactly: agent complete", + }, + ], + responseFormat: { + type: "text", + }, + maxTokens: 12, + temperature: 0, + }), + { attempts: 3, delayMs: 1_000 }, + ); + }, + ); + + await runOperation( + "mistral-agents-stream-operation", + "agents-stream", + async () => { + await withRetry( + async () => { + const stream = await client.agents.stream({ + agentId, + messages: [ + { + role: "user", + content: "Reply with exactly: agent stream", + }, + ], + responseFormat: { + type: "text", + }, + maxTokens: 12, + stream: true, + temperature: 0, + }); + await collectAsync(stream); + }, + { attempts: 3, delayMs: 1_000 }, + ); + }, + ); + + await runOperation( + "mistral-embeddings-operation", + "embeddings-create", + async () => { + await withRetry( + async () => + client.embeddings.create({ + model: EMBEDDING_MODEL, + inputs: "braintrust mistral instrumentation", + }), + { attempts: 3, delayMs: 1_000 }, + ); + }, + ); + }, + metadata: { + scenario: SCENARIO_NAME, + }, + projectNameBase: "e2e-mistral-instrumentation", + rootName: ROOT_NAME, + }); + } finally { + await cleanup(); + } +} + +export async function runWrappedMistralInstrumentation(Mistral) { + await runMistralInstrumentationScenario(Mistral, { + decorateClient: wrapMistral, + }); +} + +export async function runAutoMistralInstrumentation(Mistral) { + await runMistralInstrumentationScenario(Mistral); +} diff --git a/e2e/scenarios/mistral-instrumentation/scenario.mistral-v1-10-0.mjs b/e2e/scenarios/mistral-instrumentation/scenario.mistral-v1-10-0.mjs new file mode 100644 index 000000000..0e5445b21 --- /dev/null +++ b/e2e/scenarios/mistral-instrumentation/scenario.mistral-v1-10-0.mjs @@ -0,0 +1,5 @@ +import { Mistral } from "mistral-sdk-v1-10-0"; +import { runMain } from "../../helpers/provider-runtime.mjs"; +import { runAutoMistralInstrumentation } from "./scenario.impl.mjs"; + +runMain(async () => runAutoMistralInstrumentation(Mistral)); diff --git a/e2e/scenarios/mistral-instrumentation/scenario.mistral-v1-10-0.ts b/e2e/scenarios/mistral-instrumentation/scenario.mistral-v1-10-0.ts new file mode 100644 index 000000000..db6605274 --- /dev/null +++ b/e2e/scenarios/mistral-instrumentation/scenario.mistral-v1-10-0.ts @@ -0,0 +1,5 @@ +import { Mistral } from "mistral-sdk-v1-10-0"; +import { runMain } from "../../helpers/scenario-runtime"; +import { runWrappedMistralInstrumentation } from "./scenario.impl.mjs"; + +runMain(async () => runWrappedMistralInstrumentation(Mistral)); diff --git a/e2e/scenarios/mistral-instrumentation/scenario.mistral-v1-14-1.mjs b/e2e/scenarios/mistral-instrumentation/scenario.mistral-v1-14-1.mjs new file mode 100644 index 000000000..b7ae746f9 --- /dev/null +++ b/e2e/scenarios/mistral-instrumentation/scenario.mistral-v1-14-1.mjs @@ -0,0 +1,5 @@ +import { Mistral } from "mistral-sdk-v1-14-1"; +import { runMain } from "../../helpers/provider-runtime.mjs"; +import { runAutoMistralInstrumentation } from "./scenario.impl.mjs"; + +runMain(async () => runAutoMistralInstrumentation(Mistral)); diff --git a/e2e/scenarios/mistral-instrumentation/scenario.mistral-v1-14-1.ts b/e2e/scenarios/mistral-instrumentation/scenario.mistral-v1-14-1.ts new file mode 100644 index 000000000..5b882da41 --- /dev/null +++ b/e2e/scenarios/mistral-instrumentation/scenario.mistral-v1-14-1.ts @@ -0,0 +1,5 @@ +import { Mistral } from "mistral-sdk-v1-14-1"; +import { runMain } from "../../helpers/scenario-runtime"; +import { runWrappedMistralInstrumentation } from "./scenario.impl.mjs"; + +runMain(async () => runWrappedMistralInstrumentation(Mistral)); diff --git a/e2e/scenarios/mistral-instrumentation/scenario.mistral-v1-15-1.mjs b/e2e/scenarios/mistral-instrumentation/scenario.mistral-v1-15-1.mjs new file mode 100644 index 000000000..57938f2b6 --- /dev/null +++ b/e2e/scenarios/mistral-instrumentation/scenario.mistral-v1-15-1.mjs @@ -0,0 +1,5 @@ +import { Mistral } from "mistral-sdk-v1-15-1"; +import { runMain } from "../../helpers/provider-runtime.mjs"; +import { runAutoMistralInstrumentation } from "./scenario.impl.mjs"; + +runMain(async () => runAutoMistralInstrumentation(Mistral)); diff --git a/e2e/scenarios/mistral-instrumentation/scenario.mistral-v1-15-1.ts b/e2e/scenarios/mistral-instrumentation/scenario.mistral-v1-15-1.ts new file mode 100644 index 000000000..ca52f313f --- /dev/null +++ b/e2e/scenarios/mistral-instrumentation/scenario.mistral-v1-15-1.ts @@ -0,0 +1,5 @@ +import { Mistral } from "mistral-sdk-v1-15-1"; +import { runMain } from "../../helpers/scenario-runtime"; +import { runWrappedMistralInstrumentation } from "./scenario.impl.mjs"; + +runMain(async () => runWrappedMistralInstrumentation(Mistral)); diff --git a/e2e/scenarios/mistral-instrumentation/scenario.mistral-v1-3-4.mjs b/e2e/scenarios/mistral-instrumentation/scenario.mistral-v1-3-4.mjs new file mode 100644 index 000000000..4f772e22d --- /dev/null +++ b/e2e/scenarios/mistral-instrumentation/scenario.mistral-v1-3-4.mjs @@ -0,0 +1,5 @@ +import { Mistral } from "mistral-sdk-v1-3-4"; +import { runMain } from "../../helpers/provider-runtime.mjs"; +import { runAutoMistralInstrumentation } from "./scenario.impl.mjs"; + +runMain(async () => runAutoMistralInstrumentation(Mistral)); diff --git a/e2e/scenarios/mistral-instrumentation/scenario.mistral-v1-3-4.ts b/e2e/scenarios/mistral-instrumentation/scenario.mistral-v1-3-4.ts new file mode 100644 index 000000000..1db41c17f --- /dev/null +++ b/e2e/scenarios/mistral-instrumentation/scenario.mistral-v1-3-4.ts @@ -0,0 +1,5 @@ +import { Mistral } from "mistral-sdk-v1-3-4"; +import { runMain } from "../../helpers/scenario-runtime"; +import { runWrappedMistralInstrumentation } from "./scenario.impl.mjs"; + +runMain(async () => runWrappedMistralInstrumentation(Mistral)); diff --git a/e2e/scenarios/mistral-instrumentation/scenario.mistral-v1.mjs b/e2e/scenarios/mistral-instrumentation/scenario.mistral-v1.mjs new file mode 100644 index 000000000..53db8ddb8 --- /dev/null +++ b/e2e/scenarios/mistral-instrumentation/scenario.mistral-v1.mjs @@ -0,0 +1,5 @@ +import { Mistral } from "mistral-sdk-v1"; +import { runMain } from "../../helpers/provider-runtime.mjs"; +import { runAutoMistralInstrumentation } from "./scenario.impl.mjs"; + +runMain(async () => runAutoMistralInstrumentation(Mistral)); diff --git a/e2e/scenarios/mistral-instrumentation/scenario.mistral-v1.ts b/e2e/scenarios/mistral-instrumentation/scenario.mistral-v1.ts new file mode 100644 index 000000000..994487e1e --- /dev/null +++ b/e2e/scenarios/mistral-instrumentation/scenario.mistral-v1.ts @@ -0,0 +1,5 @@ +import { Mistral } from "mistral-sdk-v1"; +import { runMain } from "../../helpers/scenario-runtime"; +import { runWrappedMistralInstrumentation } from "./scenario.impl.mjs"; + +runMain(async () => runWrappedMistralInstrumentation(Mistral)); diff --git a/e2e/scenarios/mistral-instrumentation/scenario.mjs b/e2e/scenarios/mistral-instrumentation/scenario.mjs new file mode 100644 index 000000000..d4eb85870 --- /dev/null +++ b/e2e/scenarios/mistral-instrumentation/scenario.mjs @@ -0,0 +1,5 @@ +import { Mistral } from "mistral-sdk-v2"; +import { runMain } from "../../helpers/provider-runtime.mjs"; +import { runAutoMistralInstrumentation } from "./scenario.impl.mjs"; + +runMain(async () => runAutoMistralInstrumentation(Mistral)); diff --git a/e2e/scenarios/mistral-instrumentation/scenario.test.ts b/e2e/scenarios/mistral-instrumentation/scenario.test.ts new file mode 100644 index 000000000..54868e61b --- /dev/null +++ b/e2e/scenarios/mistral-instrumentation/scenario.test.ts @@ -0,0 +1,62 @@ +import { describe } from "vitest"; +import { + prepareScenarioDir, + readInstalledPackageVersion, + resolveScenarioDir, +} from "../../helpers/scenario-harness"; +import { defineMistralInstrumentationAssertions } from "./assertions"; +import { + MISTRAL_SCENARIO_SPECS, + MISTRAL_SCENARIO_TIMEOUT_MS, +} from "./scenario.impl.mjs"; + +const scenarioDir = await prepareScenarioDir({ + scenarioDir: resolveScenarioDir(import.meta.url), +}); + +const mistralScenarios = await Promise.all( + MISTRAL_SCENARIO_SPECS.map(async (scenario) => ({ + ...scenario, + version: await readInstalledPackageVersion( + scenarioDir, + scenario.dependencyName, + ), + })), +); + +for (const scenario of mistralScenarios) { + describe(`mistral sdk ${scenario.version}`, () => { + defineMistralInstrumentationAssertions({ + name: "wrapped instrumentation", + runScenario: async ({ runScenarioDir }) => { + await runScenarioDir({ + entry: scenario.wrapperEntry, + runContext: { variantKey: scenario.snapshotName }, + scenarioDir, + timeoutMs: MISTRAL_SCENARIO_TIMEOUT_MS, + }); + }, + snapshotName: scenario.snapshotName, + testFileUrl: import.meta.url, + timeoutMs: MISTRAL_SCENARIO_TIMEOUT_MS, + }); + + if (scenario.supportsAutoHook) { + defineMistralInstrumentationAssertions({ + name: "auto-hook instrumentation", + runScenario: async ({ runNodeScenarioDir }) => { + await runNodeScenarioDir({ + entry: scenario.autoEntry, + nodeArgs: ["--import", "braintrust/hook.mjs"], + runContext: { variantKey: scenario.snapshotName }, + scenarioDir, + timeoutMs: MISTRAL_SCENARIO_TIMEOUT_MS, + }); + }, + snapshotName: scenario.snapshotName, + testFileUrl: import.meta.url, + timeoutMs: MISTRAL_SCENARIO_TIMEOUT_MS, + }); + } + }); +} diff --git a/e2e/scenarios/mistral-instrumentation/scenario.ts b/e2e/scenarios/mistral-instrumentation/scenario.ts new file mode 100644 index 000000000..62f148ed9 --- /dev/null +++ b/e2e/scenarios/mistral-instrumentation/scenario.ts @@ -0,0 +1,5 @@ +import { Mistral } from "mistral-sdk-v2"; +import { runMain } from "../../helpers/scenario-runtime"; +import { runWrappedMistralInstrumentation } from "./scenario.impl.mjs"; + +runMain(async () => runWrappedMistralInstrumentation(Mistral)); diff --git a/e2e/scripts/run-canary-tests-docker.mjs b/e2e/scripts/run-canary-tests-docker.mjs index cc96cc0fd..130980697 100644 --- a/e2e/scripts/run-canary-tests-docker.mjs +++ b/e2e/scripts/run-canary-tests-docker.mjs @@ -21,6 +21,7 @@ const ALLOWED_ENV_KEYS = [ "OPENAI_API_KEY", "OPENAI_BASE_URL", "OPENROUTER_API_KEY", + "MISTRAL_API_KEY", ]; function getAllowedEnv() { diff --git a/js/src/auto-instrumentations/bundler/plugin.ts b/js/src/auto-instrumentations/bundler/plugin.ts index 29a25eec0..dd2f1efe2 100644 --- a/js/src/auto-instrumentations/bundler/plugin.ts +++ b/js/src/auto-instrumentations/bundler/plugin.ts @@ -26,6 +26,7 @@ import { aiSDKConfigs } from "../configs/ai-sdk"; import { claudeAgentSDKConfigs } from "../configs/claude-agent-sdk"; import { googleGenAIConfigs } from "../configs/google-genai"; import { openRouterConfigs } from "../configs/openrouter"; +import { mistralConfigs } from "../configs/mistral"; export interface BundlerPluginOptions { /** @@ -73,6 +74,7 @@ export const unplugin = createUnplugin((options = {}) => { ...claudeAgentSDKConfigs, ...googleGenAIConfigs, ...openRouterConfigs, + ...mistralConfigs, ...(options.instrumentations || []), ]; diff --git a/js/src/auto-instrumentations/bundler/webpack-loader.ts b/js/src/auto-instrumentations/bundler/webpack-loader.ts index 566d07597..2eb9cc369 100644 --- a/js/src/auto-instrumentations/bundler/webpack-loader.ts +++ b/js/src/auto-instrumentations/bundler/webpack-loader.ts @@ -33,6 +33,7 @@ import { aiSDKConfigs } from "../configs/ai-sdk"; import { claudeAgentSDKConfigs } from "../configs/claude-agent-sdk"; import { googleGenAIConfigs } from "../configs/google-genai"; import { openRouterConfigs } from "../configs/openrouter"; +import { mistralConfigs } from "../configs/mistral"; import { type BundlerPluginOptions } from "./plugin"; /** @@ -66,6 +67,7 @@ function getMatcher(options: BundlerPluginOptions): InstrumentationMatcher { ...claudeAgentSDKConfigs, ...googleGenAIConfigs, ...openRouterConfigs, + ...mistralConfigs, ...(options.instrumentations ?? []), ]; const dcModule = options.browser ? "dc-browser" : undefined; diff --git a/js/src/auto-instrumentations/configs/claude-agent-sdk.test.ts b/js/src/auto-instrumentations/configs/claude-agent-sdk.test.ts deleted file mode 100644 index 5c71ea78b..000000000 --- a/js/src/auto-instrumentations/configs/claude-agent-sdk.test.ts +++ /dev/null @@ -1,36 +0,0 @@ -import { describe, expect, it } from "vitest"; -import { claudeAgentSDKConfigs } from "./claude-agent-sdk"; -import { claudeAgentSDKChannels } from "../../instrumentation/plugins/claude-agent-sdk-channels"; - -describe("claudeAgentSDKConfigs", () => { - it("registers sync query instrumentation for 0.1.x", () => { - expect(claudeAgentSDKConfigs).toContainEqual({ - channelName: claudeAgentSDKChannels.query.channelName, - module: { - name: "@anthropic-ai/claude-agent-sdk", - versionRange: ">=0.1.0 <0.2.0", - filePath: "sdk.mjs", - }, - functionQuery: { - functionName: "query", - kind: "Sync", - }, - }); - }); - - it("registers export-alias query instrumentation for 0.2.x", () => { - expect(claudeAgentSDKConfigs).toContainEqual({ - channelName: claudeAgentSDKChannels.query.channelName, - module: { - name: "@anthropic-ai/claude-agent-sdk", - versionRange: ">=0.2.0", - filePath: "sdk.mjs", - }, - functionQuery: { - functionName: "query", - kind: "Sync", - isExportAlias: true, - }, - }); - }); -}); diff --git a/js/src/auto-instrumentations/configs/mistral.ts b/js/src/auto-instrumentations/configs/mistral.ts new file mode 100644 index 000000000..896f54db1 --- /dev/null +++ b/js/src/auto-instrumentations/configs/mistral.ts @@ -0,0 +1,200 @@ +import type { InstrumentationConfig } from "@apm-js-collab/code-transformer"; +import { mistralChannels } from "../../instrumentation/plugins/mistral-channels"; + +export const mistralConfigs: InstrumentationConfig[] = [ + { + channelName: mistralChannels.chatComplete.channelName, + module: { + name: "@mistralai/mistralai", + versionRange: ">=1.0.0 <2.0.0", + filePath: "sdk/chat.js", + }, + functionQuery: { + className: "Chat", + methodName: "complete", + kind: "Async", + }, + }, + + { + channelName: mistralChannels.chatComplete.channelName, + module: { + name: "@mistralai/mistralai", + versionRange: ">=2.0.0 <3.0.0", + filePath: "esm/sdk/chat.js", + }, + functionQuery: { + className: "Chat", + methodName: "complete", + kind: "Async", + }, + }, + + { + channelName: mistralChannels.chatStream.channelName, + module: { + name: "@mistralai/mistralai", + versionRange: ">=1.0.0 <2.0.0", + filePath: "sdk/chat.js", + }, + functionQuery: { + className: "Chat", + methodName: "stream", + kind: "Async", + }, + }, + + { + channelName: mistralChannels.chatStream.channelName, + module: { + name: "@mistralai/mistralai", + versionRange: ">=2.0.0 <3.0.0", + filePath: "esm/sdk/chat.js", + }, + functionQuery: { + className: "Chat", + methodName: "stream", + kind: "Async", + }, + }, + + { + channelName: mistralChannels.embeddingsCreate.channelName, + module: { + name: "@mistralai/mistralai", + versionRange: ">=1.0.0 <2.0.0", + filePath: "sdk/embeddings.js", + }, + functionQuery: { + className: "Embeddings", + methodName: "create", + kind: "Async", + }, + }, + + { + channelName: mistralChannels.embeddingsCreate.channelName, + module: { + name: "@mistralai/mistralai", + versionRange: ">=2.0.0 <3.0.0", + filePath: "esm/sdk/embeddings.js", + }, + functionQuery: { + className: "Embeddings", + methodName: "create", + kind: "Async", + }, + }, + + { + channelName: mistralChannels.fimComplete.channelName, + module: { + name: "@mistralai/mistralai", + versionRange: ">=1.0.0 <2.0.0", + filePath: "sdk/fim.js", + }, + functionQuery: { + className: "Fim", + methodName: "complete", + kind: "Async", + }, + }, + + { + channelName: mistralChannels.fimComplete.channelName, + module: { + name: "@mistralai/mistralai", + versionRange: ">=2.0.0 <3.0.0", + filePath: "esm/sdk/fim.js", + }, + functionQuery: { + className: "Fim", + methodName: "complete", + kind: "Async", + }, + }, + + { + channelName: mistralChannels.fimStream.channelName, + module: { + name: "@mistralai/mistralai", + versionRange: ">=1.0.0 <2.0.0", + filePath: "sdk/fim.js", + }, + functionQuery: { + className: "Fim", + methodName: "stream", + kind: "Async", + }, + }, + + { + channelName: mistralChannels.fimStream.channelName, + module: { + name: "@mistralai/mistralai", + versionRange: ">=2.0.0 <3.0.0", + filePath: "esm/sdk/fim.js", + }, + functionQuery: { + className: "Fim", + methodName: "stream", + kind: "Async", + }, + }, + + { + channelName: mistralChannels.agentsComplete.channelName, + module: { + name: "@mistralai/mistralai", + versionRange: ">=1.0.0 <2.0.0", + filePath: "sdk/agents.js", + }, + functionQuery: { + className: "Agents", + methodName: "complete", + kind: "Async", + }, + }, + + { + channelName: mistralChannels.agentsComplete.channelName, + module: { + name: "@mistralai/mistralai", + versionRange: ">=2.0.0 <3.0.0", + filePath: "esm/sdk/agents.js", + }, + functionQuery: { + className: "Agents", + methodName: "complete", + kind: "Async", + }, + }, + + { + channelName: mistralChannels.agentsStream.channelName, + module: { + name: "@mistralai/mistralai", + versionRange: ">=1.0.0 <2.0.0", + filePath: "sdk/agents.js", + }, + functionQuery: { + className: "Agents", + methodName: "stream", + kind: "Async", + }, + }, + + { + channelName: mistralChannels.agentsStream.channelName, + module: { + name: "@mistralai/mistralai", + versionRange: ">=2.0.0 <3.0.0", + filePath: "esm/sdk/agents.js", + }, + functionQuery: { + className: "Agents", + methodName: "stream", + kind: "Async", + }, + }, +]; diff --git a/js/src/auto-instrumentations/configs/openrouter.test.ts b/js/src/auto-instrumentations/configs/openrouter.test.ts deleted file mode 100644 index aef665be9..000000000 --- a/js/src/auto-instrumentations/configs/openrouter.test.ts +++ /dev/null @@ -1,21 +0,0 @@ -import { describe, expect, it } from "vitest"; -import { openRouterConfigs } from "./openrouter"; -import { openRouterChannels } from "../../instrumentation/plugins/openrouter-channels"; - -describe("openRouterConfigs", () => { - it("registers auto-instrumentation for OpenRouter.callModel", () => { - expect(openRouterConfigs).toContainEqual({ - channelName: openRouterChannels.callModel.channelName, - module: { - name: "@openrouter/sdk", - versionRange: ">=0.9.11 <1.0.0", - filePath: "esm/sdk/sdk.js", - }, - functionQuery: { - className: "OpenRouter", - methodName: "callModel", - kind: "Sync", - }, - }); - }); -}); diff --git a/js/src/auto-instrumentations/hook.mts b/js/src/auto-instrumentations/hook.mts index d7f656baa..db202a0a6 100644 --- a/js/src/auto-instrumentations/hook.mts +++ b/js/src/auto-instrumentations/hook.mts @@ -20,6 +20,7 @@ import { aiSDKConfigs } from "./configs/ai-sdk.js"; import { claudeAgentSDKConfigs } from "./configs/claude-agent-sdk.js"; import { googleGenAIConfigs } from "./configs/google-genai.js"; import { openRouterConfigs } from "./configs/openrouter.js"; +import { mistralConfigs } from "./configs/mistral.js"; import { ModulePatch } from "./loader/cjs-patch.js"; import { patchTracingChannel } from "./patch-tracing-channel.js"; @@ -38,6 +39,7 @@ const allConfigs = [ ...claudeAgentSDKConfigs, ...googleGenAIConfigs, ...openRouterConfigs, + ...mistralConfigs, ]; // 1. Register ESM loader for ESM modules diff --git a/js/src/auto-instrumentations/index.ts b/js/src/auto-instrumentations/index.ts index 2be27701d..87040081e 100644 --- a/js/src/auto-instrumentations/index.ts +++ b/js/src/auto-instrumentations/index.ts @@ -34,6 +34,7 @@ export { aiSDKConfigs } from "./configs/ai-sdk"; export { claudeAgentSDKConfigs } from "./configs/claude-agent-sdk"; export { googleGenAIConfigs } from "./configs/google-genai"; export { openRouterConfigs } from "./configs/openrouter"; +export { mistralConfigs } from "./configs/mistral"; // Re-export orchestrion configuration types // Note: ModuleMetadata and FunctionQuery are properties of InstrumentationConfig, diff --git a/js/src/exports.ts b/js/src/exports.ts index c01c1e3cb..7f705238e 100644 --- a/js/src/exports.ts +++ b/js/src/exports.ts @@ -174,6 +174,7 @@ export { wrapMastraAgent } from "./wrappers/mastra"; export { wrapClaudeAgentSDK } from "./wrappers/claude-agent-sdk/claude-agent-sdk"; export { wrapGoogleGenAI } from "./wrappers/google-genai"; export { wrapOpenRouter } from "./wrappers/openrouter"; +export { wrapMistral } from "./wrappers/mistral"; export { wrapVitest } from "./wrappers/vitest"; export { initNodeTestSuite } from "./wrappers/node-test"; diff --git a/js/src/instrumentation/braintrust-plugin.test.ts b/js/src/instrumentation/braintrust-plugin.test.ts index 2e499a4c3..d8f6a5904 100644 --- a/js/src/instrumentation/braintrust-plugin.test.ts +++ b/js/src/instrumentation/braintrust-plugin.test.ts @@ -6,6 +6,7 @@ import { AISDKPlugin } from "./plugins/ai-sdk-plugin"; import { ClaudeAgentSDKPlugin } from "./plugins/claude-agent-sdk-plugin"; import { GoogleGenAIPlugin } from "./plugins/google-genai-plugin"; import { OpenRouterPlugin } from "./plugins/openrouter-plugin"; +import { MistralPlugin } from "./plugins/mistral-plugin"; function createPluginClassMock() { return vi.fn(function MockPlugin(this: { @@ -48,6 +49,10 @@ vi.mock("./plugins/openrouter-plugin", () => ({ OpenRouterPlugin: createPluginClassMock(), })); +vi.mock("./plugins/mistral-plugin", () => ({ + MistralPlugin: createPluginClassMock(), +})); + describe("BraintrustPlugin", () => { beforeEach(() => { vi.clearAllMocks(); @@ -109,6 +114,15 @@ describe("BraintrustPlugin", () => { expect(mockInstance.enable).toHaveBeenCalledTimes(1); }); + it("should create and enable Mistral plugin by default", () => { + const plugin = new BraintrustPlugin(); + plugin.enable(); + + expect(MistralPlugin).toHaveBeenCalledTimes(1); + const mockInstance = vi.mocked(MistralPlugin).mock.results[0].value; + expect(mockInstance.enable).toHaveBeenCalledTimes(1); + }); + it("should create all plugins when enabled with no config", () => { const plugin = new BraintrustPlugin(); plugin.enable(); @@ -119,6 +133,7 @@ describe("BraintrustPlugin", () => { expect(ClaudeAgentSDKPlugin).toHaveBeenCalledTimes(1); expect(GoogleGenAIPlugin).toHaveBeenCalledTimes(1); expect(OpenRouterPlugin).toHaveBeenCalledTimes(1); + expect(MistralPlugin).toHaveBeenCalledTimes(1); }); it("should create all plugins when enabled with empty config", () => { @@ -131,6 +146,7 @@ describe("BraintrustPlugin", () => { expect(ClaudeAgentSDKPlugin).toHaveBeenCalledTimes(1); expect(GoogleGenAIPlugin).toHaveBeenCalledTimes(1); expect(OpenRouterPlugin).toHaveBeenCalledTimes(1); + expect(MistralPlugin).toHaveBeenCalledTimes(1); }); it("should create all plugins when enabled with empty integrations config", () => { @@ -143,6 +159,7 @@ describe("BraintrustPlugin", () => { expect(ClaudeAgentSDKPlugin).toHaveBeenCalledTimes(1); expect(GoogleGenAIPlugin).toHaveBeenCalledTimes(1); expect(OpenRouterPlugin).toHaveBeenCalledTimes(1); + expect(MistralPlugin).toHaveBeenCalledTimes(1); }); }); @@ -190,6 +207,7 @@ describe("BraintrustPlugin", () => { expect(ClaudeAgentSDKPlugin).toHaveBeenCalledTimes(1); expect(GoogleGenAIPlugin).toHaveBeenCalledTimes(1); expect(OpenRouterPlugin).toHaveBeenCalledTimes(1); + expect(MistralPlugin).toHaveBeenCalledTimes(1); }); it("should not create Claude Agent SDK plugin when claudeAgentSDK: false", () => { @@ -205,6 +223,7 @@ describe("BraintrustPlugin", () => { expect(AISDKPlugin).toHaveBeenCalledTimes(1); expect(GoogleGenAIPlugin).toHaveBeenCalledTimes(1); expect(OpenRouterPlugin).toHaveBeenCalledTimes(1); + expect(MistralPlugin).toHaveBeenCalledTimes(1); }); it("should not create Google GenAI plugin when googleGenAI: false", () => { @@ -220,6 +239,7 @@ describe("BraintrustPlugin", () => { expect(AISDKPlugin).toHaveBeenCalledTimes(1); expect(ClaudeAgentSDKPlugin).toHaveBeenCalledTimes(1); expect(OpenRouterPlugin).toHaveBeenCalledTimes(1); + expect(MistralPlugin).toHaveBeenCalledTimes(1); }); it("should not create OpenRouter plugin when openrouter: false", () => { @@ -234,6 +254,22 @@ describe("BraintrustPlugin", () => { expect(AISDKPlugin).toHaveBeenCalledTimes(1); expect(ClaudeAgentSDKPlugin).toHaveBeenCalledTimes(1); expect(GoogleGenAIPlugin).toHaveBeenCalledTimes(1); + expect(MistralPlugin).toHaveBeenCalledTimes(1); + }); + + it("should not create Mistral plugin when mistral: false", () => { + const plugin = new BraintrustPlugin({ + integrations: { mistral: false }, + }); + plugin.enable(); + + expect(MistralPlugin).not.toHaveBeenCalled(); + expect(OpenAIPlugin).toHaveBeenCalledTimes(1); + expect(AnthropicPlugin).toHaveBeenCalledTimes(1); + expect(AISDKPlugin).toHaveBeenCalledTimes(1); + expect(ClaudeAgentSDKPlugin).toHaveBeenCalledTimes(1); + expect(GoogleGenAIPlugin).toHaveBeenCalledTimes(1); + expect(OpenRouterPlugin).toHaveBeenCalledTimes(1); }); it("should not create any plugins when all are disabled", () => { @@ -245,6 +281,7 @@ describe("BraintrustPlugin", () => { claudeAgentSDK: false, googleGenAI: false, openrouter: false, + mistral: false, }, }); plugin.enable(); @@ -255,6 +292,7 @@ describe("BraintrustPlugin", () => { expect(ClaudeAgentSDKPlugin).not.toHaveBeenCalled(); expect(GoogleGenAIPlugin).not.toHaveBeenCalled(); expect(OpenRouterPlugin).not.toHaveBeenCalled(); + expect(MistralPlugin).not.toHaveBeenCalled(); }); it("should allow selective enabling of plugins", () => { @@ -266,6 +304,7 @@ describe("BraintrustPlugin", () => { claudeAgentSDK: true, googleGenAI: false, openrouter: true, + mistral: false, }, }); plugin.enable(); @@ -276,6 +315,7 @@ describe("BraintrustPlugin", () => { expect(AnthropicPlugin).not.toHaveBeenCalled(); expect(AISDKPlugin).not.toHaveBeenCalled(); expect(GoogleGenAIPlugin).not.toHaveBeenCalled(); + expect(MistralPlugin).not.toHaveBeenCalled(); }); }); @@ -292,6 +332,8 @@ describe("BraintrustPlugin", () => { expect(AnthropicPlugin).toHaveBeenCalledTimes(1); expect(ClaudeAgentSDKPlugin).toHaveBeenCalledTimes(1); expect(GoogleGenAIPlugin).toHaveBeenCalledTimes(1); + expect(OpenRouterPlugin).toHaveBeenCalledTimes(1); + expect(MistralPlugin).toHaveBeenCalledTimes(1); }); it("should not create Google GenAI plugin when google: false (legacy)", () => { @@ -306,6 +348,8 @@ describe("BraintrustPlugin", () => { expect(AnthropicPlugin).toHaveBeenCalledTimes(1); expect(AISDKPlugin).toHaveBeenCalledTimes(1); expect(ClaudeAgentSDKPlugin).toHaveBeenCalledTimes(1); + expect(OpenRouterPlugin).toHaveBeenCalledTimes(1); + expect(MistralPlugin).toHaveBeenCalledTimes(1); }); it("should not create AI SDK plugin when both aisdk and vercel are false", () => { @@ -376,6 +420,7 @@ describe("BraintrustPlugin", () => { const googleGenAIMock = vi.mocked(GoogleGenAIPlugin).mock.results[0].value; const openRouterMock = vi.mocked(OpenRouterPlugin).mock.results[0].value; + const mistralMock = vi.mocked(MistralPlugin).mock.results[0].value; expect(openaiMock.enable).toHaveBeenCalledTimes(1); expect(anthropicMock.enable).toHaveBeenCalledTimes(1); @@ -383,6 +428,7 @@ describe("BraintrustPlugin", () => { expect(claudeAgentSDKMock.enable).toHaveBeenCalledTimes(1); expect(googleGenAIMock.enable).toHaveBeenCalledTimes(1); expect(openRouterMock.enable).toHaveBeenCalledTimes(1); + expect(mistralMock.enable).toHaveBeenCalledTimes(1); }); it("should disable and nullify all sub-plugins when disabled", () => { @@ -397,6 +443,7 @@ describe("BraintrustPlugin", () => { const googleGenAIMock = vi.mocked(GoogleGenAIPlugin).mock.results[0].value; const openRouterMock = vi.mocked(OpenRouterPlugin).mock.results[0].value; + const mistralMock = vi.mocked(MistralPlugin).mock.results[0].value; plugin.disable(); @@ -406,6 +453,7 @@ describe("BraintrustPlugin", () => { expect(claudeAgentSDKMock.disable).toHaveBeenCalledTimes(1); expect(googleGenAIMock.disable).toHaveBeenCalledTimes(1); expect(openRouterMock.disable).toHaveBeenCalledTimes(1); + expect(mistralMock.disable).toHaveBeenCalledTimes(1); }); it("should be idempotent on multiple enable calls", () => { @@ -445,6 +493,7 @@ describe("BraintrustPlugin", () => { expect(ClaudeAgentSDKPlugin).not.toHaveBeenCalled(); expect(GoogleGenAIPlugin).not.toHaveBeenCalled(); expect(OpenRouterPlugin).not.toHaveBeenCalled(); + expect(MistralPlugin).not.toHaveBeenCalled(); }); it("should allow re-enabling after disable", () => { @@ -462,6 +511,7 @@ describe("BraintrustPlugin", () => { expect(ClaudeAgentSDKPlugin).toHaveBeenCalledTimes(1); expect(GoogleGenAIPlugin).toHaveBeenCalledTimes(1); expect(OpenRouterPlugin).toHaveBeenCalledTimes(1); + expect(MistralPlugin).toHaveBeenCalledTimes(1); }); it("should only disable plugins that were enabled", () => { @@ -473,6 +523,7 @@ describe("BraintrustPlugin", () => { claudeAgentSDK: false, googleGenAI: true, openrouter: true, + mistral: false, }, }); plugin.enable(); @@ -489,6 +540,7 @@ describe("BraintrustPlugin", () => { expect(aiSDKMock.disable).toHaveBeenCalledTimes(1); expect(googleGenAIMock.disable).toHaveBeenCalledTimes(1); expect(openRouterMock.disable).toHaveBeenCalledTimes(1); + expect(MistralPlugin).not.toHaveBeenCalled(); }); }); }); diff --git a/js/src/instrumentation/braintrust-plugin.ts b/js/src/instrumentation/braintrust-plugin.ts index 89d2b141e..726be1da2 100644 --- a/js/src/instrumentation/braintrust-plugin.ts +++ b/js/src/instrumentation/braintrust-plugin.ts @@ -5,6 +5,7 @@ import { AISDKPlugin } from "./plugins/ai-sdk-plugin"; import { ClaudeAgentSDKPlugin } from "./plugins/claude-agent-sdk-plugin"; import { GoogleGenAIPlugin } from "./plugins/google-genai-plugin"; import { OpenRouterPlugin } from "./plugins/openrouter-plugin"; +import { MistralPlugin } from "./plugins/mistral-plugin"; export interface BraintrustPluginConfig { integrations?: { @@ -16,6 +17,7 @@ export interface BraintrustPluginConfig { googleGenAI?: boolean; claudeAgentSDK?: boolean; openrouter?: boolean; + mistral?: boolean; }; } @@ -28,6 +30,7 @@ export interface BraintrustPluginConfig { * - Claude Agent SDK (agent interactions) * - Vercel AI SDK (generateText, streamText, etc.) * - Google GenAI SDK + * - Mistral SDK * * The plugin is automatically enabled when the Braintrust library is loaded. * Individual integrations can be disabled via configuration. @@ -40,6 +43,7 @@ export class BraintrustPlugin extends BasePlugin { private claudeAgentSDKPlugin: ClaudeAgentSDKPlugin | null = null; private googleGenAIPlugin: GoogleGenAIPlugin | null = null; private openRouterPlugin: OpenRouterPlugin | null = null; + private mistralPlugin: MistralPlugin | null = null; constructor(config: BraintrustPluginConfig = {}) { super(); @@ -85,6 +89,11 @@ export class BraintrustPlugin extends BasePlugin { this.openRouterPlugin = new OpenRouterPlugin(); this.openRouterPlugin.enable(); } + + if (integrations.mistral !== false) { + this.mistralPlugin = new MistralPlugin(); + this.mistralPlugin.enable(); + } } protected onDisable(): void { @@ -117,6 +126,11 @@ export class BraintrustPlugin extends BasePlugin { this.openRouterPlugin.disable(); this.openRouterPlugin = null; } + + if (this.mistralPlugin) { + this.mistralPlugin.disable(); + this.mistralPlugin = null; + } } } diff --git a/js/src/instrumentation/plugins/mistral-channels.ts b/js/src/instrumentation/plugins/mistral-channels.ts new file mode 100644 index 000000000..6328d3872 --- /dev/null +++ b/js/src/instrumentation/plugins/mistral-channels.ts @@ -0,0 +1,78 @@ +import { channel, defineChannels } from "../core/channel-definitions"; +import type { + MistralAgentsCompletionEvent, + MistralAgentsCompletionResponse, + MistralAgentsCreateParams, + MistralAgentsResult, + MistralChatCompletionEvent, + MistralChatCompletionResponse, + MistralChatCreateParams, + MistralChatResult, + MistralEmbeddingCreateParams, + MistralEmbeddingResponse, + MistralFimCompletionEvent, + MistralFimCompletionResponse, + MistralFimCreateParams, + MistralFimResult, +} from "../../vendor-sdk-types/mistral"; + +export const mistralChannels = defineChannels("@mistralai/mistralai", { + chatComplete: channel< + [MistralChatCreateParams], + MistralChatCompletionResponse + >({ + channelName: "chat.complete", + kind: "async", + }), + + chatStream: channel< + [MistralChatCreateParams], + MistralChatResult, + Record, + MistralChatCompletionEvent + >({ + channelName: "chat.stream", + kind: "async", + }), + + embeddingsCreate: channel< + [MistralEmbeddingCreateParams], + MistralEmbeddingResponse + >({ + channelName: "embeddings.create", + kind: "async", + }), + + fimComplete: channel<[MistralFimCreateParams], MistralFimCompletionResponse>({ + channelName: "fim.complete", + kind: "async", + }), + + fimStream: channel< + [MistralFimCreateParams], + MistralFimResult, + Record, + MistralFimCompletionEvent + >({ + channelName: "fim.stream", + kind: "async", + }), + + agentsComplete: channel< + [MistralAgentsCreateParams], + MistralAgentsCompletionResponse + >({ + channelName: "agents.complete", + kind: "async", + }), + + agentsStream: channel< + [MistralAgentsCreateParams], + MistralAgentsResult, + Record, + MistralAgentsCompletionEvent + >({ + channelName: "agents.stream", + kind: "async", + }), +}); diff --git a/js/src/instrumentation/plugins/mistral-plugin.test.ts b/js/src/instrumentation/plugins/mistral-plugin.test.ts new file mode 100644 index 000000000..48aed1d6c --- /dev/null +++ b/js/src/instrumentation/plugins/mistral-plugin.test.ts @@ -0,0 +1,169 @@ +import { describe, expect, it } from "vitest"; +import { + aggregateMistralStreamChunks, + parseMistralMetricsFromUsage, +} from "./mistral-plugin"; + +describe("parseMistralMetricsFromUsage", () => { + it("returns empty metrics for missing usage", () => { + expect(parseMistralMetricsFromUsage(undefined)).toEqual({}); + expect(parseMistralMetricsFromUsage(null)).toEqual({}); + }); + + it("normalizes common token counters", () => { + expect( + parseMistralMetricsFromUsage({ + promptTokens: 10, + completion_tokens: 6, + totalTokens: 16, + promptAudioSeconds: 3, + }), + ).toEqual({ + prompt_tokens: 10, + completion_tokens: 6, + tokens: 16, + prompt_audio_seconds: 3, + }); + }); + + it("normalizes token detail counters", () => { + expect( + parseMistralMetricsFromUsage({ + inputTokensDetails: { + cachedTokens: 7, + }, + output_tokens_details: { + reasoning_tokens: 2, + }, + }), + ).toEqual({ + prompt_cached_tokens: 7, + completion_reasoning_tokens: 2, + }); + }); +}); + +describe("aggregateMistralStreamChunks", () => { + it("aggregates stream text and usage into a single output row", () => { + const aggregated = aggregateMistralStreamChunks([ + { + data: { + id: "cmpl_1", + model: "mistral-small-latest", + object: "chat.completion.chunk", + created: 1, + choices: [ + { + delta: { + role: "assistant", + content: "Hello", + }, + }, + ], + }, + }, + { + data: { + id: "cmpl_1", + model: "mistral-small-latest", + object: "chat.completion.chunk", + created: 1, + usage: { + prompt_tokens: 12, + completion_tokens: 3, + total_tokens: 15, + }, + choices: [ + { + delta: { + content: " world", + }, + finish_reason: "stop", + }, + ], + }, + }, + ]); + + expect(aggregated.metrics).toMatchObject({ + prompt_tokens: 12, + completion_tokens: 3, + tokens: 15, + }); + + expect(aggregated.output?.[0]).toMatchObject({ + index: 0, + message: { + role: "assistant", + content: "Hello world", + }, + finishReason: "stop", + }); + + expect(aggregated.metadata).toMatchObject({ + id: "cmpl_1", + model: "mistral-small-latest", + object: "chat.completion.chunk", + created: 1, + }); + }); + + it("merges tool call argument deltas", () => { + const aggregated = aggregateMistralStreamChunks([ + { + data: { + choices: [ + { + delta: { + toolCalls: [ + { + id: "tool_1", + function: { + name: "lookup_weather", + arguments: '{"city":"Vie', + }, + }, + ], + }, + }, + ], + }, + }, + { + data: { + choices: [ + { + delta: { + tool_calls: [ + { + id: "tool_1", + function: { + arguments: 'nna"}', + }, + }, + ], + }, + finishReason: "tool_calls", + }, + ], + }, + }, + ]); + + expect(aggregated.output?.[0]).toMatchObject({ + message: { + content: null, + toolCalls: [ + { + id: "tool_1", + function: { + name: "lookup_weather", + arguments: '{"city":"Vienna"}', + }, + }, + ], + }, + finishReason: "tool_calls", + }); + }); +}); diff --git a/js/src/instrumentation/plugins/mistral-plugin.ts b/js/src/instrumentation/plugins/mistral-plugin.ts new file mode 100644 index 000000000..e9e977b20 --- /dev/null +++ b/js/src/instrumentation/plugins/mistral-plugin.ts @@ -0,0 +1,499 @@ +import { BasePlugin } from "../core"; +import { + traceAsyncChannel, + traceStreamingChannel, + unsubscribeAll, +} from "../core/channel-tracing"; +import { SpanTypeAttribute, isObject } from "../../../util/index"; +import { processInputAttachments } from "../../wrappers/attachment-utils"; +import { getCurrentUnixTimestamp } from "../../util"; +import { mistralChannels } from "./mistral-channels"; +import type { + MistralChatCompletionChunk, + MistralChatCompletionChunkChoice, + MistralChatCompletionEvent, + MistralChatCompletionResponse, + MistralToolCallDelta, +} from "../../vendor-sdk-types/mistral"; + +export class MistralPlugin extends BasePlugin { + protected onEnable(): void { + this.subscribeToMistralChannels(); + } + + protected onDisable(): void { + this.unsubscribers = unsubscribeAll(this.unsubscribers); + } + + private subscribeToMistralChannels(): void { + this.unsubscribers.push( + traceStreamingChannel(mistralChannels.chatComplete, { + name: "mistral.chat.complete", + type: SpanTypeAttribute.LLM, + extractInput: extractMessagesInputWithMetadata, + extractOutput: (result) => { + return result?.choices; + }, + extractMetadata: (result) => extractMistralResponseMetadata(result), + extractMetrics: (result, startTime) => + extractMistralMetrics(result?.usage, startTime), + }), + ); + + this.unsubscribers.push( + traceStreamingChannel(mistralChannels.chatStream, { + name: "mistral.chat.stream", + type: SpanTypeAttribute.LLM, + extractInput: extractMessagesInputWithMetadata, + extractOutput: extractMistralStreamOutput, + extractMetadata: (result) => extractMistralResponseMetadata(result), + extractMetrics: (result, startTime) => + extractMistralStreamingMetrics(result, startTime), + aggregateChunks: aggregateMistralStreamChunks, + }), + ); + + this.unsubscribers.push( + traceAsyncChannel(mistralChannels.embeddingsCreate, { + name: "mistral.embeddings.create", + type: SpanTypeAttribute.LLM, + extractInput: extractEmbeddingInputWithMetadata, + extractOutput: (result) => { + const embedding = result?.data?.[0]?.embedding; + return Array.isArray(embedding) + ? { embedding_length: embedding.length } + : undefined; + }, + extractMetadata: (result) => extractMistralResponseMetadata(result), + extractMetrics: (result) => parseMistralMetricsFromUsage(result?.usage), + }), + ); + + this.unsubscribers.push( + traceStreamingChannel(mistralChannels.fimComplete, { + name: "mistral.fim.complete", + type: SpanTypeAttribute.LLM, + extractInput: extractPromptInputWithMetadata, + extractOutput: (result) => { + return result?.choices; + }, + extractMetadata: (result) => extractMistralResponseMetadata(result), + extractMetrics: (result, startTime) => + extractMistralMetrics(result?.usage, startTime), + }), + ); + + this.unsubscribers.push( + traceStreamingChannel(mistralChannels.fimStream, { + name: "mistral.fim.stream", + type: SpanTypeAttribute.LLM, + extractInput: extractPromptInputWithMetadata, + extractOutput: extractMistralStreamOutput, + extractMetadata: (result) => extractMistralResponseMetadata(result), + extractMetrics: (result, startTime) => + extractMistralStreamingMetrics(result, startTime), + aggregateChunks: aggregateMistralStreamChunks, + }), + ); + + this.unsubscribers.push( + traceStreamingChannel(mistralChannels.agentsComplete, { + name: "mistral.agents.complete", + type: SpanTypeAttribute.LLM, + extractInput: extractMessagesInputWithMetadata, + extractOutput: (result) => { + return result?.choices; + }, + extractMetadata: (result) => extractMistralResponseMetadata(result), + extractMetrics: (result, startTime) => + extractMistralMetrics(result?.usage, startTime), + }), + ); + + this.unsubscribers.push( + traceStreamingChannel(mistralChannels.agentsStream, { + name: "mistral.agents.stream", + type: SpanTypeAttribute.LLM, + extractInput: extractMessagesInputWithMetadata, + extractOutput: extractMistralStreamOutput, + extractMetadata: (result) => extractMistralResponseMetadata(result), + extractMetrics: (result, startTime) => + extractMistralStreamingMetrics(result, startTime), + aggregateChunks: aggregateMistralStreamChunks, + }), + ); + } +} + +const TOKEN_NAME_MAP: Record = { + promptTokens: "prompt_tokens", + inputTokens: "prompt_tokens", + completionTokens: "completion_tokens", + outputTokens: "completion_tokens", + totalTokens: "tokens", + prompt_tokens: "prompt_tokens", + input_tokens: "prompt_tokens", + completion_tokens: "completion_tokens", + output_tokens: "completion_tokens", + total_tokens: "tokens", + promptAudioSeconds: "prompt_audio_seconds", + prompt_audio_seconds: "prompt_audio_seconds", +}; + +const TOKEN_DETAIL_PREFIX_MAP: Record = { + promptTokensDetails: "prompt", + inputTokensDetails: "prompt", + completionTokensDetails: "completion", + outputTokensDetails: "completion", + prompt_tokens_details: "prompt", + input_tokens_details: "prompt", + completion_tokens_details: "completion", + output_tokens_details: "completion", +}; + +function camelToSnake(value: string): string { + return value.replace(/[A-Z]/g, (match) => `_${match.toLowerCase()}`); +} + +function normalizeArgs(args: unknown[] | unknown): unknown[] { + if (Array.isArray(args)) { + return args; + } + + if (isArrayLike(args)) { + return Array.from(args); + } + + return [args]; +} + +function isArrayLike(value: unknown): value is ArrayLike { + return ( + isObject(value) && + "length" in value && + typeof value.length === "number" && + Number.isInteger(value.length) && + value.length >= 0 + ); +} + +function getMistralRequestArg( + args: unknown[] | unknown, +): Record | undefined { + const firstObjectArg = normalizeArgs(args).find((arg) => isObject(arg)); + return isObject(firstObjectArg) ? firstObjectArg : undefined; +} + +function addMistralProviderMetadata( + metadata: Record, +): Record { + return { + ...metadata, + provider: "mistral", + }; +} + +function extractMessagesInputWithMetadata(args: unknown[] | unknown): { + input: unknown; + metadata: Record; +} { + const params = getMistralRequestArg(args); + const { messages, ...metadata } = params || {}; + + return { + input: processInputAttachments(messages), + metadata: addMistralProviderMetadata(metadata), + }; +} + +function extractEmbeddingInputWithMetadata(args: unknown[] | unknown): { + input: unknown; + metadata: Record; +} { + const params = getMistralRequestArg(args); + const { inputs, ...metadata } = params || {}; + + return { + input: inputs, + metadata: addMistralProviderMetadata(metadata), + }; +} + +function extractPromptInputWithMetadata(args: unknown[] | unknown): { + input: unknown; + metadata: Record; +} { + const params = getMistralRequestArg(args); + const { prompt, ...metadata } = params || {}; + + return { + input: prompt, + metadata: addMistralProviderMetadata(metadata), + }; +} + +function extractMistralResponseMetadata( + result: unknown, +): Record | undefined { + if (!isObject(result)) { + return undefined; + } + + const { + choices: _choices, + usage: _usage, + data: _data, + ...metadata + } = result as Record; + + return Object.keys(metadata).length > 0 ? metadata : undefined; +} + +function extractMistralMetrics( + usage: unknown, + startTime?: number, +): Record { + const metrics = parseMistralMetricsFromUsage(usage); + if (startTime) { + metrics.time_to_first_token = getCurrentUnixTimestamp() - startTime; + } + return metrics; +} + +function extractMistralStreamOutput(result: unknown): unknown { + return isObject(result) ? result.choices : undefined; +} + +function extractMistralStreamingMetrics( + result: unknown, + startTime?: number, +): Record { + const metrics = isObject(result) + ? parseMistralMetricsFromUsage(result.usage) + : {}; + if (startTime) { + metrics.time_to_first_token = getCurrentUnixTimestamp() - startTime; + } + return metrics; +} + +function extractDeltaText(content: unknown): string | undefined { + if (typeof content === "string") { + return content; + } + + if (!Array.isArray(content)) { + return undefined; + } + + const textParts = content + .map((part) => { + if (!isObject(part) || part.type !== "text") { + return ""; + } + + return typeof part.text === "string" ? part.text : ""; + }) + .filter((part) => part.length > 0); + + return textParts.length > 0 ? textParts.join("") : undefined; +} + +function getDeltaToolCalls( + delta: Record, +): MistralToolCallDelta[] { + const toolCalls = + (Array.isArray(delta.toolCalls) && delta.toolCalls) || + (Array.isArray(delta.tool_calls) && delta.tool_calls) || + []; + + return toolCalls.filter((toolCall) => isObject(toolCall)); +} + +function mergeToolCallDeltas( + toolCalls: MistralToolCallDelta[] | undefined, + deltas: MistralToolCallDelta[], +): MistralToolCallDelta[] | undefined { + if (deltas.length === 0) { + return toolCalls; + } + + let merged = toolCalls ? [...toolCalls] : []; + + for (const delta of deltas) { + const current = merged[merged.length - 1]; + const sameTool = + current && + typeof current.id === "string" && + typeof delta.id === "string" && + current.id === delta.id; + + if (!sameTool) { + merged.push({ + ...delta, + function: { + ...delta.function, + arguments: + typeof delta.function?.arguments === "string" + ? delta.function.arguments + : "", + }, + }); + continue; + } + + const currentArguments = + typeof current.function?.arguments === "string" + ? current.function.arguments + : ""; + const deltaArguments = + typeof delta.function?.arguments === "string" + ? delta.function.arguments + : ""; + + merged[merged.length - 1] = { + ...current, + ...delta, + function: { + ...(current.function || {}), + ...(delta.function || {}), + arguments: `${currentArguments}${deltaArguments}`, + }, + }; + } + + return merged.length > 0 ? merged : undefined; +} + +function getChoiceFinishReason( + choice: MistralChatCompletionChunkChoice, +): string | null | undefined { + if (typeof choice.finishReason === "string" || choice.finishReason === null) { + return choice.finishReason; + } + + if ( + typeof choice.finish_reason === "string" || + choice.finish_reason === null + ) { + return choice.finish_reason; + } + + return undefined; +} + +export function parseMistralMetricsFromUsage( + usage: unknown, +): Record { + if (!isObject(usage)) { + return {}; + } + + const metrics: Record = {}; + + for (const [name, value] of Object.entries(usage)) { + if (typeof value === "number") { + metrics[TOKEN_NAME_MAP[name] || camelToSnake(name)] = value; + continue; + } + + if (!isObject(value)) { + continue; + } + + const prefix = TOKEN_DETAIL_PREFIX_MAP[name]; + if (!prefix) { + continue; + } + + for (const [nestedName, nestedValue] of Object.entries(value)) { + if (typeof nestedValue !== "number") { + continue; + } + + metrics[`${prefix}_${camelToSnake(nestedName)}`] = nestedValue; + } + } + + return metrics; +} + +export function aggregateMistralStreamChunks( + chunks: MistralChatCompletionEvent[], +): { + output: MistralChatCompletionResponse["choices"]; + metrics: Record; + metadata?: Record; +} { + let role: string | undefined; + let content: string | undefined; + let toolCalls: MistralToolCallDelta[] | undefined; + let finishReason: string | null | undefined; + let metrics: Record = {}; + let metadata: Record | undefined; + + for (const event of chunks) { + const chunk = isObject(event?.data) + ? (event.data as MistralChatCompletionChunk) + : undefined; + if (!chunk) { + continue; + } + + if (isObject(chunk.usage)) { + metrics = { + ...metrics, + ...parseMistralMetricsFromUsage(chunk.usage), + }; + } + + const chunkMetadata = extractMistralResponseMetadata(chunk); + if (chunkMetadata) { + metadata = { ...(metadata || {}), ...chunkMetadata }; + } + + for (const rawChoice of chunk.choices || []) { + if (!isObject(rawChoice)) { + continue; + } + const choice = rawChoice as MistralChatCompletionChunkChoice; + + const delta = isObject(choice.delta) + ? (choice.delta as Record) + : undefined; + if (delta) { + if (!role && typeof delta.role === "string") { + role = delta.role; + } + + const deltaText = extractDeltaText(delta.content); + if (deltaText) { + content = (content || "") + deltaText; + } + + toolCalls = mergeToolCallDeltas(toolCalls, getDeltaToolCalls(delta)); + } + + const choiceFinishReason = getChoiceFinishReason(choice); + if (choiceFinishReason !== undefined) { + finishReason = choiceFinishReason; + } + } + } + + return { + output: [ + { + index: 0, + message: { + ...(role ? { role } : {}), + content: content ?? null, + ...(toolCalls ? { toolCalls } : {}), + }, + ...(finishReason !== undefined ? { finishReason } : {}), + }, + ], + metrics, + ...(metadata ? { metadata } : {}), + }; +} diff --git a/js/src/instrumentation/registry.test.ts b/js/src/instrumentation/registry.test.ts index c3b62db7e..fc2c36bd3 100644 --- a/js/src/instrumentation/registry.test.ts +++ b/js/src/instrumentation/registry.test.ts @@ -119,6 +119,7 @@ describe("configureInstrumentation API", () => { openai: false, anthropic: true, openrouter: false, + mistral: false, }, }); }); diff --git a/js/src/instrumentation/registry.ts b/js/src/instrumentation/registry.ts index 79825abeb..e3521bdc3 100644 --- a/js/src/instrumentation/registry.ts +++ b/js/src/instrumentation/registry.ts @@ -21,6 +21,7 @@ export interface InstrumentationConfig { google?: boolean; claudeAgentSDK?: boolean; openrouter?: boolean; + mistral?: boolean; }; } @@ -107,6 +108,7 @@ class PluginRegistry { google: true, claudeAgentSDK: true, openrouter: true, + mistral: true, }; } diff --git a/js/src/vendor-sdk-types/mistral.ts b/js/src/vendor-sdk-types/mistral.ts new file mode 100644 index 000000000..90fd390fd --- /dev/null +++ b/js/src/vendor-sdk-types/mistral.ts @@ -0,0 +1,171 @@ +export type MistralToolCallDelta = { + id?: string; + type?: string; + function?: { + name?: string; + arguments?: string; + }; + [key: string]: unknown; +}; + +export type MistralChatMessageDelta = { + role?: string; + content?: string | null; + toolCalls?: MistralToolCallDelta[] | null; + tool_calls?: MistralToolCallDelta[] | null; + [key: string]: unknown; +}; + +export type MistralChatCompletionChoice = { + index?: number; + message?: { + role?: string; + content?: string | null; + toolCalls?: unknown; + tool_calls?: unknown; + }; + finishReason?: string | null; + finish_reason?: string | null; + [key: string]: unknown; +}; + +export type MistralChatCompletionChunkChoice = { + index?: number; + delta?: MistralChatMessageDelta; + finishReason?: string | null; + finish_reason?: string | null; + [key: string]: unknown; +}; + +export type MistralChatCompletionChunk = { + id?: string; + object?: string; + created?: number; + model?: string; + usage?: unknown; + choices?: MistralChatCompletionChunkChoice[]; + [key: string]: unknown; +}; + +export type MistralChatCompletionEvent = { + data?: MistralChatCompletionChunk; + [key: string]: unknown; +}; + +export type MistralChatCompletionResponse = { + id?: string; + object?: string; + model?: string; + created?: number; + usage?: unknown; + choices?: MistralChatCompletionChoice[]; + [key: string]: unknown; +}; + +export type MistralEmbeddingCreateParams = { + inputs?: unknown; + [key: string]: unknown; +}; + +export type MistralFimCreateParams = { + prompt?: unknown; + suffix?: unknown; + stream?: boolean; + [key: string]: unknown; +}; + +export type MistralAgentsCreateParams = { + messages?: unknown; + agentId?: string; + agent_id?: string; + stream?: boolean; + [key: string]: unknown; +}; + +export type MistralEmbeddingResponse = { + id?: string; + object?: string; + model?: string; + usage?: unknown; + data?: Array<{ + embedding?: number[] | string; + [key: string]: unknown; + }>; + [key: string]: unknown; +}; + +export type MistralChatCreateParams = { + messages?: unknown; + stream?: boolean; + [key: string]: unknown; +}; + +export type MistralChatStreamingResult = + AsyncIterable; +export type MistralFimStreamingResult = + AsyncIterable; +export type MistralAgentsStreamingResult = + AsyncIterable; + +export type MistralFimCompletionResponse = MistralChatCompletionResponse; +export type MistralAgentsCompletionResponse = MistralChatCompletionResponse; +export type MistralFimCompletionEvent = MistralChatCompletionEvent; +export type MistralAgentsCompletionEvent = MistralChatCompletionEvent; + +export type MistralChatResult = + | MistralChatCompletionResponse + | MistralChatStreamingResult; +export type MistralFimResult = + | MistralFimCompletionResponse + | MistralFimStreamingResult; +export type MistralAgentsResult = + | MistralAgentsCompletionResponse + | MistralAgentsStreamingResult; + +export type MistralChat = { + complete: ( + request: MistralChatCreateParams, + options?: unknown, + ) => Promise; + stream: ( + request: MistralChatCreateParams, + options?: unknown, + ) => Promise; +}; + +export type MistralEmbeddings = { + create: ( + request: MistralEmbeddingCreateParams, + options?: unknown, + ) => Promise; +}; + +export type MistralFim = { + complete: ( + request: MistralFimCreateParams, + options?: unknown, + ) => Promise; + stream: ( + request: MistralFimCreateParams, + options?: unknown, + ) => Promise; +}; + +export type MistralAgents = { + complete: ( + request: MistralAgentsCreateParams, + options?: unknown, + ) => Promise; + stream: ( + request: MistralAgentsCreateParams, + options?: unknown, + ) => Promise; +}; + +export type MistralClient = { + chat?: MistralChat; + fim?: MistralFim; + agents?: MistralAgents; + embeddings?: MistralEmbeddings; + [key: string]: unknown; +}; diff --git a/js/src/wrappers/mistral.ts b/js/src/wrappers/mistral.ts new file mode 100644 index 000000000..b2ce06aff --- /dev/null +++ b/js/src/wrappers/mistral.ts @@ -0,0 +1,245 @@ +import { mistralChannels } from "../instrumentation/plugins/mistral-channels"; +import type { + MistralAgents, + MistralAgentsCompletionResponse, + MistralAgentsCreateParams, + MistralAgentsStreamingResult, + MistralChat, + MistralChatCompletionResponse, + MistralChatCreateParams, + MistralChatStreamingResult, + MistralClient, + MistralEmbeddingCreateParams, + MistralEmbeddingResponse, + MistralEmbeddings, + MistralFim, + MistralFimCompletionResponse, + MistralFimCreateParams, + MistralFimStreamingResult, +} from "../vendor-sdk-types/mistral"; + +/** + * Wrap a Mistral client (created with `new Mistral(...)`) with Braintrust tracing. + */ +export function wrapMistral(mistral: T): T { + if (isSupportedMistralClient(mistral)) { + return mistralProxy(mistral) as T; + } + + // eslint-disable-next-line no-restricted-properties -- preserving intentional console usage. + console.warn("Unsupported Mistral library. Not wrapping."); + return mistral; +} + +function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null; +} + +function hasFunction(value: unknown, methodName: string): boolean { + return ( + isRecord(value) && + methodName in value && + typeof value[methodName] === "function" + ); +} + +function isSupportedMistralClient(value: unknown): value is MistralClient { + if (!isRecord(value)) { + return false; + } + + return ( + (value.chat !== undefined && hasChat(value.chat)) || + (value.embeddings !== undefined && hasEmbeddings(value.embeddings)) || + (value.fim !== undefined && hasFim(value.fim)) || + (value.agents !== undefined && hasAgents(value.agents)) + ); +} + +function hasChat(value: unknown): value is MistralChat { + return hasFunction(value, "complete") && hasFunction(value, "stream"); +} + +function hasEmbeddings(value: unknown): value is MistralEmbeddings { + return hasFunction(value, "create"); +} + +function hasFim(value: unknown): value is MistralFim { + return hasFunction(value, "complete") && hasFunction(value, "stream"); +} + +function hasAgents(value: unknown): value is MistralAgents { + return hasFunction(value, "complete") && hasFunction(value, "stream"); +} + +function mistralProxy(mistral: MistralClient): MistralClient { + return new Proxy(mistral, { + get(target, prop, receiver) { + switch (prop) { + case "chat": + return target.chat ? chatProxy(target.chat) : target.chat; + case "fim": + return target.fim ? fimProxy(target.fim) : target.fim; + case "agents": + return target.agents ? agentsProxy(target.agents) : target.agents; + case "embeddings": + return target.embeddings + ? embeddingsProxy(target.embeddings) + : target.embeddings; + default: + return Reflect.get(target, prop, receiver); + } + }, + }); +} + +function chatProxy(chat: MistralChat): MistralChat { + return new Proxy(chat, { + get(target, prop, receiver) { + if (prop === "complete") { + return wrapChatComplete(target.complete.bind(target)); + } + + if (prop === "stream") { + return wrapChatStream(target.stream.bind(target)); + } + + return Reflect.get(target, prop, receiver); + }, + }); +} + +function embeddingsProxy(embeddings: MistralEmbeddings): MistralEmbeddings { + return new Proxy(embeddings, { + get(target, prop, receiver) { + if (prop === "create") { + return wrapEmbeddingsCreate(target.create.bind(target)); + } + + return Reflect.get(target, prop, receiver); + }, + }); +} + +function fimProxy(fim: MistralFim): MistralFim { + return new Proxy(fim, { + get(target, prop, receiver) { + if (prop === "complete") { + return wrapFimComplete(target.complete.bind(target)); + } + + if (prop === "stream") { + return wrapFimStream(target.stream.bind(target)); + } + + return Reflect.get(target, prop, receiver); + }, + }); +} + +function agentsProxy(agents: MistralAgents): MistralAgents { + return new Proxy(agents, { + get(target, prop, receiver) { + if (prop === "complete") { + return wrapAgentsComplete(target.complete.bind(target)); + } + + if (prop === "stream") { + return wrapAgentsStream(target.stream.bind(target)); + } + + return Reflect.get(target, prop, receiver); + }, + }); +} + +function wrapChatComplete( + complete: ( + request: MistralChatCreateParams, + options?: unknown, + ) => Promise, +): MistralChat["complete"] { + return (request, options) => + mistralChannels.chatComplete.tracePromise( + () => complete(request, options), + { + arguments: [request], + } as Parameters[1], + ); +} + +function wrapChatStream( + stream: ( + request: MistralChatCreateParams, + options?: unknown, + ) => Promise, +): MistralChat["stream"] { + return (request, options) => + mistralChannels.chatStream.tracePromise(() => stream(request, options), { + arguments: [request], + } as Parameters[1]); +} + +function wrapEmbeddingsCreate( + create: ( + request: MistralEmbeddingCreateParams, + options?: unknown, + ) => Promise, +): MistralEmbeddings["create"] { + return (request, options) => + mistralChannels.embeddingsCreate.tracePromise( + () => create(request, options), + { arguments: [request] }, + ); +} + +function wrapFimComplete( + complete: ( + request: MistralFimCreateParams, + options?: unknown, + ) => Promise, +): MistralFim["complete"] { + return (request, options) => + mistralChannels.fimComplete.tracePromise(() => complete(request, options), { + arguments: [request], + } as Parameters[1]); +} + +function wrapFimStream( + stream: ( + request: MistralFimCreateParams, + options?: unknown, + ) => Promise, +): MistralFim["stream"] { + return (request, options) => + mistralChannels.fimStream.tracePromise(() => stream(request, options), { + arguments: [request], + } as Parameters[1]); +} + +function wrapAgentsComplete( + complete: ( + request: MistralAgentsCreateParams, + options?: unknown, + ) => Promise, +): MistralAgents["complete"] { + return (request, options) => + mistralChannels.agentsComplete.tracePromise( + () => complete(request, options), + { + arguments: [request], + } as Parameters[1], + ); +} + +function wrapAgentsStream( + stream: ( + request: MistralAgentsCreateParams, + options?: unknown, + ) => Promise, +): MistralAgents["stream"] { + return (request, options) => + mistralChannels.agentsStream.tracePromise(() => stream(request, options), { + arguments: [request], + } as Parameters[1]); +} diff --git a/turbo.json b/turbo.json index 08809361d..42046bc5d 100644 --- a/turbo.json +++ b/turbo.json @@ -5,7 +5,8 @@ "OPENAI_BASE_URL", "ANTHROPIC_API_KEY", "GEMINI_API_KEY", - "OPENROUTER_API_KEY" + "OPENROUTER_API_KEY", + "MISTRAL_API_KEY" ], "tasks": { "build": { @@ -24,7 +25,8 @@ "GEMINI_API_KEY", "OPENAI_API_KEY", "OPENAI_BASE_URL", - "OPENROUTER_API_KEY" + "OPENROUTER_API_KEY", + "MISTRAL_API_KEY" ], "dependsOn": ["^build"], "outputs": [] @@ -38,7 +40,8 @@ "GEMINI_API_KEY", "OPENAI_API_KEY", "OPENAI_BASE_URL", - "OPENROUTER_API_KEY" + "OPENROUTER_API_KEY", + "MISTRAL_API_KEY" ], "dependsOn": ["^build"], "outputs": [] @@ -57,7 +60,8 @@ "GEMINI_API_KEY", "OPENAI_API_KEY", "OPENAI_BASE_URL", - "OPENROUTER_API_KEY" + "OPENROUTER_API_KEY", + "MISTRAL_API_KEY" ], "dependsOn": [], "outputs": [] @@ -71,7 +75,8 @@ "GEMINI_API_KEY", "OPENAI_API_KEY", "OPENAI_BASE_URL", - "OPENROUTER_API_KEY" + "OPENROUTER_API_KEY", + "MISTRAL_API_KEY" ], "dependsOn": ["^build"], "outputs": [] @@ -99,7 +104,8 @@ "BRAINTRUST_API_KEY", "GEMINI_API_KEY", "OPENAI_API_KEY", - "OPENROUTER_API_KEY" + "OPENROUTER_API_KEY", + "MISTRAL_API_KEY" ], "dependsOn": ["^build", "build"] }, @@ -110,7 +116,8 @@ "BRAINTRUST_API_KEY", "GEMINI_API_KEY", "OPENAI_API_KEY", - "OPENROUTER_API_KEY" + "OPENROUTER_API_KEY", + "MISTRAL_API_KEY" ], "dependsOn": ["^build", "build"] }, @@ -121,7 +128,8 @@ "BRAINTRUST_API_KEY", "GEMINI_API_KEY", "OPENAI_API_KEY", - "OPENROUTER_API_KEY" + "OPENROUTER_API_KEY", + "MISTRAL_API_KEY" ], "dependsOn": ["^build", "build"] } From f0f80766534177fe272ae1b508fbcdd89c7e0c5c Mon Sep 17 00:00:00 2001 From: Luca Forstner Date: Thu, 2 Apr 2026 14:59:33 +0200 Subject: [PATCH 02/12] Better instrumentation --- .../mistral-v1-10-0.log-payloads.json | 67 +++++++++++++++--- .../mistral-v1-10-0.span-events.json | 66 ++++++++++++++---- .../mistral-v1-14-1.log-payloads.json | 67 +++++++++++++++--- .../mistral-v1-14-1.span-events.json | 66 ++++++++++++++---- .../mistral-v1-15-1.log-payloads.json | 67 +++++++++++++++--- .../mistral-v1-15-1.span-events.json | 66 ++++++++++++++---- .../mistral-v1-3-4.log-payloads.json | 67 +++++++++++++++--- .../mistral-v1-3-4.span-events.json | 66 ++++++++++++++---- .../mistral-v1.log-payloads.json | 67 +++++++++++++++--- .../__snapshots__/mistral-v1.span-events.json | 66 ++++++++++++++---- .../mistral-v2.log-payloads.json | 68 ++++++++++++++++--- .../__snapshots__/mistral-v2.span-events.json | 67 ++++++++++++++---- .../mistral-instrumentation/assertions.ts | 63 +++++++++++++++++ .../mistral-instrumentation/scenario.impl.mjs | 34 ++++++++++ 14 files changed, 747 insertions(+), 150 deletions(-) diff --git a/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-10-0.log-payloads.json b/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-10-0.log-payloads.json index 2812afcce..5118171ce 100644 --- a/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-10-0.log-payloads.json +++ b/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-10-0.log-payloads.json @@ -44,6 +44,7 @@ ], "output": { "choice_count": 1, + "finish_reason": "stop", "has_content": true, "role": "assistant", "tool_call_count": 0, @@ -84,6 +85,7 @@ ], "output": { "choice_count": 1, + "finish_reason": "stop", "has_content": true, "role": "assistant", "tool_call_count": 0, @@ -96,12 +98,53 @@ "has_output": false, "input": null, "metadata": { - "operation": "fim-complete" + "operation": "chat-tool-call" }, "metric_keys": [], "output": null, "span_id": "" }, + { + "has_input": true, + "has_output": true, + "input": { + "item_count": 1, + "roles": [ + "user" + ], + "type": "messages" + }, + "metadata": { + "model": "mistral-small-latest", + "provider": "mistral" + }, + "metric_keys": [ + "completion_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "output": { + "choice_count": 1, + "finish_reason": "tool_calls", + "has_content": false, + "role": "assistant", + "tool_call_count": 1, + "type": "choices" + }, + "span_id": "" + }, + { + "has_input": false, + "has_output": false, + "input": null, + "metadata": { + "operation": "fim-complete" + }, + "metric_keys": [], + "output": null, + "span_id": "" + }, { "has_input": true, "has_output": true, @@ -118,12 +161,13 @@ ], "output": { "choice_count": 1, + "finish_reason": "stop", "has_content": true, "role": "assistant", "tool_call_count": 0, "type": "choices" }, - "span_id": "" + "span_id": "" }, { "has_input": false, @@ -134,7 +178,7 @@ }, "metric_keys": [], "output": null, - "span_id": "" + "span_id": "" }, { "has_input": true, @@ -152,12 +196,13 @@ ], "output": { "choice_count": 1, + "finish_reason": "length", "has_content": true, "role": "assistant", "tool_call_count": 0, "type": "choices" }, - "span_id": "" + "span_id": "" }, { "has_input": false, @@ -168,7 +213,7 @@ }, "metric_keys": [], "output": null, - "span_id": "" + "span_id": "" }, { "has_input": true, @@ -192,12 +237,13 @@ ], "output": { "choice_count": 1, + "finish_reason": "stop", "has_content": true, "role": "assistant", "tool_call_count": 0, "type": "choices" }, - "span_id": "" + "span_id": "" }, { "has_input": false, @@ -208,7 +254,7 @@ }, "metric_keys": [], "output": null, - "span_id": "" + "span_id": "" }, { "has_input": true, @@ -232,12 +278,13 @@ ], "output": { "choice_count": 1, + "finish_reason": "stop", "has_content": true, "role": "assistant", "tool_call_count": 0, "type": "choices" }, - "span_id": "" + "span_id": "" }, { "has_input": false, @@ -248,7 +295,7 @@ }, "metric_keys": [], "output": null, - "span_id": "" + "span_id": "" }, { "has_input": true, @@ -267,6 +314,6 @@ "embedding_length": "", "type": "embedding" }, - "span_id": "" + "span_id": "" } ] diff --git a/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-10-0.span-events.json b/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-10-0.span-events.json index eb0733762..a24bb40b9 100644 --- a/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-10-0.span-events.json +++ b/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-10-0.span-events.json @@ -84,6 +84,42 @@ ], "type": "llm" }, + { + "has_input": false, + "has_output": false, + "metadata": { + "operation": "chat-tool-call" + }, + "metric_keys": [], + "name": "mistral-chat-tool-call-operation", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": null + }, + { + "has_input": true, + "has_output": true, + "metadata": { + "model": "mistral-small-latest", + "provider": "mistral" + }, + "metric_keys": [ + "completion_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "name": "mistral.chat.complete", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": "llm" + }, { "has_input": false, "has_output": false, @@ -93,7 +129,7 @@ "metric_keys": [], "name": "mistral-fim-complete-operation", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ "" ], @@ -114,9 +150,9 @@ ], "name": "mistral.fim.complete", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ - "" + "" ], "type": "llm" }, @@ -129,7 +165,7 @@ "metric_keys": [], "name": "mistral-fim-stream-operation", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ "" ], @@ -150,9 +186,9 @@ ], "name": "mistral.fim.stream", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ - "" + "" ], "type": "llm" }, @@ -165,7 +201,7 @@ "metric_keys": [], "name": "mistral-agents-complete-operation", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ "" ], @@ -186,9 +222,9 @@ ], "name": "mistral.agents.complete", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ - "" + "" ], "type": "llm" }, @@ -201,7 +237,7 @@ "metric_keys": [], "name": "mistral-agents-stream-operation", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ "" ], @@ -222,9 +258,9 @@ ], "name": "mistral.agents.stream", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ - "" + "" ], "type": "llm" }, @@ -237,7 +273,7 @@ "metric_keys": [], "name": "mistral-embeddings-operation", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ "" ], @@ -257,9 +293,9 @@ ], "name": "mistral.embeddings.create", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ - "" + "" ], "type": "llm" } diff --git a/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-14-1.log-payloads.json b/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-14-1.log-payloads.json index 2812afcce..5118171ce 100644 --- a/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-14-1.log-payloads.json +++ b/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-14-1.log-payloads.json @@ -44,6 +44,7 @@ ], "output": { "choice_count": 1, + "finish_reason": "stop", "has_content": true, "role": "assistant", "tool_call_count": 0, @@ -84,6 +85,7 @@ ], "output": { "choice_count": 1, + "finish_reason": "stop", "has_content": true, "role": "assistant", "tool_call_count": 0, @@ -96,12 +98,53 @@ "has_output": false, "input": null, "metadata": { - "operation": "fim-complete" + "operation": "chat-tool-call" }, "metric_keys": [], "output": null, "span_id": "" }, + { + "has_input": true, + "has_output": true, + "input": { + "item_count": 1, + "roles": [ + "user" + ], + "type": "messages" + }, + "metadata": { + "model": "mistral-small-latest", + "provider": "mistral" + }, + "metric_keys": [ + "completion_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "output": { + "choice_count": 1, + "finish_reason": "tool_calls", + "has_content": false, + "role": "assistant", + "tool_call_count": 1, + "type": "choices" + }, + "span_id": "" + }, + { + "has_input": false, + "has_output": false, + "input": null, + "metadata": { + "operation": "fim-complete" + }, + "metric_keys": [], + "output": null, + "span_id": "" + }, { "has_input": true, "has_output": true, @@ -118,12 +161,13 @@ ], "output": { "choice_count": 1, + "finish_reason": "stop", "has_content": true, "role": "assistant", "tool_call_count": 0, "type": "choices" }, - "span_id": "" + "span_id": "" }, { "has_input": false, @@ -134,7 +178,7 @@ }, "metric_keys": [], "output": null, - "span_id": "" + "span_id": "" }, { "has_input": true, @@ -152,12 +196,13 @@ ], "output": { "choice_count": 1, + "finish_reason": "length", "has_content": true, "role": "assistant", "tool_call_count": 0, "type": "choices" }, - "span_id": "" + "span_id": "" }, { "has_input": false, @@ -168,7 +213,7 @@ }, "metric_keys": [], "output": null, - "span_id": "" + "span_id": "" }, { "has_input": true, @@ -192,12 +237,13 @@ ], "output": { "choice_count": 1, + "finish_reason": "stop", "has_content": true, "role": "assistant", "tool_call_count": 0, "type": "choices" }, - "span_id": "" + "span_id": "" }, { "has_input": false, @@ -208,7 +254,7 @@ }, "metric_keys": [], "output": null, - "span_id": "" + "span_id": "" }, { "has_input": true, @@ -232,12 +278,13 @@ ], "output": { "choice_count": 1, + "finish_reason": "stop", "has_content": true, "role": "assistant", "tool_call_count": 0, "type": "choices" }, - "span_id": "" + "span_id": "" }, { "has_input": false, @@ -248,7 +295,7 @@ }, "metric_keys": [], "output": null, - "span_id": "" + "span_id": "" }, { "has_input": true, @@ -267,6 +314,6 @@ "embedding_length": "", "type": "embedding" }, - "span_id": "" + "span_id": "" } ] diff --git a/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-14-1.span-events.json b/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-14-1.span-events.json index eb0733762..a24bb40b9 100644 --- a/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-14-1.span-events.json +++ b/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-14-1.span-events.json @@ -84,6 +84,42 @@ ], "type": "llm" }, + { + "has_input": false, + "has_output": false, + "metadata": { + "operation": "chat-tool-call" + }, + "metric_keys": [], + "name": "mistral-chat-tool-call-operation", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": null + }, + { + "has_input": true, + "has_output": true, + "metadata": { + "model": "mistral-small-latest", + "provider": "mistral" + }, + "metric_keys": [ + "completion_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "name": "mistral.chat.complete", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": "llm" + }, { "has_input": false, "has_output": false, @@ -93,7 +129,7 @@ "metric_keys": [], "name": "mistral-fim-complete-operation", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ "" ], @@ -114,9 +150,9 @@ ], "name": "mistral.fim.complete", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ - "" + "" ], "type": "llm" }, @@ -129,7 +165,7 @@ "metric_keys": [], "name": "mistral-fim-stream-operation", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ "" ], @@ -150,9 +186,9 @@ ], "name": "mistral.fim.stream", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ - "" + "" ], "type": "llm" }, @@ -165,7 +201,7 @@ "metric_keys": [], "name": "mistral-agents-complete-operation", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ "" ], @@ -186,9 +222,9 @@ ], "name": "mistral.agents.complete", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ - "" + "" ], "type": "llm" }, @@ -201,7 +237,7 @@ "metric_keys": [], "name": "mistral-agents-stream-operation", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ "" ], @@ -222,9 +258,9 @@ ], "name": "mistral.agents.stream", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ - "" + "" ], "type": "llm" }, @@ -237,7 +273,7 @@ "metric_keys": [], "name": "mistral-embeddings-operation", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ "" ], @@ -257,9 +293,9 @@ ], "name": "mistral.embeddings.create", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ - "" + "" ], "type": "llm" } diff --git a/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-15-1.log-payloads.json b/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-15-1.log-payloads.json index 2812afcce..5118171ce 100644 --- a/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-15-1.log-payloads.json +++ b/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-15-1.log-payloads.json @@ -44,6 +44,7 @@ ], "output": { "choice_count": 1, + "finish_reason": "stop", "has_content": true, "role": "assistant", "tool_call_count": 0, @@ -84,6 +85,7 @@ ], "output": { "choice_count": 1, + "finish_reason": "stop", "has_content": true, "role": "assistant", "tool_call_count": 0, @@ -96,12 +98,53 @@ "has_output": false, "input": null, "metadata": { - "operation": "fim-complete" + "operation": "chat-tool-call" }, "metric_keys": [], "output": null, "span_id": "" }, + { + "has_input": true, + "has_output": true, + "input": { + "item_count": 1, + "roles": [ + "user" + ], + "type": "messages" + }, + "metadata": { + "model": "mistral-small-latest", + "provider": "mistral" + }, + "metric_keys": [ + "completion_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "output": { + "choice_count": 1, + "finish_reason": "tool_calls", + "has_content": false, + "role": "assistant", + "tool_call_count": 1, + "type": "choices" + }, + "span_id": "" + }, + { + "has_input": false, + "has_output": false, + "input": null, + "metadata": { + "operation": "fim-complete" + }, + "metric_keys": [], + "output": null, + "span_id": "" + }, { "has_input": true, "has_output": true, @@ -118,12 +161,13 @@ ], "output": { "choice_count": 1, + "finish_reason": "stop", "has_content": true, "role": "assistant", "tool_call_count": 0, "type": "choices" }, - "span_id": "" + "span_id": "" }, { "has_input": false, @@ -134,7 +178,7 @@ }, "metric_keys": [], "output": null, - "span_id": "" + "span_id": "" }, { "has_input": true, @@ -152,12 +196,13 @@ ], "output": { "choice_count": 1, + "finish_reason": "length", "has_content": true, "role": "assistant", "tool_call_count": 0, "type": "choices" }, - "span_id": "" + "span_id": "" }, { "has_input": false, @@ -168,7 +213,7 @@ }, "metric_keys": [], "output": null, - "span_id": "" + "span_id": "" }, { "has_input": true, @@ -192,12 +237,13 @@ ], "output": { "choice_count": 1, + "finish_reason": "stop", "has_content": true, "role": "assistant", "tool_call_count": 0, "type": "choices" }, - "span_id": "" + "span_id": "" }, { "has_input": false, @@ -208,7 +254,7 @@ }, "metric_keys": [], "output": null, - "span_id": "" + "span_id": "" }, { "has_input": true, @@ -232,12 +278,13 @@ ], "output": { "choice_count": 1, + "finish_reason": "stop", "has_content": true, "role": "assistant", "tool_call_count": 0, "type": "choices" }, - "span_id": "" + "span_id": "" }, { "has_input": false, @@ -248,7 +295,7 @@ }, "metric_keys": [], "output": null, - "span_id": "" + "span_id": "" }, { "has_input": true, @@ -267,6 +314,6 @@ "embedding_length": "", "type": "embedding" }, - "span_id": "" + "span_id": "" } ] diff --git a/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-15-1.span-events.json b/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-15-1.span-events.json index eb0733762..a24bb40b9 100644 --- a/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-15-1.span-events.json +++ b/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-15-1.span-events.json @@ -84,6 +84,42 @@ ], "type": "llm" }, + { + "has_input": false, + "has_output": false, + "metadata": { + "operation": "chat-tool-call" + }, + "metric_keys": [], + "name": "mistral-chat-tool-call-operation", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": null + }, + { + "has_input": true, + "has_output": true, + "metadata": { + "model": "mistral-small-latest", + "provider": "mistral" + }, + "metric_keys": [ + "completion_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "name": "mistral.chat.complete", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": "llm" + }, { "has_input": false, "has_output": false, @@ -93,7 +129,7 @@ "metric_keys": [], "name": "mistral-fim-complete-operation", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ "" ], @@ -114,9 +150,9 @@ ], "name": "mistral.fim.complete", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ - "" + "" ], "type": "llm" }, @@ -129,7 +165,7 @@ "metric_keys": [], "name": "mistral-fim-stream-operation", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ "" ], @@ -150,9 +186,9 @@ ], "name": "mistral.fim.stream", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ - "" + "" ], "type": "llm" }, @@ -165,7 +201,7 @@ "metric_keys": [], "name": "mistral-agents-complete-operation", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ "" ], @@ -186,9 +222,9 @@ ], "name": "mistral.agents.complete", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ - "" + "" ], "type": "llm" }, @@ -201,7 +237,7 @@ "metric_keys": [], "name": "mistral-agents-stream-operation", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ "" ], @@ -222,9 +258,9 @@ ], "name": "mistral.agents.stream", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ - "" + "" ], "type": "llm" }, @@ -237,7 +273,7 @@ "metric_keys": [], "name": "mistral-embeddings-operation", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ "" ], @@ -257,9 +293,9 @@ ], "name": "mistral.embeddings.create", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ - "" + "" ], "type": "llm" } diff --git a/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-3-4.log-payloads.json b/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-3-4.log-payloads.json index 2812afcce..5118171ce 100644 --- a/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-3-4.log-payloads.json +++ b/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-3-4.log-payloads.json @@ -44,6 +44,7 @@ ], "output": { "choice_count": 1, + "finish_reason": "stop", "has_content": true, "role": "assistant", "tool_call_count": 0, @@ -84,6 +85,7 @@ ], "output": { "choice_count": 1, + "finish_reason": "stop", "has_content": true, "role": "assistant", "tool_call_count": 0, @@ -96,12 +98,53 @@ "has_output": false, "input": null, "metadata": { - "operation": "fim-complete" + "operation": "chat-tool-call" }, "metric_keys": [], "output": null, "span_id": "" }, + { + "has_input": true, + "has_output": true, + "input": { + "item_count": 1, + "roles": [ + "user" + ], + "type": "messages" + }, + "metadata": { + "model": "mistral-small-latest", + "provider": "mistral" + }, + "metric_keys": [ + "completion_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "output": { + "choice_count": 1, + "finish_reason": "tool_calls", + "has_content": false, + "role": "assistant", + "tool_call_count": 1, + "type": "choices" + }, + "span_id": "" + }, + { + "has_input": false, + "has_output": false, + "input": null, + "metadata": { + "operation": "fim-complete" + }, + "metric_keys": [], + "output": null, + "span_id": "" + }, { "has_input": true, "has_output": true, @@ -118,12 +161,13 @@ ], "output": { "choice_count": 1, + "finish_reason": "stop", "has_content": true, "role": "assistant", "tool_call_count": 0, "type": "choices" }, - "span_id": "" + "span_id": "" }, { "has_input": false, @@ -134,7 +178,7 @@ }, "metric_keys": [], "output": null, - "span_id": "" + "span_id": "" }, { "has_input": true, @@ -152,12 +196,13 @@ ], "output": { "choice_count": 1, + "finish_reason": "length", "has_content": true, "role": "assistant", "tool_call_count": 0, "type": "choices" }, - "span_id": "" + "span_id": "" }, { "has_input": false, @@ -168,7 +213,7 @@ }, "metric_keys": [], "output": null, - "span_id": "" + "span_id": "" }, { "has_input": true, @@ -192,12 +237,13 @@ ], "output": { "choice_count": 1, + "finish_reason": "stop", "has_content": true, "role": "assistant", "tool_call_count": 0, "type": "choices" }, - "span_id": "" + "span_id": "" }, { "has_input": false, @@ -208,7 +254,7 @@ }, "metric_keys": [], "output": null, - "span_id": "" + "span_id": "" }, { "has_input": true, @@ -232,12 +278,13 @@ ], "output": { "choice_count": 1, + "finish_reason": "stop", "has_content": true, "role": "assistant", "tool_call_count": 0, "type": "choices" }, - "span_id": "" + "span_id": "" }, { "has_input": false, @@ -248,7 +295,7 @@ }, "metric_keys": [], "output": null, - "span_id": "" + "span_id": "" }, { "has_input": true, @@ -267,6 +314,6 @@ "embedding_length": "", "type": "embedding" }, - "span_id": "" + "span_id": "" } ] diff --git a/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-3-4.span-events.json b/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-3-4.span-events.json index eb0733762..a24bb40b9 100644 --- a/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-3-4.span-events.json +++ b/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-3-4.span-events.json @@ -84,6 +84,42 @@ ], "type": "llm" }, + { + "has_input": false, + "has_output": false, + "metadata": { + "operation": "chat-tool-call" + }, + "metric_keys": [], + "name": "mistral-chat-tool-call-operation", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": null + }, + { + "has_input": true, + "has_output": true, + "metadata": { + "model": "mistral-small-latest", + "provider": "mistral" + }, + "metric_keys": [ + "completion_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "name": "mistral.chat.complete", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": "llm" + }, { "has_input": false, "has_output": false, @@ -93,7 +129,7 @@ "metric_keys": [], "name": "mistral-fim-complete-operation", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ "" ], @@ -114,9 +150,9 @@ ], "name": "mistral.fim.complete", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ - "" + "" ], "type": "llm" }, @@ -129,7 +165,7 @@ "metric_keys": [], "name": "mistral-fim-stream-operation", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ "" ], @@ -150,9 +186,9 @@ ], "name": "mistral.fim.stream", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ - "" + "" ], "type": "llm" }, @@ -165,7 +201,7 @@ "metric_keys": [], "name": "mistral-agents-complete-operation", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ "" ], @@ -186,9 +222,9 @@ ], "name": "mistral.agents.complete", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ - "" + "" ], "type": "llm" }, @@ -201,7 +237,7 @@ "metric_keys": [], "name": "mistral-agents-stream-operation", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ "" ], @@ -222,9 +258,9 @@ ], "name": "mistral.agents.stream", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ - "" + "" ], "type": "llm" }, @@ -237,7 +273,7 @@ "metric_keys": [], "name": "mistral-embeddings-operation", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ "" ], @@ -257,9 +293,9 @@ ], "name": "mistral.embeddings.create", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ - "" + "" ], "type": "llm" } diff --git a/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1.log-payloads.json b/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1.log-payloads.json index 2812afcce..5118171ce 100644 --- a/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1.log-payloads.json +++ b/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1.log-payloads.json @@ -44,6 +44,7 @@ ], "output": { "choice_count": 1, + "finish_reason": "stop", "has_content": true, "role": "assistant", "tool_call_count": 0, @@ -84,6 +85,7 @@ ], "output": { "choice_count": 1, + "finish_reason": "stop", "has_content": true, "role": "assistant", "tool_call_count": 0, @@ -96,12 +98,53 @@ "has_output": false, "input": null, "metadata": { - "operation": "fim-complete" + "operation": "chat-tool-call" }, "metric_keys": [], "output": null, "span_id": "" }, + { + "has_input": true, + "has_output": true, + "input": { + "item_count": 1, + "roles": [ + "user" + ], + "type": "messages" + }, + "metadata": { + "model": "mistral-small-latest", + "provider": "mistral" + }, + "metric_keys": [ + "completion_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "output": { + "choice_count": 1, + "finish_reason": "tool_calls", + "has_content": false, + "role": "assistant", + "tool_call_count": 1, + "type": "choices" + }, + "span_id": "" + }, + { + "has_input": false, + "has_output": false, + "input": null, + "metadata": { + "operation": "fim-complete" + }, + "metric_keys": [], + "output": null, + "span_id": "" + }, { "has_input": true, "has_output": true, @@ -118,12 +161,13 @@ ], "output": { "choice_count": 1, + "finish_reason": "stop", "has_content": true, "role": "assistant", "tool_call_count": 0, "type": "choices" }, - "span_id": "" + "span_id": "" }, { "has_input": false, @@ -134,7 +178,7 @@ }, "metric_keys": [], "output": null, - "span_id": "" + "span_id": "" }, { "has_input": true, @@ -152,12 +196,13 @@ ], "output": { "choice_count": 1, + "finish_reason": "length", "has_content": true, "role": "assistant", "tool_call_count": 0, "type": "choices" }, - "span_id": "" + "span_id": "" }, { "has_input": false, @@ -168,7 +213,7 @@ }, "metric_keys": [], "output": null, - "span_id": "" + "span_id": "" }, { "has_input": true, @@ -192,12 +237,13 @@ ], "output": { "choice_count": 1, + "finish_reason": "stop", "has_content": true, "role": "assistant", "tool_call_count": 0, "type": "choices" }, - "span_id": "" + "span_id": "" }, { "has_input": false, @@ -208,7 +254,7 @@ }, "metric_keys": [], "output": null, - "span_id": "" + "span_id": "" }, { "has_input": true, @@ -232,12 +278,13 @@ ], "output": { "choice_count": 1, + "finish_reason": "stop", "has_content": true, "role": "assistant", "tool_call_count": 0, "type": "choices" }, - "span_id": "" + "span_id": "" }, { "has_input": false, @@ -248,7 +295,7 @@ }, "metric_keys": [], "output": null, - "span_id": "" + "span_id": "" }, { "has_input": true, @@ -267,6 +314,6 @@ "embedding_length": "", "type": "embedding" }, - "span_id": "" + "span_id": "" } ] diff --git a/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1.span-events.json b/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1.span-events.json index eb0733762..a24bb40b9 100644 --- a/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1.span-events.json +++ b/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1.span-events.json @@ -84,6 +84,42 @@ ], "type": "llm" }, + { + "has_input": false, + "has_output": false, + "metadata": { + "operation": "chat-tool-call" + }, + "metric_keys": [], + "name": "mistral-chat-tool-call-operation", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": null + }, + { + "has_input": true, + "has_output": true, + "metadata": { + "model": "mistral-small-latest", + "provider": "mistral" + }, + "metric_keys": [ + "completion_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "name": "mistral.chat.complete", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": "llm" + }, { "has_input": false, "has_output": false, @@ -93,7 +129,7 @@ "metric_keys": [], "name": "mistral-fim-complete-operation", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ "" ], @@ -114,9 +150,9 @@ ], "name": "mistral.fim.complete", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ - "" + "" ], "type": "llm" }, @@ -129,7 +165,7 @@ "metric_keys": [], "name": "mistral-fim-stream-operation", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ "" ], @@ -150,9 +186,9 @@ ], "name": "mistral.fim.stream", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ - "" + "" ], "type": "llm" }, @@ -165,7 +201,7 @@ "metric_keys": [], "name": "mistral-agents-complete-operation", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ "" ], @@ -186,9 +222,9 @@ ], "name": "mistral.agents.complete", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ - "" + "" ], "type": "llm" }, @@ -201,7 +237,7 @@ "metric_keys": [], "name": "mistral-agents-stream-operation", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ "" ], @@ -222,9 +258,9 @@ ], "name": "mistral.agents.stream", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ - "" + "" ], "type": "llm" }, @@ -237,7 +273,7 @@ "metric_keys": [], "name": "mistral-embeddings-operation", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ "" ], @@ -257,9 +293,9 @@ ], "name": "mistral.embeddings.create", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ - "" + "" ], "type": "llm" } diff --git a/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v2.log-payloads.json b/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v2.log-payloads.json index abb4d8a80..ab8e22f4f 100644 --- a/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v2.log-payloads.json +++ b/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v2.log-payloads.json @@ -45,6 +45,7 @@ ], "output": { "choice_count": 1, + "finish_reason": "stop", "has_content": true, "role": "assistant", "tool_call_count": 0, @@ -86,6 +87,7 @@ ], "output": { "choice_count": 1, + "finish_reason": "stop", "has_content": true, "role": "assistant", "tool_call_count": 0, @@ -98,12 +100,54 @@ "has_output": false, "input": null, "metadata": { - "operation": "fim-complete" + "operation": "chat-tool-call" }, "metric_keys": [], "output": null, "span_id": "" }, + { + "has_input": true, + "has_output": true, + "input": { + "item_count": 1, + "roles": [ + "user" + ], + "type": "messages" + }, + "metadata": { + "model": "mistral-small-latest", + "provider": "mistral" + }, + "metric_keys": [ + "completion_tokens", + "prompt_cached_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "output": { + "choice_count": 1, + "finish_reason": "tool_calls", + "has_content": false, + "role": "assistant", + "tool_call_count": 1, + "type": "choices" + }, + "span_id": "" + }, + { + "has_input": false, + "has_output": false, + "input": null, + "metadata": { + "operation": "fim-complete" + }, + "metric_keys": [], + "output": null, + "span_id": "" + }, { "has_input": true, "has_output": true, @@ -121,12 +165,13 @@ ], "output": { "choice_count": 1, + "finish_reason": "stop", "has_content": true, "role": "assistant", "tool_call_count": 0, "type": "choices" }, - "span_id": "" + "span_id": "" }, { "has_input": false, @@ -137,7 +182,7 @@ }, "metric_keys": [], "output": null, - "span_id": "" + "span_id": "" }, { "has_input": true, @@ -156,12 +201,13 @@ ], "output": { "choice_count": 1, + "finish_reason": "length", "has_content": true, "role": "assistant", "tool_call_count": 0, "type": "choices" }, - "span_id": "" + "span_id": "" }, { "has_input": false, @@ -172,7 +218,7 @@ }, "metric_keys": [], "output": null, - "span_id": "" + "span_id": "" }, { "has_input": true, @@ -197,12 +243,13 @@ ], "output": { "choice_count": 1, + "finish_reason": "stop", "has_content": true, "role": "assistant", "tool_call_count": 0, "type": "choices" }, - "span_id": "" + "span_id": "" }, { "has_input": false, @@ -213,7 +260,7 @@ }, "metric_keys": [], "output": null, - "span_id": "" + "span_id": "" }, { "has_input": true, @@ -238,12 +285,13 @@ ], "output": { "choice_count": 1, + "finish_reason": "stop", "has_content": true, "role": "assistant", "tool_call_count": 0, "type": "choices" }, - "span_id": "" + "span_id": "" }, { "has_input": false, @@ -254,7 +302,7 @@ }, "metric_keys": [], "output": null, - "span_id": "" + "span_id": "" }, { "has_input": true, @@ -273,6 +321,6 @@ "embedding_length": "", "type": "embedding" }, - "span_id": "" + "span_id": "" } ] diff --git a/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v2.span-events.json b/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v2.span-events.json index a55bc5909..85a4050a1 100644 --- a/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v2.span-events.json +++ b/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v2.span-events.json @@ -86,6 +86,43 @@ ], "type": "llm" }, + { + "has_input": false, + "has_output": false, + "metadata": { + "operation": "chat-tool-call" + }, + "metric_keys": [], + "name": "mistral-chat-tool-call-operation", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": null + }, + { + "has_input": true, + "has_output": true, + "metadata": { + "model": "mistral-small-latest", + "provider": "mistral" + }, + "metric_keys": [ + "completion_tokens", + "prompt_cached_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "name": "mistral.chat.complete", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": "llm" + }, { "has_input": false, "has_output": false, @@ -95,7 +132,7 @@ "metric_keys": [], "name": "mistral-fim-complete-operation", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ "" ], @@ -117,9 +154,9 @@ ], "name": "mistral.fim.complete", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ - "" + "" ], "type": "llm" }, @@ -132,7 +169,7 @@ "metric_keys": [], "name": "mistral-fim-stream-operation", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ "" ], @@ -154,9 +191,9 @@ ], "name": "mistral.fim.stream", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ - "" + "" ], "type": "llm" }, @@ -169,7 +206,7 @@ "metric_keys": [], "name": "mistral-agents-complete-operation", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ "" ], @@ -191,9 +228,9 @@ ], "name": "mistral.agents.complete", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ - "" + "" ], "type": "llm" }, @@ -206,7 +243,7 @@ "metric_keys": [], "name": "mistral-agents-stream-operation", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ "" ], @@ -228,9 +265,9 @@ ], "name": "mistral.agents.stream", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ - "" + "" ], "type": "llm" }, @@ -243,7 +280,7 @@ "metric_keys": [], "name": "mistral-embeddings-operation", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ "" ], @@ -263,9 +300,9 @@ ], "name": "mistral.embeddings.create", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ - "" + "" ], "type": "llm" } diff --git a/e2e/scenarios/mistral-instrumentation/assertions.ts b/e2e/scenarios/mistral-instrumentation/assertions.ts index 2dccabdaa..1d4643da5 100644 --- a/e2e/scenarios/mistral-instrumentation/assertions.ts +++ b/e2e/scenarios/mistral-instrumentation/assertions.ts @@ -118,11 +118,20 @@ function summarizeOutput(output: unknown): Json { if (!isRecord(firstChoice) || !isRecord(firstChoice.message)) { return { choice_count: output.length, + finish_reason: null, type: "array", }; } const message = firstChoice.message; + const finishReason = + typeof firstChoice.finishReason === "string" || + firstChoice.finishReason === null + ? firstChoice.finishReason + : typeof firstChoice.finish_reason === "string" || + firstChoice.finish_reason === null + ? firstChoice.finish_reason + : null; const toolCalls = (Array.isArray(message.tool_calls) && message.tool_calls) || (Array.isArray(message.toolCalls) && message.toolCalls) || @@ -130,6 +139,7 @@ function summarizeOutput(output: unknown): Json { return { choice_count: output.length, + finish_reason: finishReason, has_content: typeof message.content === "string" ? message.content.length > 0 @@ -251,6 +261,10 @@ function buildSpanSummary(events: CapturedLogEvent[]): Json { events, "mistral-chat-stream-operation", ); + const chatToolCallOperation = findLatestSpan( + events, + "mistral-chat-tool-call-operation", + ); const fimCompleteOperation = findLatestSpan( events, "mistral-fim-complete-operation", @@ -283,6 +297,10 @@ function buildSpanSummary(events: CapturedLogEvent[]): Json { findMistralSpan(events, chatStreamOperation?.span.id, [ "mistral.chat.stream", ]), + chatToolCallOperation, + findMistralSpan(events, chatToolCallOperation?.span.id, [ + "mistral.chat.complete", + ]), fimCompleteOperation, findMistralSpan(events, fimCompleteOperation?.span.id, [ "mistral.fim.complete", @@ -406,6 +424,51 @@ export function defineMistralInstrumentationAssertions(options: { expect(span?.output).toBeDefined(); }); + test("captures trace for chat.complete() tool calling", testConfig, () => { + const root = findLatestSpan(events, ROOT_NAME); + const operation = findLatestSpan( + events, + "mistral-chat-tool-call-operation", + ); + const span = findMistralSpan(events, operation?.span.id, [ + "mistral.chat.complete", + ]); + const output = span?.output as + | Array<{ + finishReason?: unknown; + finish_reason?: unknown; + message?: { + toolCalls?: unknown; + tool_calls?: unknown; + }; + }> + | undefined; + const firstChoice = Array.isArray(output) ? output[0] : undefined; + const toolCalls = + (Array.isArray(firstChoice?.message?.tool_calls) && + firstChoice.message.tool_calls) || + (Array.isArray(firstChoice?.message?.toolCalls) && + firstChoice.message.toolCalls) || + []; + const finishReason = + typeof firstChoice?.finishReason === "string" + ? firstChoice.finishReason + : typeof firstChoice?.finish_reason === "string" + ? firstChoice.finish_reason + : undefined; + + expect(operation).toBeDefined(); + expect(span).toBeDefined(); + expect(operation?.span.parentIds).toEqual([root?.span.id ?? ""]); + expect(span?.span.type).toBe("llm"); + expect(span?.row.metadata).toMatchObject({ + model: CHAT_MODEL, + provider: "mistral", + }); + expect(toolCalls.length).toBeGreaterThan(0); + expect(finishReason).toEqual(expect.any(String)); + }); + test("captures trace for fim.complete()", testConfig, () => { const root = findLatestSpan(events, ROOT_NAME); const operation = findLatestSpan( diff --git a/e2e/scenarios/mistral-instrumentation/scenario.impl.mjs b/e2e/scenarios/mistral-instrumentation/scenario.impl.mjs index e4d4bc30b..78c076a76 100644 --- a/e2e/scenarios/mistral-instrumentation/scenario.impl.mjs +++ b/e2e/scenarios/mistral-instrumentation/scenario.impl.mjs @@ -317,6 +317,40 @@ async function runMistralInstrumentationScenario( }, ); + await runOperation( + "mistral-chat-tool-call-operation", + "chat-tool-call", + async () => { + await withRetry( + async () => + client.chat.complete({ + model: CHAT_MODEL, + messages: [ + { + role: "user", + content: + "Call the get_weather tool. Do not answer with plain text.", + }, + ], + tools: [ + { + type: "function", + function: { + name: "get_weather", + description: "Get weather for a city.", + parameters: {}, + }, + }, + ], + toolChoice: "required", + maxTokens: 48, + temperature: 0, + }), + { attempts: 3, delayMs: 1_000 }, + ); + }, + ); + await runOperation( "mistral-fim-complete-operation", "fim-complete", From 6c6d488509bd8dea752b4929addf5f8c23144dbb Mon Sep 17 00:00:00 2001 From: Luca Forstner Date: Thu, 2 Apr 2026 16:02:06 +0200 Subject: [PATCH 03/12] fix lint --- js/src/instrumentation/plugins/mistral-plugin.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/js/src/instrumentation/plugins/mistral-plugin.ts b/js/src/instrumentation/plugins/mistral-plugin.ts index e9e977b20..0b56f0540 100644 --- a/js/src/instrumentation/plugins/mistral-plugin.ts +++ b/js/src/instrumentation/plugins/mistral-plugin.ts @@ -318,7 +318,7 @@ function mergeToolCallDeltas( return toolCalls; } - let merged = toolCalls ? [...toolCalls] : []; + const merged = toolCalls ? [...toolCalls] : []; for (const delta of deltas) { const current = merged[merged.length - 1]; From 7519c4a6a24096f515570147b889e7a9932de2f6 Mon Sep 17 00:00:00 2001 From: Luca Forstner Date: Thu, 2 Apr 2026 16:45:41 +0200 Subject: [PATCH 04/12] Fix tool call deltas --- .../plugins/mistral-plugin.test.ts | 78 +++++++++++ .../instrumentation/plugins/mistral-plugin.ts | 132 +++++++++++++----- 2 files changed, 174 insertions(+), 36 deletions(-) diff --git a/js/src/instrumentation/plugins/mistral-plugin.test.ts b/js/src/instrumentation/plugins/mistral-plugin.test.ts index 48aed1d6c..97b94e560 100644 --- a/js/src/instrumentation/plugins/mistral-plugin.test.ts +++ b/js/src/instrumentation/plugins/mistral-plugin.test.ts @@ -166,4 +166,82 @@ describe("aggregateMistralStreamChunks", () => { finishReason: "tool_calls", }); }); + + it("merges interleaved tool call deltas by index", () => { + const aggregated = aggregateMistralStreamChunks([ + { + data: { + choices: [ + { + delta: { + toolCalls: [ + { + index: 0, + id: "tool_0", + function: { + name: "first_tool", + arguments: '{"city":"Vie', + }, + }, + { + index: 1, + id: "tool_1", + function: { + name: "second_tool", + arguments: '{"unit":"c"}', + }, + }, + ], + }, + }, + ], + }, + }, + { + data: { + choices: [ + { + delta: { + tool_calls: [ + { + index: 0, + id: "tool_0", + function: { + arguments: 'nna"}', + }, + }, + ], + }, + finishReason: "tool_calls", + }, + ], + }, + }, + ]); + + expect(aggregated.output?.[0]).toMatchObject({ + message: { + content: null, + toolCalls: [ + { + index: 0, + id: "tool_0", + function: { + name: "first_tool", + arguments: '{"city":"Vienna"}', + }, + }, + { + index: 1, + id: "tool_1", + function: { + name: "second_tool", + arguments: '{"unit":"c"}', + }, + }, + ], + }, + finishReason: "tool_calls", + }); + }); }); diff --git a/js/src/instrumentation/plugins/mistral-plugin.ts b/js/src/instrumentation/plugins/mistral-plugin.ts index 0b56f0540..168e9ca28 100644 --- a/js/src/instrumentation/plugins/mistral-plugin.ts +++ b/js/src/instrumentation/plugins/mistral-plugin.ts @@ -310,6 +310,51 @@ function getDeltaToolCalls( return toolCalls.filter((toolCall) => isObject(toolCall)); } +function getToolCallIndex(toolCall: MistralToolCallDelta): number | undefined { + return typeof toolCall.index === "number" && toolCall.index >= 0 + ? toolCall.index + : undefined; +} + +function createMergedToolCallDelta( + delta: MistralToolCallDelta, +): MistralToolCallDelta { + return { + ...delta, + function: { + ...delta.function, + arguments: + typeof delta.function?.arguments === "string" + ? delta.function.arguments + : "", + }, + }; +} + +function mergeToolCallDeltaPair( + current: MistralToolCallDelta, + delta: MistralToolCallDelta, +): MistralToolCallDelta { + const currentArguments = + typeof current.function?.arguments === "string" + ? current.function.arguments + : ""; + const deltaArguments = + typeof delta.function?.arguments === "string" + ? delta.function.arguments + : ""; + + return { + ...current, + ...delta, + function: { + ...(current.function || {}), + ...(delta.function || {}), + arguments: `${currentArguments}${deltaArguments}`, + }, + }; +} + function mergeToolCallDeltas( toolCalls: MistralToolCallDelta[] | undefined, deltas: MistralToolCallDelta[], @@ -319,47 +364,62 @@ function mergeToolCallDeltas( } const merged = toolCalls ? [...toolCalls] : []; + const indexToPosition = new Map(); + const idToPosition = new Map(); + + for (const [position, toolCall] of merged.entries()) { + const index = getToolCallIndex(toolCall); + if (index !== undefined && !indexToPosition.has(index)) { + indexToPosition.set(index, position); + } + + if (typeof toolCall.id === "string" && !idToPosition.has(toolCall.id)) { + idToPosition.set(toolCall.id, position); + } + } for (const delta of deltas) { - const current = merged[merged.length - 1]; - const sameTool = - current && - typeof current.id === "string" && - typeof delta.id === "string" && - current.id === delta.id; - - if (!sameTool) { - merged.push({ - ...delta, - function: { - ...delta.function, - arguments: - typeof delta.function?.arguments === "string" - ? delta.function.arguments - : "", - }, - }); + const deltaIndex = getToolCallIndex(delta); + const existingByIndex = + deltaIndex !== undefined ? indexToPosition.get(deltaIndex) : undefined; + const existingById = + typeof delta.id === "string" ? idToPosition.get(delta.id) : undefined; + const existingPosition = existingByIndex ?? existingById; + + if (existingPosition === undefined) { + const newToolCall = createMergedToolCallDelta(delta); + merged.push(newToolCall); + + const newPosition = merged.length - 1; + const newIndex = getToolCallIndex(newToolCall); + if (newIndex !== undefined && !indexToPosition.has(newIndex)) { + indexToPosition.set(newIndex, newPosition); + } + if ( + typeof newToolCall.id === "string" && + !idToPosition.has(newToolCall.id) + ) { + idToPosition.set(newToolCall.id, newPosition); + } continue; } - const currentArguments = - typeof current.function?.arguments === "string" - ? current.function.arguments - : ""; - const deltaArguments = - typeof delta.function?.arguments === "string" - ? delta.function.arguments - : ""; - - merged[merged.length - 1] = { - ...current, - ...delta, - function: { - ...(current.function || {}), - ...(delta.function || {}), - arguments: `${currentArguments}${deltaArguments}`, - }, - }; + const mergedToolCall = mergeToolCallDeltaPair( + merged[existingPosition], + delta, + ); + merged[existingPosition] = mergedToolCall; + + const mergedIndex = getToolCallIndex(mergedToolCall); + if (mergedIndex !== undefined && !indexToPosition.has(mergedIndex)) { + indexToPosition.set(mergedIndex, existingPosition); + } + if ( + typeof mergedToolCall.id === "string" && + !idToPosition.has(mergedToolCall.id) + ) { + idToPosition.set(mergedToolCall.id, existingPosition); + } } return merged.length > 0 ? merged : undefined; From 1e25bf5878497435467493222798e55ab86b9690 Mon Sep 17 00:00:00 2001 From: Luca Forstner Date: Thu, 2 Apr 2026 18:05:10 +0200 Subject: [PATCH 05/12] fix --- e2e/config/pr-comment-scenarios.json | 13 ++ .../plugins/mistral-plugin.test.ts | 123 ++++++++++ .../instrumentation/plugins/mistral-plugin.ts | 211 ++++++++++++++---- 3 files changed, 305 insertions(+), 42 deletions(-) diff --git a/e2e/config/pr-comment-scenarios.json b/e2e/config/pr-comment-scenarios.json index 02435c1ed..94a29964f 100644 --- a/e2e/config/pr-comment-scenarios.json +++ b/e2e/config/pr-comment-scenarios.json @@ -33,6 +33,19 @@ { "variantKey": "google-genai-v1460", "label": "v1.46.0" } ] }, + { + "scenarioDirName": "mistral-instrumentation", + "label": "Mistral Instrumentation", + "metadataScenario": "mistral-instrumentation", + "variants": [ + { "variantKey": "mistral-v1-3-4", "label": "v1.3.4" }, + { "variantKey": "mistral-v1-10-0", "label": "v1.10.0" }, + { "variantKey": "mistral-v1-14-1", "label": "v1.14.1" }, + { "variantKey": "mistral-v1-15-1", "label": "v1.15.1" }, + { "variantKey": "mistral-v1", "label": "latest v1" }, + { "variantKey": "mistral-v2", "label": "latest v2" } + ] + }, { "scenarioDirName": "openrouter-instrumentation", "label": "OpenRouter Instrumentation", diff --git a/js/src/instrumentation/plugins/mistral-plugin.test.ts b/js/src/instrumentation/plugins/mistral-plugin.test.ts index 97b94e560..e73d63076 100644 --- a/js/src/instrumentation/plugins/mistral-plugin.test.ts +++ b/js/src/instrumentation/plugins/mistral-plugin.test.ts @@ -1,9 +1,74 @@ import { describe, expect, it } from "vitest"; import { aggregateMistralStreamChunks, + extractMistralRequestMetadata, + extractMistralResponseMetadata, parseMistralMetricsFromUsage, } from "./mistral-plugin"; +describe("extractMistralRequestMetadata", () => { + it("keeps only allowlisted request metadata", () => { + expect( + extractMistralRequestMetadata({ + model: "mistral-large-latest", + maxTokens: 128, + temperature: 0.4, + n: 2, + safe_prompt: true, + toolChoice: "auto", + messages: [{ role: "user", content: "hi" }], + tools: [{ type: "function" }], + suffix: "ignored", + arbitrary: "ignored", + }), + ).toEqual({ + model: "mistral-large-latest", + maxTokens: 128, + temperature: 0.4, + n: 2, + safe_prompt: true, + toolChoice: "auto", + }); + }); + + it("returns empty metadata for missing input", () => { + expect(extractMistralRequestMetadata(undefined)).toEqual({}); + }); +}); + +describe("extractMistralResponseMetadata", () => { + it("keeps only allowlisted response metadata", () => { + expect( + extractMistralResponseMetadata({ + id: "cmpl_123", + created: 1234, + object: "chat.completion", + model: "mistral-large-latest", + agentId: "agent_123", + usage: { total_tokens: 20 }, + choices: [{ index: 0 }], + data: [{ embedding: [0.1] }], + arbitrary: "ignored", + }), + ).toEqual({ + id: "cmpl_123", + created: 1234, + object: "chat.completion", + model: "mistral-large-latest", + agentId: "agent_123", + }); + }); + + it("returns undefined when no allowlisted keys are present", () => { + expect( + extractMistralResponseMetadata({ + usage: { total_tokens: 20 }, + choices: [{ index: 0 }], + }), + ).toBeUndefined(); + }); +}); + describe("parseMistralMetricsFromUsage", () => { it("returns empty metrics for missing usage", () => { expect(parseMistralMetricsFromUsage(undefined)).toEqual({}); @@ -244,4 +309,62 @@ describe("aggregateMistralStreamChunks", () => { finishReason: "tool_calls", }); }); + + it("keeps streamed choices separated when multiple choices are returned", () => { + const aggregated = aggregateMistralStreamChunks([ + { + data: { + choices: [ + { + index: 0, + delta: { + role: "assistant", + content: "a", + }, + }, + { + index: 1, + delta: { + role: "assistant", + content: "b", + }, + }, + ], + }, + }, + { + data: { + choices: [ + { + index: 0, + finishReason: "stop", + }, + { + index: 1, + finishReason: "length", + }, + ], + }, + }, + ]); + + expect(aggregated.output).toEqual([ + { + index: 0, + message: { + role: "assistant", + content: "a", + }, + finishReason: "stop", + }, + { + index: 1, + message: { + role: "assistant", + content: "b", + }, + finishReason: "length", + }, + ]); + }); }); diff --git a/js/src/instrumentation/plugins/mistral-plugin.ts b/js/src/instrumentation/plugins/mistral-plugin.ts index 168e9ca28..88ae25f46 100644 --- a/js/src/instrumentation/plugins/mistral-plugin.ts +++ b/js/src/instrumentation/plugins/mistral-plugin.ts @@ -151,6 +151,43 @@ const TOKEN_DETAIL_PREFIX_MAP: Record = { output_tokens_details: "completion", }; +const MISTRAL_REQUEST_METADATA_ALLOWLIST = new Set([ + "agentId", + "agent_id", + "encodingFormat", + "encoding_format", + "frequencyPenalty", + "frequency_penalty", + "maxTokens", + "max_tokens", + "model", + "n", + "presencePenalty", + "presence_penalty", + "randomSeed", + "random_seed", + "responseFormat", + "response_format", + "safePrompt", + "safe_prompt", + "stream", + "stop", + "temperature", + "toolChoice", + "tool_choice", + "topP", + "top_p", +]); + +const MISTRAL_RESPONSE_METADATA_ALLOWLIST = new Set([ + "agentId", + "agent_id", + "created", + "id", + "model", + "object", +]); + function camelToSnake(value: string): string { return value.replace(/[A-Z]/g, (match) => `_${match.toLowerCase()}`); } @@ -193,16 +230,54 @@ function addMistralProviderMetadata( }; } +function pickAllowedMetadata( + metadata: Record | undefined, + allowlist: ReadonlySet, +): Record { + if (!metadata) { + return {}; + } + + const picked: Record = {}; + for (const key of allowlist) { + const value = metadata[key]; + if (value !== undefined) { + picked[key] = value; + } + } + return picked; +} + +export function extractMistralRequestMetadata( + metadata: Record | undefined, +): Record { + return pickAllowedMetadata(metadata, MISTRAL_REQUEST_METADATA_ALLOWLIST); +} + +function isMistralChatCompletionChunk( + value: unknown, +): value is MistralChatCompletionChunk { + return isObject(value); +} + +function isMistralChunkChoice( + value: unknown, +): value is MistralChatCompletionChunkChoice { + return isObject(value); +} + function extractMessagesInputWithMetadata(args: unknown[] | unknown): { input: unknown; metadata: Record; } { const params = getMistralRequestArg(args); - const { messages, ...metadata } = params || {}; + const { messages, ...rawMetadata } = params || {}; return { input: processInputAttachments(messages), - metadata: addMistralProviderMetadata(metadata), + metadata: addMistralProviderMetadata( + extractMistralRequestMetadata(rawMetadata), + ), }; } @@ -211,11 +286,13 @@ function extractEmbeddingInputWithMetadata(args: unknown[] | unknown): { metadata: Record; } { const params = getMistralRequestArg(args); - const { inputs, ...metadata } = params || {}; + const { inputs, ...rawMetadata } = params || {}; return { input: inputs, - metadata: addMistralProviderMetadata(metadata), + metadata: addMistralProviderMetadata( + extractMistralRequestMetadata(rawMetadata), + ), }; } @@ -224,29 +301,30 @@ function extractPromptInputWithMetadata(args: unknown[] | unknown): { metadata: Record; } { const params = getMistralRequestArg(args); - const { prompt, ...metadata } = params || {}; + const { prompt, ...rawMetadata } = params || {}; return { input: prompt, - metadata: addMistralProviderMetadata(metadata), + metadata: addMistralProviderMetadata( + extractMistralRequestMetadata(rawMetadata), + ), }; } -function extractMistralResponseMetadata( +export function extractMistralResponseMetadata( result: unknown, ): Record | undefined { if (!isObject(result)) { return undefined; } - const { - choices: _choices, - usage: _usage, - data: _data, - ...metadata - } = result as Record; + const { choices: _choices, usage: _usage, data: _data, ...metadata } = result; + const picked = pickAllowedMetadata( + metadata, + MISTRAL_RESPONSE_METADATA_ALLOWLIST, + ); - return Object.keys(metadata).length > 0 ? metadata : undefined; + return Object.keys(picked).length > 0 ? picked : undefined; } function extractMistralMetrics( @@ -442,6 +520,15 @@ function getChoiceFinishReason( return undefined; } +type MistralChoiceAccumulator = { + content?: string; + finishReason?: string | null; + index: number; + order: number; + role?: string; + toolCalls?: MistralToolCallDelta[]; +}; + export function parseMistralMetricsFromUsage( usage: unknown, ): Record { @@ -485,16 +572,16 @@ export function aggregateMistralStreamChunks( metrics: Record; metadata?: Record; } { - let role: string | undefined; - let content: string | undefined; - let toolCalls: MistralToolCallDelta[] | undefined; - let finishReason: string | null | undefined; + const choiceAccumulators = new Map(); + const indexToAccumulatorKey = new Map(); + const positionToAccumulatorKey = new Map(); + let nextAccumulatorOrder = 0; let metrics: Record = {}; let metadata: Record | undefined; for (const event of chunks) { - const chunk = isObject(event?.data) - ? (event.data as MistralChatCompletionChunk) + const chunk = isMistralChatCompletionChunk(event?.data) + ? event.data : undefined; if (!chunk) { continue; @@ -512,47 +599,87 @@ export function aggregateMistralStreamChunks( metadata = { ...(metadata || {}), ...chunkMetadata }; } - for (const rawChoice of chunk.choices || []) { - if (!isObject(rawChoice)) { + for (const [choicePosition, rawChoice] of (chunk.choices || []).entries()) { + if (!isMistralChunkChoice(rawChoice)) { continue; } - const choice = rawChoice as MistralChatCompletionChunkChoice; + const choice = rawChoice; + const choiceIndex = + typeof choice.index === "number" && choice.index >= 0 + ? choice.index + : undefined; + let accumulatorKey = + choiceIndex !== undefined + ? indexToAccumulatorKey.get(choiceIndex) + : undefined; + if (!accumulatorKey) { + accumulatorKey = positionToAccumulatorKey.get(choicePosition); + } + if (!accumulatorKey) { + const initialIndex = choiceIndex ?? choicePosition; + const keyPrefix = choiceIndex !== undefined ? "index" : "position"; + accumulatorKey = `${keyPrefix}:${initialIndex}`; + choiceAccumulators.set(accumulatorKey, { + index: initialIndex, + order: nextAccumulatorOrder++, + }); + } - const delta = isObject(choice.delta) - ? (choice.delta as Record) - : undefined; + const accumulator = choiceAccumulators.get(accumulatorKey); + if (!accumulator) { + continue; + } + + if (choiceIndex !== undefined) { + accumulator.index = choiceIndex; + indexToAccumulatorKey.set(choiceIndex, accumulatorKey); + } + positionToAccumulatorKey.set(choicePosition, accumulatorKey); + + const delta = isObject(choice.delta) ? choice.delta : undefined; if (delta) { - if (!role && typeof delta.role === "string") { - role = delta.role; + if (!accumulator.role && typeof delta.role === "string") { + accumulator.role = delta.role; } const deltaText = extractDeltaText(delta.content); if (deltaText) { - content = (content || "") + deltaText; + accumulator.content = `${accumulator.content || ""}${deltaText}`; } - toolCalls = mergeToolCallDeltas(toolCalls, getDeltaToolCalls(delta)); + accumulator.toolCalls = mergeToolCallDeltas( + accumulator.toolCalls, + getDeltaToolCalls(delta), + ); } const choiceFinishReason = getChoiceFinishReason(choice); if (choiceFinishReason !== undefined) { - finishReason = choiceFinishReason; + accumulator.finishReason = choiceFinishReason; } } } - return { - output: [ - { - index: 0, - message: { - ...(role ? { role } : {}), - content: content ?? null, - ...(toolCalls ? { toolCalls } : {}), - }, - ...(finishReason !== undefined ? { finishReason } : {}), + const output = Array.from(choiceAccumulators.values()) + .sort((left, right) => + left.index === right.index + ? left.order - right.order + : left.index - right.index, + ) + .map((choice) => ({ + index: choice.index, + message: { + ...(choice.role ? { role: choice.role } : {}), + content: choice.content ?? null, + ...(choice.toolCalls ? { toolCalls: choice.toolCalls } : {}), }, - ], + ...(choice.finishReason !== undefined + ? { finishReason: choice.finishReason } + : {}), + })); + + return { + output, metrics, ...(metadata ? { metadata } : {}), }; From c643bc8ddf8b2c47db9b799e74b65e484d890ae4 Mon Sep 17 00:00:00 2001 From: Luca Forstner Date: Thu, 2 Apr 2026 18:31:02 +0200 Subject: [PATCH 06/12] fix --- .../mistral-v1-3-4.log-payloads.json | 7 +- .../mistral-v1-3-4.span-events.json | 5 +- .../mistral-instrumentation/assertions.ts | 104 ++++++++++++++++-- 3 files changed, 97 insertions(+), 19 deletions(-) diff --git a/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-3-4.log-payloads.json b/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-3-4.log-payloads.json index 5118171ce..f15ddfa04 100644 --- a/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-3-4.log-payloads.json +++ b/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-3-4.log-payloads.json @@ -189,14 +189,11 @@ "provider": "mistral" }, "metric_keys": [ - "completion_tokens", - "prompt_tokens", - "time_to_first_token", - "tokens" + "time_to_first_token" ], "output": { "choice_count": 1, - "finish_reason": "length", + "finish_reason": null, "has_content": true, "role": "assistant", "tool_call_count": 0, diff --git a/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-3-4.span-events.json b/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-3-4.span-events.json index a24bb40b9..970134bf2 100644 --- a/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-3-4.span-events.json +++ b/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-3-4.span-events.json @@ -179,10 +179,7 @@ "provider": "mistral" }, "metric_keys": [ - "completion_tokens", - "prompt_tokens", - "time_to_first_token", - "tokens" + "time_to_first_token" ], "name": "mistral.fim.stream", "root_span_id": "", diff --git a/e2e/scenarios/mistral-instrumentation/assertions.ts b/e2e/scenarios/mistral-instrumentation/assertions.ts index 1d4643da5..79ea578a8 100644 --- a/e2e/scenarios/mistral-instrumentation/assertions.ts +++ b/e2e/scenarios/mistral-instrumentation/assertions.ts @@ -191,6 +191,63 @@ function summarizePayloadRow(row: CapturedLogRow): Json { } satisfies Json; } +function normalizeLegacyV134MetricKeys(metricKeys: Json): Json { + if (!Array.isArray(metricKeys)) { + return metricKeys; + } + + return metricKeys.includes("time_to_first_token") + ? ["time_to_first_token"] + : metricKeys; +} + +function normalizeLegacyV134SpanSummaryRow( + summaryRow: Json, + snapshotName: string, +): Json { + if (snapshotName !== "mistral-v1-3-4" || !isRecord(summaryRow)) { + return summaryRow; + } + + if (summaryRow.name !== "mistral.fim.stream") { + return summaryRow; + } + + return { + ...summaryRow, + metric_keys: normalizeLegacyV134MetricKeys(summaryRow.metric_keys), + }; +} + +function normalizeLegacyV134PayloadSummaryRow( + summaryRow: Json, + snapshotName: string, + spanName: string | undefined, +): Json { + if ( + snapshotName !== "mistral-v1-3-4" || + spanName !== "mistral.fim.stream" || + !isRecord(summaryRow) + ) { + return summaryRow; + } + + const output = isRecord(summaryRow.output) ? summaryRow.output : null; + + return { + ...summaryRow, + metric_keys: normalizeLegacyV134MetricKeys(summaryRow.metric_keys), + ...(output + ? { + output: { + ...output, + finish_reason: null, + }, + } + : {}), + }; +} + function mergeRecordValues( left: Record | undefined, right: unknown, @@ -252,7 +309,10 @@ function mergePayloadRows(rows: CapturedLogRow[]): CapturedLogRow[] { .filter((row): row is CapturedLogRow => row !== undefined); } -function buildSpanSummary(events: CapturedLogEvent[]): Json { +function buildSpanSummary( + events: CapturedLogEvent[], + snapshotName: string, +): Json { const chatCompleteOperation = findLatestSpan( events, "mistral-chat-complete-operation", @@ -322,12 +382,15 @@ function buildSpanSummary(events: CapturedLogEvent[]): Json { "mistral.embeddings.create", ]), ].map((event) => - summarizeWrapperContract(event!, [ - "model", - "operation", - "provider", - "scenario", - ]), + normalizeLegacyV134SpanSummaryRow( + summarizeWrapperContract(event!, [ + "model", + "operation", + "provider", + "scenario", + ]), + snapshotName, + ), ) as Json, ); } @@ -335,12 +398,31 @@ function buildSpanSummary(events: CapturedLogEvent[]): Json { function buildPayloadSummary( events: CapturedLogEvent[], payloads: CapturedLogPayload[], + snapshotName: string, ): Json { + const spanNameById = new Map(); + for (const event of events) { + if ( + typeof event?.span?.id === "string" && + typeof event?.span?.name === "string" + ) { + spanNameById.set(event.span.id, event.span.name); + } + } + const root = findLatestSpan(events, ROOT_NAME); const payloadRows = payloadRowsForRootSpan(payloads, root?.span.id); const mergedRows = mergePayloadRows(payloadRows); return normalizeForSnapshot( - mergedRows.map((row) => summarizePayloadRow(row)), + mergedRows.map((row) => + normalizeLegacyV134PayloadSummaryRow( + summarizePayloadRow(row), + snapshotName, + typeof row.span_id === "string" + ? spanNameById.get(row.span_id) + : undefined, + ), + ), ); } @@ -596,13 +678,15 @@ export function defineMistralInstrumentationAssertions(options: { test("matches the shared span snapshot", testConfig, async () => { await expect( - formatJsonFileSnapshot(buildSpanSummary(events)), + formatJsonFileSnapshot(buildSpanSummary(events, options.snapshotName)), ).toMatchFileSnapshot(spanSnapshotPath); }); test("matches the shared payload snapshot", testConfig, async () => { await expect( - formatJsonFileSnapshot(buildPayloadSummary(events, payloads)), + formatJsonFileSnapshot( + buildPayloadSummary(events, payloads, options.snapshotName), + ), ).toMatchFileSnapshot(payloadSnapshotPath); }); }); From 3256d5cc0080a27e51ff74521659325621395d63 Mon Sep 17 00:00:00 2001 From: Luca Forstner Date: Thu, 2 Apr 2026 23:10:16 +0200 Subject: [PATCH 07/12] tools --- .../mistral-v1-10-0.log-payloads.json | 76 ++++++-- .../mistral-v1-10-0.span-events.json | 81 +++++++-- .../mistral-v1-14-1.log-payloads.json | 76 ++++++-- .../mistral-v1-14-1.span-events.json | 81 +++++++-- .../mistral-v1-15-1.log-payloads.json | 76 ++++++-- .../mistral-v1-15-1.span-events.json | 81 +++++++-- .../mistral-v1-3-4.log-payloads.json | 76 ++++++-- .../mistral-v1-3-4.span-events.json | 81 +++++++-- .../mistral-v1.log-payloads.json | 76 ++++++-- .../__snapshots__/mistral-v1.span-events.json | 81 +++++++-- .../mistral-v2.log-payloads.json | 77 ++++++-- .../__snapshots__/mistral-v2.span-events.json | 82 +++++++-- .../mistral-instrumentation/assertions.ts | 169 +++++++++++++----- .../mistral-instrumentation/scenario.impl.mjs | 30 +++- .../instrumentation/core/channel-tracing.ts | 72 ++++++++ .../plugins/mistral-plugin.test.ts | 60 +++++++ .../instrumentation/plugins/mistral-plugin.ts | 162 +++++++++++++++++ 17 files changed, 1240 insertions(+), 197 deletions(-) diff --git a/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-10-0.log-payloads.json b/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-10-0.log-payloads.json index 5118171ce..a5ff90d87 100644 --- a/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-10-0.log-payloads.json +++ b/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-10-0.log-payloads.json @@ -134,6 +134,62 @@ }, "span_id": "" }, + { + "has_input": true, + "has_output": false, + "input": { + "type": "object" + }, + "metadata": { + "provider": "mistral" + }, + "metric_keys": [], + "output": null, + "span_id": "" + }, + { + "has_input": true, + "has_output": true, + "input": { + "item_count": 1, + "roles": [ + "user" + ], + "type": "messages" + }, + "metadata": { + "model": "mistral-small-latest", + "provider": "mistral" + }, + "metric_keys": [ + "completion_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "output": { + "choice_count": 1, + "finish_reason": "tool_calls", + "has_content": false, + "role": "assistant", + "tool_call_count": 1, + "type": "choices" + }, + "span_id": "" + }, + { + "has_input": true, + "has_output": false, + "input": { + "type": "object" + }, + "metadata": { + "provider": "mistral" + }, + "metric_keys": [], + "output": null, + "span_id": "" + }, { "has_input": false, "has_output": false, @@ -143,7 +199,7 @@ }, "metric_keys": [], "output": null, - "span_id": "" + "span_id": "" }, { "has_input": true, @@ -167,7 +223,7 @@ "tool_call_count": 0, "type": "choices" }, - "span_id": "" + "span_id": "" }, { "has_input": false, @@ -178,7 +234,7 @@ }, "metric_keys": [], "output": null, - "span_id": "" + "span_id": "" }, { "has_input": true, @@ -202,7 +258,7 @@ "tool_call_count": 0, "type": "choices" }, - "span_id": "" + "span_id": "" }, { "has_input": false, @@ -213,7 +269,7 @@ }, "metric_keys": [], "output": null, - "span_id": "" + "span_id": "" }, { "has_input": true, @@ -243,7 +299,7 @@ "tool_call_count": 0, "type": "choices" }, - "span_id": "" + "span_id": "" }, { "has_input": false, @@ -254,7 +310,7 @@ }, "metric_keys": [], "output": null, - "span_id": "" + "span_id": "" }, { "has_input": true, @@ -284,7 +340,7 @@ "tool_call_count": 0, "type": "choices" }, - "span_id": "" + "span_id": "" }, { "has_input": false, @@ -295,7 +351,7 @@ }, "metric_keys": [], "output": null, - "span_id": "" + "span_id": "" }, { "has_input": true, @@ -314,6 +370,6 @@ "embedding_length": "", "type": "embedding" }, - "span_id": "" + "span_id": "" } ] diff --git a/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-10-0.span-events.json b/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-10-0.span-events.json index a24bb40b9..cd67b9364 100644 --- a/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-10-0.span-events.json +++ b/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-10-0.span-events.json @@ -120,6 +120,57 @@ ], "type": "llm" }, + { + "has_input": true, + "has_output": true, + "metadata": { + "model": "mistral-small-latest", + "provider": "mistral" + }, + "metric_keys": [ + "completion_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "name": "mistral.chat.complete", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": "llm" + }, + { + "has_input": true, + "has_output": false, + "metadata": { + "provider": "mistral" + }, + "metric_keys": [], + "name": "mistral.tool", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": "tool" + }, + { + "has_input": true, + "has_output": false, + "metadata": { + "provider": "mistral" + }, + "metric_keys": [], + "name": "mistral.tool", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": "tool" + }, { "has_input": false, "has_output": false, @@ -129,7 +180,7 @@ "metric_keys": [], "name": "mistral-fim-complete-operation", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ "" ], @@ -150,9 +201,9 @@ ], "name": "mistral.fim.complete", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ - "" + "" ], "type": "llm" }, @@ -165,7 +216,7 @@ "metric_keys": [], "name": "mistral-fim-stream-operation", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ "" ], @@ -186,9 +237,9 @@ ], "name": "mistral.fim.stream", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ - "" + "" ], "type": "llm" }, @@ -201,7 +252,7 @@ "metric_keys": [], "name": "mistral-agents-complete-operation", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ "" ], @@ -222,9 +273,9 @@ ], "name": "mistral.agents.complete", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ - "" + "" ], "type": "llm" }, @@ -237,7 +288,7 @@ "metric_keys": [], "name": "mistral-agents-stream-operation", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ "" ], @@ -258,9 +309,9 @@ ], "name": "mistral.agents.stream", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ - "" + "" ], "type": "llm" }, @@ -273,7 +324,7 @@ "metric_keys": [], "name": "mistral-embeddings-operation", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ "" ], @@ -293,9 +344,9 @@ ], "name": "mistral.embeddings.create", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ - "" + "" ], "type": "llm" } diff --git a/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-14-1.log-payloads.json b/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-14-1.log-payloads.json index 5118171ce..a5ff90d87 100644 --- a/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-14-1.log-payloads.json +++ b/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-14-1.log-payloads.json @@ -134,6 +134,62 @@ }, "span_id": "" }, + { + "has_input": true, + "has_output": false, + "input": { + "type": "object" + }, + "metadata": { + "provider": "mistral" + }, + "metric_keys": [], + "output": null, + "span_id": "" + }, + { + "has_input": true, + "has_output": true, + "input": { + "item_count": 1, + "roles": [ + "user" + ], + "type": "messages" + }, + "metadata": { + "model": "mistral-small-latest", + "provider": "mistral" + }, + "metric_keys": [ + "completion_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "output": { + "choice_count": 1, + "finish_reason": "tool_calls", + "has_content": false, + "role": "assistant", + "tool_call_count": 1, + "type": "choices" + }, + "span_id": "" + }, + { + "has_input": true, + "has_output": false, + "input": { + "type": "object" + }, + "metadata": { + "provider": "mistral" + }, + "metric_keys": [], + "output": null, + "span_id": "" + }, { "has_input": false, "has_output": false, @@ -143,7 +199,7 @@ }, "metric_keys": [], "output": null, - "span_id": "" + "span_id": "" }, { "has_input": true, @@ -167,7 +223,7 @@ "tool_call_count": 0, "type": "choices" }, - "span_id": "" + "span_id": "" }, { "has_input": false, @@ -178,7 +234,7 @@ }, "metric_keys": [], "output": null, - "span_id": "" + "span_id": "" }, { "has_input": true, @@ -202,7 +258,7 @@ "tool_call_count": 0, "type": "choices" }, - "span_id": "" + "span_id": "" }, { "has_input": false, @@ -213,7 +269,7 @@ }, "metric_keys": [], "output": null, - "span_id": "" + "span_id": "" }, { "has_input": true, @@ -243,7 +299,7 @@ "tool_call_count": 0, "type": "choices" }, - "span_id": "" + "span_id": "" }, { "has_input": false, @@ -254,7 +310,7 @@ }, "metric_keys": [], "output": null, - "span_id": "" + "span_id": "" }, { "has_input": true, @@ -284,7 +340,7 @@ "tool_call_count": 0, "type": "choices" }, - "span_id": "" + "span_id": "" }, { "has_input": false, @@ -295,7 +351,7 @@ }, "metric_keys": [], "output": null, - "span_id": "" + "span_id": "" }, { "has_input": true, @@ -314,6 +370,6 @@ "embedding_length": "", "type": "embedding" }, - "span_id": "" + "span_id": "" } ] diff --git a/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-14-1.span-events.json b/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-14-1.span-events.json index a24bb40b9..cd67b9364 100644 --- a/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-14-1.span-events.json +++ b/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-14-1.span-events.json @@ -120,6 +120,57 @@ ], "type": "llm" }, + { + "has_input": true, + "has_output": true, + "metadata": { + "model": "mistral-small-latest", + "provider": "mistral" + }, + "metric_keys": [ + "completion_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "name": "mistral.chat.complete", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": "llm" + }, + { + "has_input": true, + "has_output": false, + "metadata": { + "provider": "mistral" + }, + "metric_keys": [], + "name": "mistral.tool", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": "tool" + }, + { + "has_input": true, + "has_output": false, + "metadata": { + "provider": "mistral" + }, + "metric_keys": [], + "name": "mistral.tool", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": "tool" + }, { "has_input": false, "has_output": false, @@ -129,7 +180,7 @@ "metric_keys": [], "name": "mistral-fim-complete-operation", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ "" ], @@ -150,9 +201,9 @@ ], "name": "mistral.fim.complete", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ - "" + "" ], "type": "llm" }, @@ -165,7 +216,7 @@ "metric_keys": [], "name": "mistral-fim-stream-operation", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ "" ], @@ -186,9 +237,9 @@ ], "name": "mistral.fim.stream", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ - "" + "" ], "type": "llm" }, @@ -201,7 +252,7 @@ "metric_keys": [], "name": "mistral-agents-complete-operation", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ "" ], @@ -222,9 +273,9 @@ ], "name": "mistral.agents.complete", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ - "" + "" ], "type": "llm" }, @@ -237,7 +288,7 @@ "metric_keys": [], "name": "mistral-agents-stream-operation", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ "" ], @@ -258,9 +309,9 @@ ], "name": "mistral.agents.stream", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ - "" + "" ], "type": "llm" }, @@ -273,7 +324,7 @@ "metric_keys": [], "name": "mistral-embeddings-operation", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ "" ], @@ -293,9 +344,9 @@ ], "name": "mistral.embeddings.create", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ - "" + "" ], "type": "llm" } diff --git a/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-15-1.log-payloads.json b/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-15-1.log-payloads.json index 5118171ce..a5ff90d87 100644 --- a/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-15-1.log-payloads.json +++ b/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-15-1.log-payloads.json @@ -134,6 +134,62 @@ }, "span_id": "" }, + { + "has_input": true, + "has_output": false, + "input": { + "type": "object" + }, + "metadata": { + "provider": "mistral" + }, + "metric_keys": [], + "output": null, + "span_id": "" + }, + { + "has_input": true, + "has_output": true, + "input": { + "item_count": 1, + "roles": [ + "user" + ], + "type": "messages" + }, + "metadata": { + "model": "mistral-small-latest", + "provider": "mistral" + }, + "metric_keys": [ + "completion_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "output": { + "choice_count": 1, + "finish_reason": "tool_calls", + "has_content": false, + "role": "assistant", + "tool_call_count": 1, + "type": "choices" + }, + "span_id": "" + }, + { + "has_input": true, + "has_output": false, + "input": { + "type": "object" + }, + "metadata": { + "provider": "mistral" + }, + "metric_keys": [], + "output": null, + "span_id": "" + }, { "has_input": false, "has_output": false, @@ -143,7 +199,7 @@ }, "metric_keys": [], "output": null, - "span_id": "" + "span_id": "" }, { "has_input": true, @@ -167,7 +223,7 @@ "tool_call_count": 0, "type": "choices" }, - "span_id": "" + "span_id": "" }, { "has_input": false, @@ -178,7 +234,7 @@ }, "metric_keys": [], "output": null, - "span_id": "" + "span_id": "" }, { "has_input": true, @@ -202,7 +258,7 @@ "tool_call_count": 0, "type": "choices" }, - "span_id": "" + "span_id": "" }, { "has_input": false, @@ -213,7 +269,7 @@ }, "metric_keys": [], "output": null, - "span_id": "" + "span_id": "" }, { "has_input": true, @@ -243,7 +299,7 @@ "tool_call_count": 0, "type": "choices" }, - "span_id": "" + "span_id": "" }, { "has_input": false, @@ -254,7 +310,7 @@ }, "metric_keys": [], "output": null, - "span_id": "" + "span_id": "" }, { "has_input": true, @@ -284,7 +340,7 @@ "tool_call_count": 0, "type": "choices" }, - "span_id": "" + "span_id": "" }, { "has_input": false, @@ -295,7 +351,7 @@ }, "metric_keys": [], "output": null, - "span_id": "" + "span_id": "" }, { "has_input": true, @@ -314,6 +370,6 @@ "embedding_length": "", "type": "embedding" }, - "span_id": "" + "span_id": "" } ] diff --git a/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-15-1.span-events.json b/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-15-1.span-events.json index a24bb40b9..cd67b9364 100644 --- a/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-15-1.span-events.json +++ b/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-15-1.span-events.json @@ -120,6 +120,57 @@ ], "type": "llm" }, + { + "has_input": true, + "has_output": true, + "metadata": { + "model": "mistral-small-latest", + "provider": "mistral" + }, + "metric_keys": [ + "completion_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "name": "mistral.chat.complete", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": "llm" + }, + { + "has_input": true, + "has_output": false, + "metadata": { + "provider": "mistral" + }, + "metric_keys": [], + "name": "mistral.tool", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": "tool" + }, + { + "has_input": true, + "has_output": false, + "metadata": { + "provider": "mistral" + }, + "metric_keys": [], + "name": "mistral.tool", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": "tool" + }, { "has_input": false, "has_output": false, @@ -129,7 +180,7 @@ "metric_keys": [], "name": "mistral-fim-complete-operation", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ "" ], @@ -150,9 +201,9 @@ ], "name": "mistral.fim.complete", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ - "" + "" ], "type": "llm" }, @@ -165,7 +216,7 @@ "metric_keys": [], "name": "mistral-fim-stream-operation", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ "" ], @@ -186,9 +237,9 @@ ], "name": "mistral.fim.stream", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ - "" + "" ], "type": "llm" }, @@ -201,7 +252,7 @@ "metric_keys": [], "name": "mistral-agents-complete-operation", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ "" ], @@ -222,9 +273,9 @@ ], "name": "mistral.agents.complete", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ - "" + "" ], "type": "llm" }, @@ -237,7 +288,7 @@ "metric_keys": [], "name": "mistral-agents-stream-operation", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ "" ], @@ -258,9 +309,9 @@ ], "name": "mistral.agents.stream", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ - "" + "" ], "type": "llm" }, @@ -273,7 +324,7 @@ "metric_keys": [], "name": "mistral-embeddings-operation", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ "" ], @@ -293,9 +344,9 @@ ], "name": "mistral.embeddings.create", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ - "" + "" ], "type": "llm" } diff --git a/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-3-4.log-payloads.json b/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-3-4.log-payloads.json index f15ddfa04..91d8993c2 100644 --- a/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-3-4.log-payloads.json +++ b/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-3-4.log-payloads.json @@ -134,6 +134,62 @@ }, "span_id": "" }, + { + "has_input": true, + "has_output": false, + "input": { + "type": "object" + }, + "metadata": { + "provider": "mistral" + }, + "metric_keys": [], + "output": null, + "span_id": "" + }, + { + "has_input": true, + "has_output": true, + "input": { + "item_count": 1, + "roles": [ + "user" + ], + "type": "messages" + }, + "metadata": { + "model": "mistral-small-latest", + "provider": "mistral" + }, + "metric_keys": [ + "completion_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "output": { + "choice_count": 1, + "finish_reason": "tool_calls", + "has_content": false, + "role": "assistant", + "tool_call_count": 1, + "type": "choices" + }, + "span_id": "" + }, + { + "has_input": true, + "has_output": false, + "input": { + "type": "object" + }, + "metadata": { + "provider": "mistral" + }, + "metric_keys": [], + "output": null, + "span_id": "" + }, { "has_input": false, "has_output": false, @@ -143,7 +199,7 @@ }, "metric_keys": [], "output": null, - "span_id": "" + "span_id": "" }, { "has_input": true, @@ -167,7 +223,7 @@ "tool_call_count": 0, "type": "choices" }, - "span_id": "" + "span_id": "" }, { "has_input": false, @@ -178,7 +234,7 @@ }, "metric_keys": [], "output": null, - "span_id": "" + "span_id": "" }, { "has_input": true, @@ -199,7 +255,7 @@ "tool_call_count": 0, "type": "choices" }, - "span_id": "" + "span_id": "" }, { "has_input": false, @@ -210,7 +266,7 @@ }, "metric_keys": [], "output": null, - "span_id": "" + "span_id": "" }, { "has_input": true, @@ -240,7 +296,7 @@ "tool_call_count": 0, "type": "choices" }, - "span_id": "" + "span_id": "" }, { "has_input": false, @@ -251,7 +307,7 @@ }, "metric_keys": [], "output": null, - "span_id": "" + "span_id": "" }, { "has_input": true, @@ -281,7 +337,7 @@ "tool_call_count": 0, "type": "choices" }, - "span_id": "" + "span_id": "" }, { "has_input": false, @@ -292,7 +348,7 @@ }, "metric_keys": [], "output": null, - "span_id": "" + "span_id": "" }, { "has_input": true, @@ -311,6 +367,6 @@ "embedding_length": "", "type": "embedding" }, - "span_id": "" + "span_id": "" } ] diff --git a/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-3-4.span-events.json b/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-3-4.span-events.json index 970134bf2..6990e605a 100644 --- a/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-3-4.span-events.json +++ b/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-3-4.span-events.json @@ -120,6 +120,57 @@ ], "type": "llm" }, + { + "has_input": true, + "has_output": true, + "metadata": { + "model": "mistral-small-latest", + "provider": "mistral" + }, + "metric_keys": [ + "completion_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "name": "mistral.chat.complete", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": "llm" + }, + { + "has_input": true, + "has_output": false, + "metadata": { + "provider": "mistral" + }, + "metric_keys": [], + "name": "mistral.tool", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": "tool" + }, + { + "has_input": true, + "has_output": false, + "metadata": { + "provider": "mistral" + }, + "metric_keys": [], + "name": "mistral.tool", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": "tool" + }, { "has_input": false, "has_output": false, @@ -129,7 +180,7 @@ "metric_keys": [], "name": "mistral-fim-complete-operation", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ "" ], @@ -150,9 +201,9 @@ ], "name": "mistral.fim.complete", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ - "" + "" ], "type": "llm" }, @@ -165,7 +216,7 @@ "metric_keys": [], "name": "mistral-fim-stream-operation", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ "" ], @@ -183,9 +234,9 @@ ], "name": "mistral.fim.stream", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ - "" + "" ], "type": "llm" }, @@ -198,7 +249,7 @@ "metric_keys": [], "name": "mistral-agents-complete-operation", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ "" ], @@ -219,9 +270,9 @@ ], "name": "mistral.agents.complete", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ - "" + "" ], "type": "llm" }, @@ -234,7 +285,7 @@ "metric_keys": [], "name": "mistral-agents-stream-operation", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ "" ], @@ -255,9 +306,9 @@ ], "name": "mistral.agents.stream", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ - "" + "" ], "type": "llm" }, @@ -270,7 +321,7 @@ "metric_keys": [], "name": "mistral-embeddings-operation", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ "" ], @@ -290,9 +341,9 @@ ], "name": "mistral.embeddings.create", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ - "" + "" ], "type": "llm" } diff --git a/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1.log-payloads.json b/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1.log-payloads.json index 5118171ce..a5ff90d87 100644 --- a/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1.log-payloads.json +++ b/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1.log-payloads.json @@ -134,6 +134,62 @@ }, "span_id": "" }, + { + "has_input": true, + "has_output": false, + "input": { + "type": "object" + }, + "metadata": { + "provider": "mistral" + }, + "metric_keys": [], + "output": null, + "span_id": "" + }, + { + "has_input": true, + "has_output": true, + "input": { + "item_count": 1, + "roles": [ + "user" + ], + "type": "messages" + }, + "metadata": { + "model": "mistral-small-latest", + "provider": "mistral" + }, + "metric_keys": [ + "completion_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "output": { + "choice_count": 1, + "finish_reason": "tool_calls", + "has_content": false, + "role": "assistant", + "tool_call_count": 1, + "type": "choices" + }, + "span_id": "" + }, + { + "has_input": true, + "has_output": false, + "input": { + "type": "object" + }, + "metadata": { + "provider": "mistral" + }, + "metric_keys": [], + "output": null, + "span_id": "" + }, { "has_input": false, "has_output": false, @@ -143,7 +199,7 @@ }, "metric_keys": [], "output": null, - "span_id": "" + "span_id": "" }, { "has_input": true, @@ -167,7 +223,7 @@ "tool_call_count": 0, "type": "choices" }, - "span_id": "" + "span_id": "" }, { "has_input": false, @@ -178,7 +234,7 @@ }, "metric_keys": [], "output": null, - "span_id": "" + "span_id": "" }, { "has_input": true, @@ -202,7 +258,7 @@ "tool_call_count": 0, "type": "choices" }, - "span_id": "" + "span_id": "" }, { "has_input": false, @@ -213,7 +269,7 @@ }, "metric_keys": [], "output": null, - "span_id": "" + "span_id": "" }, { "has_input": true, @@ -243,7 +299,7 @@ "tool_call_count": 0, "type": "choices" }, - "span_id": "" + "span_id": "" }, { "has_input": false, @@ -254,7 +310,7 @@ }, "metric_keys": [], "output": null, - "span_id": "" + "span_id": "" }, { "has_input": true, @@ -284,7 +340,7 @@ "tool_call_count": 0, "type": "choices" }, - "span_id": "" + "span_id": "" }, { "has_input": false, @@ -295,7 +351,7 @@ }, "metric_keys": [], "output": null, - "span_id": "" + "span_id": "" }, { "has_input": true, @@ -314,6 +370,6 @@ "embedding_length": "", "type": "embedding" }, - "span_id": "" + "span_id": "" } ] diff --git a/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1.span-events.json b/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1.span-events.json index a24bb40b9..cd67b9364 100644 --- a/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1.span-events.json +++ b/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1.span-events.json @@ -120,6 +120,57 @@ ], "type": "llm" }, + { + "has_input": true, + "has_output": true, + "metadata": { + "model": "mistral-small-latest", + "provider": "mistral" + }, + "metric_keys": [ + "completion_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "name": "mistral.chat.complete", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": "llm" + }, + { + "has_input": true, + "has_output": false, + "metadata": { + "provider": "mistral" + }, + "metric_keys": [], + "name": "mistral.tool", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": "tool" + }, + { + "has_input": true, + "has_output": false, + "metadata": { + "provider": "mistral" + }, + "metric_keys": [], + "name": "mistral.tool", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": "tool" + }, { "has_input": false, "has_output": false, @@ -129,7 +180,7 @@ "metric_keys": [], "name": "mistral-fim-complete-operation", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ "" ], @@ -150,9 +201,9 @@ ], "name": "mistral.fim.complete", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ - "" + "" ], "type": "llm" }, @@ -165,7 +216,7 @@ "metric_keys": [], "name": "mistral-fim-stream-operation", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ "" ], @@ -186,9 +237,9 @@ ], "name": "mistral.fim.stream", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ - "" + "" ], "type": "llm" }, @@ -201,7 +252,7 @@ "metric_keys": [], "name": "mistral-agents-complete-operation", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ "" ], @@ -222,9 +273,9 @@ ], "name": "mistral.agents.complete", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ - "" + "" ], "type": "llm" }, @@ -237,7 +288,7 @@ "metric_keys": [], "name": "mistral-agents-stream-operation", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ "" ], @@ -258,9 +309,9 @@ ], "name": "mistral.agents.stream", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ - "" + "" ], "type": "llm" }, @@ -273,7 +324,7 @@ "metric_keys": [], "name": "mistral-embeddings-operation", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ "" ], @@ -293,9 +344,9 @@ ], "name": "mistral.embeddings.create", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ - "" + "" ], "type": "llm" } diff --git a/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v2.log-payloads.json b/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v2.log-payloads.json index ab8e22f4f..4bb90778e 100644 --- a/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v2.log-payloads.json +++ b/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v2.log-payloads.json @@ -137,6 +137,63 @@ }, "span_id": "" }, + { + "has_input": true, + "has_output": false, + "input": { + "type": "object" + }, + "metadata": { + "provider": "mistral" + }, + "metric_keys": [], + "output": null, + "span_id": "" + }, + { + "has_input": true, + "has_output": true, + "input": { + "item_count": 1, + "roles": [ + "user" + ], + "type": "messages" + }, + "metadata": { + "model": "mistral-small-latest", + "provider": "mistral" + }, + "metric_keys": [ + "completion_tokens", + "prompt_cached_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "output": { + "choice_count": 1, + "finish_reason": "tool_calls", + "has_content": false, + "role": "assistant", + "tool_call_count": 1, + "type": "choices" + }, + "span_id": "" + }, + { + "has_input": true, + "has_output": false, + "input": { + "type": "object" + }, + "metadata": { + "provider": "mistral" + }, + "metric_keys": [], + "output": null, + "span_id": "" + }, { "has_input": false, "has_output": false, @@ -146,7 +203,7 @@ }, "metric_keys": [], "output": null, - "span_id": "" + "span_id": "" }, { "has_input": true, @@ -171,7 +228,7 @@ "tool_call_count": 0, "type": "choices" }, - "span_id": "" + "span_id": "" }, { "has_input": false, @@ -182,7 +239,7 @@ }, "metric_keys": [], "output": null, - "span_id": "" + "span_id": "" }, { "has_input": true, @@ -207,7 +264,7 @@ "tool_call_count": 0, "type": "choices" }, - "span_id": "" + "span_id": "" }, { "has_input": false, @@ -218,7 +275,7 @@ }, "metric_keys": [], "output": null, - "span_id": "" + "span_id": "" }, { "has_input": true, @@ -249,7 +306,7 @@ "tool_call_count": 0, "type": "choices" }, - "span_id": "" + "span_id": "" }, { "has_input": false, @@ -260,7 +317,7 @@ }, "metric_keys": [], "output": null, - "span_id": "" + "span_id": "" }, { "has_input": true, @@ -291,7 +348,7 @@ "tool_call_count": 0, "type": "choices" }, - "span_id": "" + "span_id": "" }, { "has_input": false, @@ -302,7 +359,7 @@ }, "metric_keys": [], "output": null, - "span_id": "" + "span_id": "" }, { "has_input": true, @@ -321,6 +378,6 @@ "embedding_length": "", "type": "embedding" }, - "span_id": "" + "span_id": "" } ] diff --git a/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v2.span-events.json b/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v2.span-events.json index 85a4050a1..c07a2170e 100644 --- a/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v2.span-events.json +++ b/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v2.span-events.json @@ -123,6 +123,58 @@ ], "type": "llm" }, + { + "has_input": true, + "has_output": true, + "metadata": { + "model": "mistral-small-latest", + "provider": "mistral" + }, + "metric_keys": [ + "completion_tokens", + "prompt_cached_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "name": "mistral.chat.complete", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": "llm" + }, + { + "has_input": true, + "has_output": false, + "metadata": { + "provider": "mistral" + }, + "metric_keys": [], + "name": "mistral.tool", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": "tool" + }, + { + "has_input": true, + "has_output": false, + "metadata": { + "provider": "mistral" + }, + "metric_keys": [], + "name": "mistral.tool", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": "tool" + }, { "has_input": false, "has_output": false, @@ -132,7 +184,7 @@ "metric_keys": [], "name": "mistral-fim-complete-operation", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ "" ], @@ -154,9 +206,9 @@ ], "name": "mistral.fim.complete", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ - "" + "" ], "type": "llm" }, @@ -169,7 +221,7 @@ "metric_keys": [], "name": "mistral-fim-stream-operation", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ "" ], @@ -191,9 +243,9 @@ ], "name": "mistral.fim.stream", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ - "" + "" ], "type": "llm" }, @@ -206,7 +258,7 @@ "metric_keys": [], "name": "mistral-agents-complete-operation", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ "" ], @@ -228,9 +280,9 @@ ], "name": "mistral.agents.complete", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ - "" + "" ], "type": "llm" }, @@ -243,7 +295,7 @@ "metric_keys": [], "name": "mistral-agents-stream-operation", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ "" ], @@ -265,9 +317,9 @@ ], "name": "mistral.agents.stream", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ - "" + "" ], "type": "llm" }, @@ -280,7 +332,7 @@ "metric_keys": [], "name": "mistral-embeddings-operation", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ "" ], @@ -300,9 +352,9 @@ ], "name": "mistral.embeddings.create", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ - "" + "" ], "type": "llm" } diff --git a/e2e/scenarios/mistral-instrumentation/assertions.ts b/e2e/scenarios/mistral-instrumentation/assertions.ts index 79ea578a8..c5dacf1de 100644 --- a/e2e/scenarios/mistral-instrumentation/assertions.ts +++ b/e2e/scenarios/mistral-instrumentation/assertions.ts @@ -81,7 +81,10 @@ function pickMetadata( return Object.keys(picked).length > 0 ? (picked as Json) : null; } -function summarizeInput(input: unknown): Json { +function summarizeInput( + input: unknown, + options?: { omitObjectKeys?: boolean }, +): Json { if (Array.isArray(input)) { const roles = input .map((message) => @@ -106,6 +109,12 @@ function summarizeInput(input: unknown): Json { return null; } + if (options?.omitObjectKeys) { + return { + type: "object", + }; + } + return { keys: Object.keys(input).sort(), type: "object", @@ -167,18 +176,24 @@ function summarizeOutput(output: unknown): Json { }; } -function summarizePayloadRow(row: CapturedLogRow): Json { +function summarizePayloadRow( + row: CapturedLogRow, + options?: { spanName?: string }, +): Json { const metrics = typeof row.metrics === "object" && row.metrics !== null && !Array.isArray(row.metrics) ? (row.metrics as Record) : {}; + const omitObjectKeysForInput = options?.spanName === "mistral.tool"; return { has_input: row.input !== undefined && row.input !== null, has_output: row.output !== undefined && row.output !== null, - input: summarizeInput(row.input), + input: summarizeInput(row.input, { + omitObjectKeys: omitObjectKeysForInput, + }), metadata: pickMetadata( row.metadata as Record | undefined, ["model", "operation", "provider", "scenario"], @@ -325,6 +340,14 @@ function buildSpanSummary( events, "mistral-chat-tool-call-operation", ); + const chatToolCallSpans = findChildSpans( + events, + "mistral.chat.complete", + chatToolCallOperation?.span.id, + ); + const chatToolSpans = chatToolCallSpans.flatMap((chatToolCallSpan) => + findChildSpans(events, "mistral.tool", chatToolCallSpan.span.id), + ); const fimCompleteOperation = findLatestSpan( events, "mistral-fim-complete-operation", @@ -358,9 +381,8 @@ function buildSpanSummary( "mistral.chat.stream", ]), chatToolCallOperation, - findMistralSpan(events, chatToolCallOperation?.span.id, [ - "mistral.chat.complete", - ]), + ...chatToolCallSpans, + ...chatToolSpans, fimCompleteOperation, findMistralSpan(events, fimCompleteOperation?.span.id, [ "mistral.fim.complete", @@ -414,15 +436,18 @@ function buildPayloadSummary( const payloadRows = payloadRowsForRootSpan(payloads, root?.span.id); const mergedRows = mergePayloadRows(payloadRows); return normalizeForSnapshot( - mergedRows.map((row) => - normalizeLegacyV134PayloadSummaryRow( - summarizePayloadRow(row), - snapshotName, + mergedRows.map((row) => { + const spanName = typeof row.span_id === "string" ? spanNameById.get(row.span_id) - : undefined, - ), - ), + : undefined; + + return normalizeLegacyV134PayloadSummaryRow( + summarizePayloadRow(row, { spanName }), + snapshotName, + spanName, + ); + }), ); } @@ -512,43 +537,95 @@ export function defineMistralInstrumentationAssertions(options: { events, "mistral-chat-tool-call-operation", ); - const span = findMistralSpan(events, operation?.span.id, [ + const spans = findChildSpans( + events, "mistral.chat.complete", - ]); - const output = span?.output as - | Array<{ - finishReason?: unknown; - finish_reason?: unknown; - message?: { - toolCalls?: unknown; - tool_calls?: unknown; - }; - }> - | undefined; - const firstChoice = Array.isArray(output) ? output[0] : undefined; - const toolCalls = - (Array.isArray(firstChoice?.message?.tool_calls) && - firstChoice.message.tool_calls) || - (Array.isArray(firstChoice?.message?.toolCalls) && - firstChoice.message.toolCalls) || - []; - const finishReason = - typeof firstChoice?.finishReason === "string" - ? firstChoice.finishReason - : typeof firstChoice?.finish_reason === "string" - ? firstChoice.finish_reason - : undefined; + operation?.span.id, + ); + const spanIds = new Set(spans.map((span) => span.span.id)); + const toolSpans = spans.flatMap((span) => + findChildSpans(events, "mistral.tool", span.span.id), + ); + const spansWithToolCalls = spans.filter((span) => { + const output = span.output as + | Array<{ + message?: { + toolCalls?: unknown; + tool_calls?: unknown; + }; + }> + | undefined; + const firstChoice = Array.isArray(output) ? output[0] : undefined; + const toolCalls = + (Array.isArray(firstChoice?.message?.tool_calls) && + firstChoice.message.tool_calls) || + (Array.isArray(firstChoice?.message?.toolCalls) && + firstChoice.message.toolCalls) || + []; + return toolCalls.length > 0; + }); + const finishReasons = spans + .map((span) => { + const output = span.output as + | Array<{ + finishReason?: unknown; + finish_reason?: unknown; + }> + | undefined; + const firstChoice = Array.isArray(output) ? output[0] : undefined; + if (typeof firstChoice?.finishReason === "string") { + return firstChoice.finishReason; + } + if (typeof firstChoice?.finish_reason === "string") { + return firstChoice.finish_reason; + } + return undefined; + }) + .filter((value): value is string => typeof value === "string"); + const toolParentIds = new Set( + toolSpans.flatMap((toolSpan) => + toolSpan.span.parentIds.filter((parentId) => spanIds.has(parentId)), + ), + ); + const toolNames = new Set( + toolSpans + .map( + (toolSpan) => + (toolSpan.row.metadata as { tool_name?: unknown } | undefined) + ?.tool_name, + ) + .filter((name): name is string => typeof name === "string"), + ); expect(operation).toBeDefined(); - expect(span).toBeDefined(); expect(operation?.span.parentIds).toEqual([root?.span.id ?? ""]); - expect(span?.span.type).toBe("llm"); - expect(span?.row.metadata).toMatchObject({ - model: CHAT_MODEL, - provider: "mistral", - }); - expect(toolCalls.length).toBeGreaterThan(0); - expect(finishReason).toEqual(expect.any(String)); + expect(spans.length).toBeGreaterThanOrEqual(2); + expect(spansWithToolCalls.length).toBeGreaterThanOrEqual(2); + expect(finishReasons.length).toBeGreaterThanOrEqual(2); + + for (const span of spans) { + expect(span.span.type).toBe("llm"); + expect(span.row.metadata).toMatchObject({ + model: CHAT_MODEL, + provider: "mistral", + }); + } + + expect(toolSpans.length).toBeGreaterThanOrEqual(2); + expect(toolParentIds.size).toBeGreaterThanOrEqual(2); + for (const toolSpan of toolSpans) { + expect(toolSpan.span.type).toBe("tool"); + expect(toolSpan.row.metadata).toMatchObject({ + provider: "mistral", + tool_name: expect.any(String), + }); + expect( + toolSpan.span.parentIds.some((parentId) => spanIds.has(parentId)), + ).toBe(true); + expect(toolSpan.input).toBeDefined(); + } + expect(toolNames.has("get_weather")).toBe(true); + expect(toolNames.has("get_exchange_rate")).toBe(true); }); test("captures trace for fim.complete()", testConfig, () => { diff --git a/e2e/scenarios/mistral-instrumentation/scenario.impl.mjs b/e2e/scenarios/mistral-instrumentation/scenario.impl.mjs index 78c076a76..53044efd4 100644 --- a/e2e/scenarios/mistral-instrumentation/scenario.impl.mjs +++ b/e2e/scenarios/mistral-instrumentation/scenario.impl.mjs @@ -329,7 +329,7 @@ async function runMistralInstrumentationScenario( { role: "user", content: - "Call the get_weather tool. Do not answer with plain text.", + "Call the get_weather tool for Vienna. Do not answer with plain text.", }, ], tools: [ @@ -348,6 +348,34 @@ async function runMistralInstrumentationScenario( }), { attempts: 3, delayMs: 1_000 }, ); + + await withRetry( + async () => + client.chat.complete({ + model: CHAT_MODEL, + messages: [ + { + role: "user", + content: + "Call the get_exchange_rate tool for USD to EUR. Do not answer with plain text.", + }, + ], + tools: [ + { + type: "function", + function: { + name: "get_exchange_rate", + description: "Get currency exchange rate.", + parameters: {}, + }, + }, + ], + toolChoice: "required", + maxTokens: 48, + temperature: 0, + }), + { attempts: 3, delayMs: 1_000 }, + ); }, ); diff --git a/js/src/instrumentation/core/channel-tracing.ts b/js/src/instrumentation/core/channel-tracing.ts index 9abb30c36..87e04109f 100644 --- a/js/src/instrumentation/core/channel-tracing.ts +++ b/js/src/instrumentation/core/channel-tracing.ts @@ -104,6 +104,17 @@ export type StreamingChannelSpanConfig = span: Span; startTime: number; }) => boolean; + onComplete?: (args: { + channelName: string; + chunks?: ChunkOf[]; + endEvent: AsyncEndOf; + metadata?: Record; + metrics: Record; + output: unknown; + result: StreamingResult; + span: Span; + startTime: number; + }) => void; }; export type SyncStreamChannelSpanConfig = @@ -295,6 +306,40 @@ function logErrorAndEnd< states.delete(event as object); } +function runStreamingCompletionHook(args: { + channelName: string; + config: StreamingChannelSpanConfig; + chunks?: ChunkOf[]; + endEvent: AsyncEndOf; + metadata?: Record; + metrics: Record; + output: unknown; + result: StreamingResult; + span: Span; + startTime: number; +}): void { + if (!args.config.onComplete) { + return; + } + + try { + args.config.onComplete({ + channelName: args.channelName, + ...(args.chunks ? { chunks: args.chunks } : {}), + endEvent: args.endEvent, + ...(args.metadata !== undefined ? { metadata: args.metadata } : {}), + metrics: args.metrics, + output: args.output, + result: args.result, + span: args.span, + startTime: args.startTime, + }); + } catch (error) { + // eslint-disable-next-line no-restricted-properties -- preserving intentional console usage. + console.error(`Error in onComplete hook for ${args.channelName}:`, error); + } +} + export function traceAsyncChannel( channel: TChannel, config: AsyncChannelSpanConfig, @@ -456,6 +501,19 @@ export function traceStreamingChannel( getCurrentUnixTimestamp() - startTime; } + runStreamingCompletionHook({ + channelName, + chunks, + config, + endEvent: asyncEndEvent, + ...(metadata !== undefined ? { metadata } : {}), + metrics, + output, + result: asyncEndEvent.result as StreamingResult, + span, + startTime, + }); + span.log({ output, ...(metadata !== undefined ? { metadata } : {}), @@ -511,6 +569,20 @@ export function traceStreamingChannel( asyncEndEvent, ); + runStreamingCompletionHook({ + channelName, + config, + endEvent: asyncEndEvent, + ...(normalizeMetadata(metadata) !== undefined + ? { metadata: normalizeMetadata(metadata) } + : {}), + metrics, + output, + result: asyncEndEvent.result as StreamingResult, + span, + startTime, + }); + span.log({ output, ...(normalizeMetadata(metadata) !== undefined diff --git a/js/src/instrumentation/plugins/mistral-plugin.test.ts b/js/src/instrumentation/plugins/mistral-plugin.test.ts index e73d63076..bb0a1d2a0 100644 --- a/js/src/instrumentation/plugins/mistral-plugin.test.ts +++ b/js/src/instrumentation/plugins/mistral-plugin.test.ts @@ -3,6 +3,7 @@ import { aggregateMistralStreamChunks, extractMistralRequestMetadata, extractMistralResponseMetadata, + extractMistralToolCallsFromOutput, parseMistralMetricsFromUsage, } from "./mistral-plugin"; @@ -69,6 +70,65 @@ describe("extractMistralResponseMetadata", () => { }); }); +describe("extractMistralToolCallsFromOutput", () => { + it("extracts tool calls from snake_case and camelCase output", () => { + expect( + extractMistralToolCallsFromOutput([ + { + index: 0, + message: { + tool_calls: [ + { + id: "call_1", + index: 0, + type: "function", + function: { + name: "lookup_weather", + arguments: '{"city":"Vienna"}', + }, + }, + ], + }, + }, + { + index: 1, + message: { + toolCalls: [ + { + id: "call_2", + function: { + name: "lookup_time", + arguments: '{"timezone":"Europe/Vienna"}', + }, + }, + ], + }, + }, + ]), + ).toEqual([ + { + choiceIndex: 0, + id: "call_1", + index: 0, + type: "function", + name: "lookup_weather", + arguments: '{"city":"Vienna"}', + }, + { + choiceIndex: 1, + id: "call_2", + name: "lookup_time", + arguments: '{"timezone":"Europe/Vienna"}', + }, + ]); + }); + + it("returns empty list for non-choice output", () => { + expect(extractMistralToolCallsFromOutput(undefined)).toEqual([]); + expect(extractMistralToolCallsFromOutput({})).toEqual([]); + }); +}); + describe("parseMistralMetricsFromUsage", () => { it("returns empty metrics for missing usage", () => { expect(parseMistralMetricsFromUsage(undefined)).toEqual({}); diff --git a/js/src/instrumentation/plugins/mistral-plugin.ts b/js/src/instrumentation/plugins/mistral-plugin.ts index 88ae25f46..8548218f4 100644 --- a/js/src/instrumentation/plugins/mistral-plugin.ts +++ b/js/src/instrumentation/plugins/mistral-plugin.ts @@ -5,6 +5,7 @@ import { unsubscribeAll, } from "../core/channel-tracing"; import { SpanTypeAttribute, isObject } from "../../../util/index"; +import type { Span } from "../../logger"; import { processInputAttachments } from "../../wrappers/attachment-utils"; import { getCurrentUnixTimestamp } from "../../util"; import { mistralChannels } from "./mistral-channels"; @@ -37,6 +38,11 @@ export class MistralPlugin extends BasePlugin { extractMetadata: (result) => extractMistralResponseMetadata(result), extractMetrics: (result, startTime) => extractMistralMetrics(result?.usage, startTime), + onComplete: ({ output, result, span }) => + createMistralToolCallSpans(span, { + output, + result, + }), }), ); @@ -50,6 +56,11 @@ export class MistralPlugin extends BasePlugin { extractMetrics: (result, startTime) => extractMistralStreamingMetrics(result, startTime), aggregateChunks: aggregateMistralStreamChunks, + onComplete: ({ output, result, span }) => + createMistralToolCallSpans(span, { + output, + result, + }), }), ); @@ -80,6 +91,11 @@ export class MistralPlugin extends BasePlugin { extractMetadata: (result) => extractMistralResponseMetadata(result), extractMetrics: (result, startTime) => extractMistralMetrics(result?.usage, startTime), + onComplete: ({ output, result, span }) => + createMistralToolCallSpans(span, { + output, + result, + }), }), ); @@ -107,6 +123,11 @@ export class MistralPlugin extends BasePlugin { extractMetadata: (result) => extractMistralResponseMetadata(result), extractMetrics: (result, startTime) => extractMistralMetrics(result?.usage, startTime), + onComplete: ({ output, result, span }) => + createMistralToolCallSpans(span, { + output, + result, + }), }), ); @@ -120,6 +141,11 @@ export class MistralPlugin extends BasePlugin { extractMetrics: (result, startTime) => extractMistralStreamingMetrics(result, startTime), aggregateChunks: aggregateMistralStreamChunks, + onComplete: ({ output, result, span }) => + createMistralToolCallSpans(span, { + output, + result, + }), }), ); } @@ -342,6 +368,142 @@ function extractMistralStreamOutput(result: unknown): unknown { return isObject(result) ? result.choices : undefined; } +function getMessageToolCalls( + message: Record | undefined, +): Record[] { + if (!message) { + return []; + } + + const rawToolCalls = + (Array.isArray(message.toolCalls) && message.toolCalls) || + (Array.isArray(message.tool_calls) && message.tool_calls) || + []; + + return rawToolCalls.filter((toolCall) => isObject(toolCall)); +} + +type MistralExtractedToolCall = { + arguments?: string; + choiceIndex: number; + id?: string; + index?: number; + name?: string; + type?: string; +}; + +export function extractMistralToolCallsFromOutput( + output: unknown, +): MistralExtractedToolCall[] { + const choices = Array.isArray(output) + ? output + : isArrayLike(output) + ? Array.from(output) + : undefined; + + if (!choices) { + return []; + } + + const extracted: MistralExtractedToolCall[] = []; + + for (const rawChoice of choices) { + if (!isObject(rawChoice) || !isObject(rawChoice.message)) { + continue; + } + + const choiceIndex = + typeof rawChoice.index === "number" && rawChoice.index >= 0 + ? rawChoice.index + : 0; + const toolCalls = getMessageToolCalls(rawChoice.message); + + for (const toolCall of toolCalls) { + const toolFunction = isObject(toolCall.function) ? toolCall.function : {}; + extracted.push({ + ...(typeof toolCall.id === "string" ? { id: toolCall.id } : {}), + ...(typeof toolCall.index === "number" + ? { index: toolCall.index } + : {}), + ...(typeof toolCall.type === "string" ? { type: toolCall.type } : {}), + ...(typeof toolFunction.name === "string" + ? { name: toolFunction.name } + : {}), + ...(typeof toolFunction.arguments === "string" + ? { arguments: toolFunction.arguments } + : {}), + choiceIndex, + }); + } + } + + return extracted; +} + +function parseToolArguments(argumentsValue: string | undefined): unknown { + if (typeof argumentsValue !== "string") { + return undefined; + } + + try { + return JSON.parse(argumentsValue); + } catch { + return argumentsValue; + } +} + +function createMistralToolCallSpansFromOutput( + parentSpan: Span, + output: unknown, +): void { + const toolCalls = extractMistralToolCallsFromOutput(output); + for (const toolCall of toolCalls) { + const parsedToolArguments = parseToolArguments(toolCall.arguments); + const toolSpan = parentSpan.startSpan({ + name: "mistral.tool", + spanAttributes: { + type: SpanTypeAttribute.TOOL, + }, + }); + + toolSpan.log({ + ...(parsedToolArguments !== undefined + ? { input: parsedToolArguments } + : {}), + metadata: { + provider: "mistral", + ...(toolCall.name ? { tool_name: toolCall.name } : {}), + ...(toolCall.id ? { tool_call_id: toolCall.id } : {}), + ...(toolCall.index !== undefined ? { tool_index: toolCall.index } : {}), + ...(toolCall.type ? { tool_type: toolCall.type } : {}), + choice_index: toolCall.choiceIndex, + }, + }); + toolSpan.end(); + } +} + +function createMistralToolCallSpans( + parentSpan: Span, + args: { + output: unknown; + result: unknown; + }, +): void { + const outputToolCalls = extractMistralToolCallsFromOutput(args.output); + if (outputToolCalls.length > 0) { + createMistralToolCallSpansFromOutput(parentSpan, args.output); + return; + } + + const resultToolCalls = isObject(args.result) + ? extractMistralToolCallsFromOutput(args.result.choices) + : []; + if (resultToolCalls.length > 0) { + createMistralToolCallSpansFromOutput(parentSpan, args.result.choices); + } +} + function extractMistralStreamingMetrics( result: unknown, startTime?: number, From e4b21bf126132e9a2f285a05a77bbe2e1f6ca7cc Mon Sep 17 00:00:00 2001 From: Luca Forstner Date: Thu, 2 Apr 2026 23:15:16 +0200 Subject: [PATCH 08/12] fix --- js/src/instrumentation/plugins/mistral-plugin.ts | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/js/src/instrumentation/plugins/mistral-plugin.ts b/js/src/instrumentation/plugins/mistral-plugin.ts index 8548218f4..81cb41dee 100644 --- a/js/src/instrumentation/plugins/mistral-plugin.ts +++ b/js/src/instrumentation/plugins/mistral-plugin.ts @@ -496,11 +496,14 @@ function createMistralToolCallSpans( return; } - const resultToolCalls = isObject(args.result) - ? extractMistralToolCallsFromOutput(args.result.choices) - : []; + if (!isObject(args.result)) { + return; + } + + const resultChoices = args.result.choices; + const resultToolCalls = extractMistralToolCallsFromOutput(resultChoices); if (resultToolCalls.length > 0) { - createMistralToolCallSpansFromOutput(parentSpan, args.result.choices); + createMistralToolCallSpansFromOutput(parentSpan, resultChoices); } } From 161c6f6aa153ff4e17d51edf5b0d3674ad11bf0d Mon Sep 17 00:00:00 2001 From: Luca Forstner Date: Fri, 3 Apr 2026 10:45:50 +0200 Subject: [PATCH 09/12] Fix tool calls? --- .../mistral-v1-10-0.log-payloads.json | 62 +++++- .../mistral-v1-10-0.span-events.json | 63 +++++- .../mistral-v1-14-1.log-payloads.json | 62 +++++- .../mistral-v1-14-1.span-events.json | 63 +++++- .../mistral-v1-15-1.log-payloads.json | 62 +++++- .../mistral-v1-15-1.span-events.json | 63 +++++- .../mistral-v1-3-4.log-payloads.json | 135 ++++++++++-- .../mistral-v1-3-4.span-events.json | 140 +++++++++++-- .../mistral-v1.log-payloads.json | 62 +++++- .../__snapshots__/mistral-v1.span-events.json | 63 +++++- .../mistral-v2.log-payloads.json | 63 +++++- .../__snapshots__/mistral-v2.span-events.json | 64 +++++- .../mistral-instrumentation/assertions.ts | 85 ++++++++ .../mistral-instrumentation/scenario.impl.mjs | 196 +++++++++++++++--- .../plugins/mistral-plugin.test.ts | 22 ++ .../instrumentation/plugins/mistral-plugin.ts | 70 ++++++- 16 files changed, 1158 insertions(+), 117 deletions(-) diff --git a/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-10-0.log-payloads.json b/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-10-0.log-payloads.json index a5ff90d87..5600434bb 100644 --- a/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-10-0.log-payloads.json +++ b/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-10-0.log-payloads.json @@ -306,12 +306,66 @@ "has_output": false, "input": null, "metadata": { - "operation": "agents-stream" + "operation": "agents-tool-call" }, "metric_keys": [], "output": null, "span_id": "" }, + { + "has_input": true, + "has_output": true, + "input": { + "item_count": 1, + "roles": [ + "user" + ], + "type": "messages" + }, + "metadata": { + "model": "mistral-small-latest", + "provider": "mistral" + }, + "metric_keys": [ + "completion_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "output": { + "choice_count": 1, + "finish_reason": "tool_calls", + "has_content": false, + "role": "assistant", + "tool_call_count": 1, + "type": "choices" + }, + "span_id": "" + }, + { + "has_input": true, + "has_output": false, + "input": { + "type": "object" + }, + "metadata": { + "provider": "mistral" + }, + "metric_keys": [], + "output": null, + "span_id": "" + }, + { + "has_input": false, + "has_output": false, + "input": null, + "metadata": { + "operation": "agents-stream" + }, + "metric_keys": [], + "output": null, + "span_id": "" + }, { "has_input": true, "has_output": true, @@ -340,7 +394,7 @@ "tool_call_count": 0, "type": "choices" }, - "span_id": "" + "span_id": "" }, { "has_input": false, @@ -351,7 +405,7 @@ }, "metric_keys": [], "output": null, - "span_id": "" + "span_id": "" }, { "has_input": true, @@ -370,6 +424,6 @@ "embedding_length": "", "type": "embedding" }, - "span_id": "" + "span_id": "" } ] diff --git a/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-10-0.span-events.json b/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-10-0.span-events.json index cd67b9364..749d92a35 100644 --- a/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-10-0.span-events.json +++ b/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-10-0.span-events.json @@ -283,10 +283,10 @@ "has_input": false, "has_output": false, "metadata": { - "operation": "agents-stream" + "operation": "agents-tool-call" }, "metric_keys": [], - "name": "mistral-agents-stream-operation", + "name": "mistral-agents-tool-call-operation", "root_span_id": "", "span_id": "", "span_parents": [ @@ -307,7 +307,7 @@ "time_to_first_token", "tokens" ], - "name": "mistral.agents.stream", + "name": "mistral.agents.complete", "root_span_id": "", "span_id": "", "span_parents": [ @@ -315,6 +315,57 @@ ], "type": "llm" }, + { + "has_input": true, + "has_output": false, + "metadata": { + "provider": "mistral" + }, + "metric_keys": [], + "name": "mistral.tool", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": "tool" + }, + { + "has_input": false, + "has_output": false, + "metadata": { + "operation": "agents-stream" + }, + "metric_keys": [], + "name": "mistral-agents-stream-operation", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": null + }, + { + "has_input": true, + "has_output": true, + "metadata": { + "model": "mistral-small-latest", + "provider": "mistral" + }, + "metric_keys": [ + "completion_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "name": "mistral.agents.stream", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": "llm" + }, { "has_input": false, "has_output": false, @@ -324,7 +375,7 @@ "metric_keys": [], "name": "mistral-embeddings-operation", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ "" ], @@ -344,9 +395,9 @@ ], "name": "mistral.embeddings.create", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ - "" + "" ], "type": "llm" } diff --git a/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-14-1.log-payloads.json b/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-14-1.log-payloads.json index a5ff90d87..5600434bb 100644 --- a/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-14-1.log-payloads.json +++ b/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-14-1.log-payloads.json @@ -306,12 +306,66 @@ "has_output": false, "input": null, "metadata": { - "operation": "agents-stream" + "operation": "agents-tool-call" }, "metric_keys": [], "output": null, "span_id": "" }, + { + "has_input": true, + "has_output": true, + "input": { + "item_count": 1, + "roles": [ + "user" + ], + "type": "messages" + }, + "metadata": { + "model": "mistral-small-latest", + "provider": "mistral" + }, + "metric_keys": [ + "completion_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "output": { + "choice_count": 1, + "finish_reason": "tool_calls", + "has_content": false, + "role": "assistant", + "tool_call_count": 1, + "type": "choices" + }, + "span_id": "" + }, + { + "has_input": true, + "has_output": false, + "input": { + "type": "object" + }, + "metadata": { + "provider": "mistral" + }, + "metric_keys": [], + "output": null, + "span_id": "" + }, + { + "has_input": false, + "has_output": false, + "input": null, + "metadata": { + "operation": "agents-stream" + }, + "metric_keys": [], + "output": null, + "span_id": "" + }, { "has_input": true, "has_output": true, @@ -340,7 +394,7 @@ "tool_call_count": 0, "type": "choices" }, - "span_id": "" + "span_id": "" }, { "has_input": false, @@ -351,7 +405,7 @@ }, "metric_keys": [], "output": null, - "span_id": "" + "span_id": "" }, { "has_input": true, @@ -370,6 +424,6 @@ "embedding_length": "", "type": "embedding" }, - "span_id": "" + "span_id": "" } ] diff --git a/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-14-1.span-events.json b/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-14-1.span-events.json index cd67b9364..749d92a35 100644 --- a/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-14-1.span-events.json +++ b/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-14-1.span-events.json @@ -283,10 +283,10 @@ "has_input": false, "has_output": false, "metadata": { - "operation": "agents-stream" + "operation": "agents-tool-call" }, "metric_keys": [], - "name": "mistral-agents-stream-operation", + "name": "mistral-agents-tool-call-operation", "root_span_id": "", "span_id": "", "span_parents": [ @@ -307,7 +307,7 @@ "time_to_first_token", "tokens" ], - "name": "mistral.agents.stream", + "name": "mistral.agents.complete", "root_span_id": "", "span_id": "", "span_parents": [ @@ -315,6 +315,57 @@ ], "type": "llm" }, + { + "has_input": true, + "has_output": false, + "metadata": { + "provider": "mistral" + }, + "metric_keys": [], + "name": "mistral.tool", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": "tool" + }, + { + "has_input": false, + "has_output": false, + "metadata": { + "operation": "agents-stream" + }, + "metric_keys": [], + "name": "mistral-agents-stream-operation", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": null + }, + { + "has_input": true, + "has_output": true, + "metadata": { + "model": "mistral-small-latest", + "provider": "mistral" + }, + "metric_keys": [ + "completion_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "name": "mistral.agents.stream", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": "llm" + }, { "has_input": false, "has_output": false, @@ -324,7 +375,7 @@ "metric_keys": [], "name": "mistral-embeddings-operation", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ "" ], @@ -344,9 +395,9 @@ ], "name": "mistral.embeddings.create", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ - "" + "" ], "type": "llm" } diff --git a/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-15-1.log-payloads.json b/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-15-1.log-payloads.json index a5ff90d87..5600434bb 100644 --- a/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-15-1.log-payloads.json +++ b/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-15-1.log-payloads.json @@ -306,12 +306,66 @@ "has_output": false, "input": null, "metadata": { - "operation": "agents-stream" + "operation": "agents-tool-call" }, "metric_keys": [], "output": null, "span_id": "" }, + { + "has_input": true, + "has_output": true, + "input": { + "item_count": 1, + "roles": [ + "user" + ], + "type": "messages" + }, + "metadata": { + "model": "mistral-small-latest", + "provider": "mistral" + }, + "metric_keys": [ + "completion_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "output": { + "choice_count": 1, + "finish_reason": "tool_calls", + "has_content": false, + "role": "assistant", + "tool_call_count": 1, + "type": "choices" + }, + "span_id": "" + }, + { + "has_input": true, + "has_output": false, + "input": { + "type": "object" + }, + "metadata": { + "provider": "mistral" + }, + "metric_keys": [], + "output": null, + "span_id": "" + }, + { + "has_input": false, + "has_output": false, + "input": null, + "metadata": { + "operation": "agents-stream" + }, + "metric_keys": [], + "output": null, + "span_id": "" + }, { "has_input": true, "has_output": true, @@ -340,7 +394,7 @@ "tool_call_count": 0, "type": "choices" }, - "span_id": "" + "span_id": "" }, { "has_input": false, @@ -351,7 +405,7 @@ }, "metric_keys": [], "output": null, - "span_id": "" + "span_id": "" }, { "has_input": true, @@ -370,6 +424,6 @@ "embedding_length": "", "type": "embedding" }, - "span_id": "" + "span_id": "" } ] diff --git a/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-15-1.span-events.json b/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-15-1.span-events.json index cd67b9364..749d92a35 100644 --- a/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-15-1.span-events.json +++ b/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-15-1.span-events.json @@ -283,10 +283,10 @@ "has_input": false, "has_output": false, "metadata": { - "operation": "agents-stream" + "operation": "agents-tool-call" }, "metric_keys": [], - "name": "mistral-agents-stream-operation", + "name": "mistral-agents-tool-call-operation", "root_span_id": "", "span_id": "", "span_parents": [ @@ -307,7 +307,7 @@ "time_to_first_token", "tokens" ], - "name": "mistral.agents.stream", + "name": "mistral.agents.complete", "root_span_id": "", "span_id": "", "span_parents": [ @@ -315,6 +315,57 @@ ], "type": "llm" }, + { + "has_input": true, + "has_output": false, + "metadata": { + "provider": "mistral" + }, + "metric_keys": [], + "name": "mistral.tool", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": "tool" + }, + { + "has_input": false, + "has_output": false, + "metadata": { + "operation": "agents-stream" + }, + "metric_keys": [], + "name": "mistral-agents-stream-operation", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": null + }, + { + "has_input": true, + "has_output": true, + "metadata": { + "model": "mistral-small-latest", + "provider": "mistral" + }, + "metric_keys": [ + "completion_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "name": "mistral.agents.stream", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": "llm" + }, { "has_input": false, "has_output": false, @@ -324,7 +375,7 @@ "metric_keys": [], "name": "mistral-embeddings-operation", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ "" ], @@ -344,9 +395,9 @@ ], "name": "mistral.embeddings.create", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ - "" + "" ], "type": "llm" } diff --git a/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-3-4.log-payloads.json b/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-3-4.log-payloads.json index 91d8993c2..0fe9355b4 100644 --- a/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-3-4.log-payloads.json +++ b/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-3-4.log-payloads.json @@ -104,6 +104,24 @@ "output": null, "span_id": "" }, + { + "has_input": true, + "has_output": false, + "input": { + "item_count": 1, + "roles": [ + "user" + ], + "type": "messages" + }, + "metadata": { + "model": "mistral-small-latest", + "provider": "mistral" + }, + "metric_keys": [], + "output": null, + "span_id": "" + }, { "has_input": true, "has_output": true, @@ -132,7 +150,7 @@ "tool_call_count": 1, "type": "choices" }, - "span_id": "" + "span_id": "" }, { "has_input": true, @@ -145,7 +163,25 @@ }, "metric_keys": [], "output": null, - "span_id": "" + "span_id": "" + }, + { + "has_input": true, + "has_output": false, + "input": { + "item_count": 1, + "roles": [ + "user" + ], + "type": "messages" + }, + "metadata": { + "model": "mistral-small-latest", + "provider": "mistral" + }, + "metric_keys": [], + "output": null, + "span_id": "" }, { "has_input": true, @@ -175,7 +211,7 @@ "tool_call_count": 1, "type": "choices" }, - "span_id": "" + "span_id": "" }, { "has_input": true, @@ -188,7 +224,7 @@ }, "metric_keys": [], "output": null, - "span_id": "" + "span_id": "" }, { "has_input": false, @@ -199,7 +235,7 @@ }, "metric_keys": [], "output": null, - "span_id": "" + "span_id": "" }, { "has_input": true, @@ -223,7 +259,7 @@ "tool_call_count": 0, "type": "choices" }, - "span_id": "" + "span_id": "" }, { "has_input": false, @@ -234,7 +270,7 @@ }, "metric_keys": [], "output": null, - "span_id": "" + "span_id": "" }, { "has_input": true, @@ -255,7 +291,7 @@ "tool_call_count": 0, "type": "choices" }, - "span_id": "" + "span_id": "" }, { "has_input": false, @@ -266,7 +302,7 @@ }, "metric_keys": [], "output": null, - "span_id": "" + "span_id": "" }, { "has_input": true, @@ -296,7 +332,78 @@ "tool_call_count": 0, "type": "choices" }, - "span_id": "" + "span_id": "" + }, + { + "has_input": false, + "has_output": false, + "input": null, + "metadata": { + "operation": "agents-tool-call" + }, + "metric_keys": [], + "output": null, + "span_id": "" + }, + { + "has_input": true, + "has_output": false, + "input": { + "item_count": 1, + "roles": [ + "user" + ], + "type": "messages" + }, + "metadata": { + "provider": "mistral" + }, + "metric_keys": [], + "output": null, + "span_id": "" + }, + { + "has_input": true, + "has_output": true, + "input": { + "item_count": 1, + "roles": [ + "user" + ], + "type": "messages" + }, + "metadata": { + "model": "mistral-small-latest", + "provider": "mistral" + }, + "metric_keys": [ + "completion_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "output": { + "choice_count": 1, + "finish_reason": "tool_calls", + "has_content": false, + "role": "assistant", + "tool_call_count": 1, + "type": "choices" + }, + "span_id": "" + }, + { + "has_input": true, + "has_output": false, + "input": { + "type": "object" + }, + "metadata": { + "provider": "mistral" + }, + "metric_keys": [], + "output": null, + "span_id": "" }, { "has_input": false, @@ -307,7 +414,7 @@ }, "metric_keys": [], "output": null, - "span_id": "" + "span_id": "" }, { "has_input": true, @@ -337,7 +444,7 @@ "tool_call_count": 0, "type": "choices" }, - "span_id": "" + "span_id": "" }, { "has_input": false, @@ -348,7 +455,7 @@ }, "metric_keys": [], "output": null, - "span_id": "" + "span_id": "" }, { "has_input": true, @@ -367,6 +474,6 @@ "embedding_length": "", "type": "embedding" }, - "span_id": "" + "span_id": "" } ] diff --git a/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-3-4.span-events.json b/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-3-4.span-events.json index 6990e605a..341ca8762 100644 --- a/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-3-4.span-events.json +++ b/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-3-4.span-events.json @@ -99,6 +99,22 @@ ], "type": null }, + { + "has_input": true, + "has_output": false, + "metadata": { + "model": "mistral-small-latest", + "provider": "mistral" + }, + "metric_keys": [], + "name": "mistral.chat.complete", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": "llm" + }, { "has_input": true, "has_output": true, @@ -114,7 +130,23 @@ ], "name": "mistral.chat.complete", "root_span_id": "", - "span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": "llm" + }, + { + "has_input": true, + "has_output": false, + "metadata": { + "model": "mistral-small-latest", + "provider": "mistral" + }, + "metric_keys": [], + "name": "mistral.chat.complete", + "root_span_id": "", + "span_id": "", "span_parents": [ "" ], @@ -135,7 +167,7 @@ ], "name": "mistral.chat.complete", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ "" ], @@ -150,9 +182,9 @@ "metric_keys": [], "name": "mistral.tool", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ - "" + "" ], "type": "tool" }, @@ -165,9 +197,9 @@ "metric_keys": [], "name": "mistral.tool", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ - "" + "" ], "type": "tool" }, @@ -180,7 +212,7 @@ "metric_keys": [], "name": "mistral-fim-complete-operation", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ "" ], @@ -201,9 +233,9 @@ ], "name": "mistral.fim.complete", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ - "" + "" ], "type": "llm" }, @@ -216,7 +248,7 @@ "metric_keys": [], "name": "mistral-fim-stream-operation", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ "" ], @@ -234,9 +266,9 @@ ], "name": "mistral.fim.stream", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ - "" + "" ], "type": "llm" }, @@ -249,7 +281,7 @@ "metric_keys": [], "name": "mistral-agents-complete-operation", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ "" ], @@ -270,12 +302,78 @@ ], "name": "mistral.agents.complete", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ - "" + "" + ], + "type": "llm" + }, + { + "has_input": false, + "has_output": false, + "metadata": { + "operation": "agents-tool-call" + }, + "metric_keys": [], + "name": "mistral-agents-tool-call-operation", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": null + }, + { + "has_input": true, + "has_output": false, + "metadata": { + "provider": "mistral" + }, + "metric_keys": [], + "name": "mistral.agents.complete", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" ], "type": "llm" }, + { + "has_input": true, + "has_output": true, + "metadata": { + "model": "mistral-small-latest", + "provider": "mistral" + }, + "metric_keys": [ + "completion_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "name": "mistral.agents.complete", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": "llm" + }, + { + "has_input": true, + "has_output": false, + "metadata": { + "provider": "mistral" + }, + "metric_keys": [], + "name": "mistral.tool", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": "tool" + }, { "has_input": false, "has_output": false, @@ -285,7 +383,7 @@ "metric_keys": [], "name": "mistral-agents-stream-operation", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ "" ], @@ -306,9 +404,9 @@ ], "name": "mistral.agents.stream", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ - "" + "" ], "type": "llm" }, @@ -321,7 +419,7 @@ "metric_keys": [], "name": "mistral-embeddings-operation", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ "" ], @@ -341,9 +439,9 @@ ], "name": "mistral.embeddings.create", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ - "" + "" ], "type": "llm" } diff --git a/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1.log-payloads.json b/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1.log-payloads.json index a5ff90d87..5600434bb 100644 --- a/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1.log-payloads.json +++ b/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1.log-payloads.json @@ -306,12 +306,66 @@ "has_output": false, "input": null, "metadata": { - "operation": "agents-stream" + "operation": "agents-tool-call" }, "metric_keys": [], "output": null, "span_id": "" }, + { + "has_input": true, + "has_output": true, + "input": { + "item_count": 1, + "roles": [ + "user" + ], + "type": "messages" + }, + "metadata": { + "model": "mistral-small-latest", + "provider": "mistral" + }, + "metric_keys": [ + "completion_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "output": { + "choice_count": 1, + "finish_reason": "tool_calls", + "has_content": false, + "role": "assistant", + "tool_call_count": 1, + "type": "choices" + }, + "span_id": "" + }, + { + "has_input": true, + "has_output": false, + "input": { + "type": "object" + }, + "metadata": { + "provider": "mistral" + }, + "metric_keys": [], + "output": null, + "span_id": "" + }, + { + "has_input": false, + "has_output": false, + "input": null, + "metadata": { + "operation": "agents-stream" + }, + "metric_keys": [], + "output": null, + "span_id": "" + }, { "has_input": true, "has_output": true, @@ -340,7 +394,7 @@ "tool_call_count": 0, "type": "choices" }, - "span_id": "" + "span_id": "" }, { "has_input": false, @@ -351,7 +405,7 @@ }, "metric_keys": [], "output": null, - "span_id": "" + "span_id": "" }, { "has_input": true, @@ -370,6 +424,6 @@ "embedding_length": "", "type": "embedding" }, - "span_id": "" + "span_id": "" } ] diff --git a/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1.span-events.json b/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1.span-events.json index cd67b9364..749d92a35 100644 --- a/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1.span-events.json +++ b/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1.span-events.json @@ -283,10 +283,10 @@ "has_input": false, "has_output": false, "metadata": { - "operation": "agents-stream" + "operation": "agents-tool-call" }, "metric_keys": [], - "name": "mistral-agents-stream-operation", + "name": "mistral-agents-tool-call-operation", "root_span_id": "", "span_id": "", "span_parents": [ @@ -307,7 +307,7 @@ "time_to_first_token", "tokens" ], - "name": "mistral.agents.stream", + "name": "mistral.agents.complete", "root_span_id": "", "span_id": "", "span_parents": [ @@ -315,6 +315,57 @@ ], "type": "llm" }, + { + "has_input": true, + "has_output": false, + "metadata": { + "provider": "mistral" + }, + "metric_keys": [], + "name": "mistral.tool", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": "tool" + }, + { + "has_input": false, + "has_output": false, + "metadata": { + "operation": "agents-stream" + }, + "metric_keys": [], + "name": "mistral-agents-stream-operation", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": null + }, + { + "has_input": true, + "has_output": true, + "metadata": { + "model": "mistral-small-latest", + "provider": "mistral" + }, + "metric_keys": [ + "completion_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "name": "mistral.agents.stream", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": "llm" + }, { "has_input": false, "has_output": false, @@ -324,7 +375,7 @@ "metric_keys": [], "name": "mistral-embeddings-operation", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ "" ], @@ -344,9 +395,9 @@ ], "name": "mistral.embeddings.create", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ - "" + "" ], "type": "llm" } diff --git a/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v2.log-payloads.json b/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v2.log-payloads.json index 4bb90778e..31b8bb5b4 100644 --- a/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v2.log-payloads.json +++ b/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v2.log-payloads.json @@ -313,12 +313,67 @@ "has_output": false, "input": null, "metadata": { - "operation": "agents-stream" + "operation": "agents-tool-call" }, "metric_keys": [], "output": null, "span_id": "" }, + { + "has_input": true, + "has_output": true, + "input": { + "item_count": 1, + "roles": [ + "user" + ], + "type": "messages" + }, + "metadata": { + "model": "mistral-small-latest", + "provider": "mistral" + }, + "metric_keys": [ + "completion_tokens", + "prompt_cached_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "output": { + "choice_count": 1, + "finish_reason": "tool_calls", + "has_content": false, + "role": "assistant", + "tool_call_count": 1, + "type": "choices" + }, + "span_id": "" + }, + { + "has_input": true, + "has_output": false, + "input": { + "type": "object" + }, + "metadata": { + "provider": "mistral" + }, + "metric_keys": [], + "output": null, + "span_id": "" + }, + { + "has_input": false, + "has_output": false, + "input": null, + "metadata": { + "operation": "agents-stream" + }, + "metric_keys": [], + "output": null, + "span_id": "" + }, { "has_input": true, "has_output": true, @@ -348,7 +403,7 @@ "tool_call_count": 0, "type": "choices" }, - "span_id": "" + "span_id": "" }, { "has_input": false, @@ -359,7 +414,7 @@ }, "metric_keys": [], "output": null, - "span_id": "" + "span_id": "" }, { "has_input": true, @@ -378,6 +433,6 @@ "embedding_length": "", "type": "embedding" }, - "span_id": "" + "span_id": "" } ] diff --git a/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v2.span-events.json b/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v2.span-events.json index c07a2170e..6170b4952 100644 --- a/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v2.span-events.json +++ b/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v2.span-events.json @@ -290,10 +290,10 @@ "has_input": false, "has_output": false, "metadata": { - "operation": "agents-stream" + "operation": "agents-tool-call" }, "metric_keys": [], - "name": "mistral-agents-stream-operation", + "name": "mistral-agents-tool-call-operation", "root_span_id": "", "span_id": "", "span_parents": [ @@ -315,7 +315,7 @@ "time_to_first_token", "tokens" ], - "name": "mistral.agents.stream", + "name": "mistral.agents.complete", "root_span_id": "", "span_id": "", "span_parents": [ @@ -323,6 +323,58 @@ ], "type": "llm" }, + { + "has_input": true, + "has_output": false, + "metadata": { + "provider": "mistral" + }, + "metric_keys": [], + "name": "mistral.tool", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": "tool" + }, + { + "has_input": false, + "has_output": false, + "metadata": { + "operation": "agents-stream" + }, + "metric_keys": [], + "name": "mistral-agents-stream-operation", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": null + }, + { + "has_input": true, + "has_output": true, + "metadata": { + "model": "mistral-small-latest", + "provider": "mistral" + }, + "metric_keys": [ + "completion_tokens", + "prompt_cached_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "name": "mistral.agents.stream", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": "llm" + }, { "has_input": false, "has_output": false, @@ -332,7 +384,7 @@ "metric_keys": [], "name": "mistral-embeddings-operation", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ "" ], @@ -352,9 +404,9 @@ ], "name": "mistral.embeddings.create", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ - "" + "" ], "type": "llm" } diff --git a/e2e/scenarios/mistral-instrumentation/assertions.ts b/e2e/scenarios/mistral-instrumentation/assertions.ts index c5dacf1de..c4925ca0b 100644 --- a/e2e/scenarios/mistral-instrumentation/assertions.ts +++ b/e2e/scenarios/mistral-instrumentation/assertions.ts @@ -360,6 +360,18 @@ function buildSpanSummary( events, "mistral-agents-complete-operation", ); + const agentsToolCallOperation = findLatestSpan( + events, + "mistral-agents-tool-call-operation", + ); + const agentsToolCallSpans = findChildSpans( + events, + "mistral.agents.complete", + agentsToolCallOperation?.span.id, + ); + const agentsToolSpans = agentsToolCallSpans.flatMap((agentsToolCallSpan) => + findChildSpans(events, "mistral.tool", agentsToolCallSpan.span.id), + ); const agentsStreamOperation = findLatestSpan( events, "mistral-agents-stream-operation", @@ -395,6 +407,9 @@ function buildSpanSummary( findMistralSpan(events, agentsCompleteOperation?.span.id, [ "mistral.agents.complete", ]), + agentsToolCallOperation, + ...agentsToolCallSpans, + ...agentsToolSpans, agentsStreamOperation, findMistralSpan(events, agentsStreamOperation?.span.id, [ "mistral.agents.stream", @@ -622,6 +637,9 @@ export function defineMistralInstrumentationAssertions(options: { expect( toolSpan.span.parentIds.some((parentId) => spanIds.has(parentId)), ).toBe(true); + expect( + (toolSpan.row.metrics as { end?: unknown } | undefined)?.end, + ).toEqual(expect.any(Number)); expect(toolSpan.input).toBeDefined(); } expect(toolNames.has("get_weather")).toBe(true); @@ -702,6 +720,73 @@ export function defineMistralInstrumentationAssertions(options: { expect(span?.output).toBeDefined(); }); + test( + "captures trace for agents.complete() tool calling", + testConfig, + () => { + const root = findLatestSpan(events, ROOT_NAME); + const operation = findLatestSpan( + events, + "mistral-agents-tool-call-operation", + ); + const spans = findChildSpans( + events, + "mistral.agents.complete", + operation?.span.id, + ); + const spanIds = new Set(spans.map((span) => span.span.id)); + const toolSpans = spans.flatMap((span) => + findChildSpans(events, "mistral.tool", span.span.id), + ); + const spansWithToolCalls = spans.filter((span) => { + const output = span.output as + | Array<{ + message?: { + toolCalls?: unknown; + tool_calls?: unknown; + }; + }> + | undefined; + const firstChoice = Array.isArray(output) ? output[0] : undefined; + const toolCalls = + (Array.isArray(firstChoice?.message?.tool_calls) && + firstChoice.message.tool_calls) || + (Array.isArray(firstChoice?.message?.toolCalls) && + firstChoice.message.toolCalls) || + []; + return toolCalls.length > 0; + }); + + expect(operation).toBeDefined(); + expect(operation?.span.parentIds).toEqual([root?.span.id ?? ""]); + expect(spans.length).toBeGreaterThanOrEqual(1); + expect(spansWithToolCalls.length).toBeGreaterThanOrEqual(1); + + for (const span of spans) { + expect(span.span.type).toBe("llm"); + expect(span.row.metadata).toMatchObject({ + provider: "mistral", + }); + } + + expect(toolSpans.length).toBeGreaterThanOrEqual(1); + for (const toolSpan of toolSpans) { + expect(toolSpan.span.type).toBe("tool"); + expect(toolSpan.row.metadata).toMatchObject({ + provider: "mistral", + tool_name: expect.any(String), + }); + expect( + toolSpan.span.parentIds.some((parentId) => spanIds.has(parentId)), + ).toBe(true); + expect( + (toolSpan.row.metrics as { end?: unknown } | undefined)?.end, + ).toEqual(expect.any(Number)); + expect(toolSpan.input).toBeDefined(); + } + }, + ); + test("captures trace for agents.stream()", testConfig, () => { const root = findLatestSpan(events, ROOT_NAME); const operation = findLatestSpan( diff --git a/e2e/scenarios/mistral-instrumentation/scenario.impl.mjs b/e2e/scenarios/mistral-instrumentation/scenario.impl.mjs index 53044efd4..a19718fac 100644 --- a/e2e/scenarios/mistral-instrumentation/scenario.impl.mjs +++ b/e2e/scenarios/mistral-instrumentation/scenario.impl.mjs @@ -14,6 +14,7 @@ import { } from "./constants.mjs"; export const MISTRAL_SCENARIO_TIMEOUT_MS = 240_000; +const TEST_TOOL_DELAY_MS = 50; export const MISTRAL_SCENARIO_SPECS = [ { autoEntry: "scenario.mistral-v1-3-4.mjs", @@ -65,6 +66,84 @@ function nonEmptyString(value) { : null; } +function isMistralInputValidationError(error) { + return ( + error instanceof Error && + typeof error.message === "string" && + error.message.includes("Input validation failed") + ); +} + +function getWeatherToolDefinition({ legacy = false } = {}) { + return { + type: "function", + function: { + name: "get_weather", + description: "Get weather for a city.", + parameters: legacy + ? {} + : { + type: "object", + properties: { + location: { + type: "string", + description: "City name, e.g. Vienna.", + }, + }, + required: ["location"], + }, + }, + }; +} + +function getExchangeRateToolDefinition({ legacy = false } = {}) { + return { + type: "function", + function: { + name: "get_exchange_rate", + description: "Get currency exchange rate.", + parameters: legacy + ? {} + : { + type: "object", + properties: { + from_currency: { + type: "string", + description: "Base currency code, e.g. USD.", + }, + to_currency: { + type: "string", + description: "Target currency code, e.g. EUR.", + }, + }, + required: ["from_currency", "to_currency"], + }, + }, + }; +} + +function getAgentTimeToolDefinition({ legacy = false } = {}) { + return { + type: "function", + function: { + name: "get_time_in_city", + description: "Get the local time in a city.", + parameters: legacy + ? {} + : { + type: "object", + properties: { + city: { + type: "string", + description: "City name, e.g. Vienna.", + }, + }, + required: ["city"], + }, + }, + }; +} + function isRecord(value) { return typeof value === "object" && value !== null; } @@ -133,6 +212,10 @@ async function withRetry(callback, { attempts = 3, delayMs = 1_000 } = {}) { throw lastError; } +async function simulateToolExecutionDelay() { + await new Promise((resolve) => setTimeout(resolve, TEST_TOOL_DELAY_MS)); +} + async function createAgentViaHttp(client, apiKey) { const baseUrl = getMistralApiBaseUrl(client); const response = await withRetry( @@ -322,8 +405,8 @@ async function runMistralInstrumentationScenario( "chat-tool-call", async () => { await withRetry( - async () => - client.chat.complete({ + async () => { + const request = { model: CHAT_MODEL, messages: [ { @@ -332,26 +415,34 @@ async function runMistralInstrumentationScenario( "Call the get_weather tool for Vienna. Do not answer with plain text.", }, ], - tools: [ - { - type: "function", - function: { - name: "get_weather", - description: "Get weather for a city.", - parameters: {}, - }, - }, - ], toolChoice: "required", maxTokens: 48, temperature: 0, - }), + }; + + try { + return await client.chat.complete({ + ...request, + tools: [getWeatherToolDefinition()], + }); + } catch (error) { + if (!isMistralInputValidationError(error)) { + throw error; + } + + return await client.chat.complete({ + ...request, + tools: [getWeatherToolDefinition({ legacy: true })], + }); + } + }, { attempts: 3, delayMs: 1_000 }, ); + await simulateToolExecutionDelay(); await withRetry( - async () => - client.chat.complete({ + async () => { + const request = { model: CHAT_MODEL, messages: [ { @@ -360,22 +451,30 @@ async function runMistralInstrumentationScenario( "Call the get_exchange_rate tool for USD to EUR. Do not answer with plain text.", }, ], - tools: [ - { - type: "function", - function: { - name: "get_exchange_rate", - description: "Get currency exchange rate.", - parameters: {}, - }, - }, - ], toolChoice: "required", maxTokens: 48, temperature: 0, - }), + }; + + try { + return await client.chat.complete({ + ...request, + tools: [getExchangeRateToolDefinition()], + }); + } catch (error) { + if (!isMistralInputValidationError(error)) { + throw error; + } + + return await client.chat.complete({ + ...request, + tools: [getExchangeRateToolDefinition({ legacy: true })], + }); + } + }, { attempts: 3, delayMs: 1_000 }, ); + await simulateToolExecutionDelay(); }, ); @@ -443,6 +542,51 @@ async function runMistralInstrumentationScenario( }, ); + await runOperation( + "mistral-agents-tool-call-operation", + "agents-tool-call", + async () => { + await withRetry( + async () => { + const request = { + agentId, + messages: [ + { + role: "user", + content: + "Call the get_time_in_city tool for Vienna. Do not answer with plain text.", + }, + ], + responseFormat: { + type: "text", + }, + toolChoice: "required", + maxTokens: 32, + temperature: 0, + }; + + try { + return await client.agents.complete({ + ...request, + tools: [getAgentTimeToolDefinition()], + }); + } catch (error) { + if (!isMistralInputValidationError(error)) { + throw error; + } + + return await client.agents.complete({ + ...request, + tools: [getAgentTimeToolDefinition({ legacy: true })], + }); + } + }, + { attempts: 3, delayMs: 1_000 }, + ); + await simulateToolExecutionDelay(); + }, + ); + await runOperation( "mistral-agents-stream-operation", "agents-stream", diff --git a/js/src/instrumentation/plugins/mistral-plugin.test.ts b/js/src/instrumentation/plugins/mistral-plugin.test.ts index bb0a1d2a0..69f37eace 100644 --- a/js/src/instrumentation/plugins/mistral-plugin.test.ts +++ b/js/src/instrumentation/plugins/mistral-plugin.test.ts @@ -104,6 +104,21 @@ describe("extractMistralToolCallsFromOutput", () => { ], }, }, + { + index: 2, + message: { + tool_calls: [ + { + id: "call_3", + function: { + name: "lookup_clock", + arguments: { city: "Vienna" }, + output: { current_time: "09:00" }, + }, + }, + ], + }, + }, ]), ).toEqual([ { @@ -120,6 +135,13 @@ describe("extractMistralToolCallsFromOutput", () => { name: "lookup_time", arguments: '{"timezone":"Europe/Vienna"}', }, + { + choiceIndex: 2, + id: "call_3", + name: "lookup_clock", + arguments: { city: "Vienna" }, + output: { current_time: "09:00" }, + }, ]); }); diff --git a/js/src/instrumentation/plugins/mistral-plugin.ts b/js/src/instrumentation/plugins/mistral-plugin.ts index 81cb41dee..91c3baeef 100644 --- a/js/src/instrumentation/plugins/mistral-plugin.ts +++ b/js/src/instrumentation/plugins/mistral-plugin.ts @@ -384,14 +384,38 @@ function getMessageToolCalls( } type MistralExtractedToolCall = { - arguments?: string; + arguments?: unknown; choiceIndex: number; id?: string; index?: number; name?: string; + output?: unknown; type?: string; }; +function extractToolCallOutput( + toolCall: Record, + toolFunction: Record, +): unknown { + const candidates = [ + toolFunction.output, + toolFunction.result, + toolCall.output, + toolCall.result, + toolCall.response, + toolCall.toolOutput, + toolCall.tool_output, + ]; + + for (const candidate of candidates) { + if (candidate !== undefined) { + return candidate; + } + } + + return undefined; +} + export function extractMistralToolCallsFromOutput( output: unknown, ): MistralExtractedToolCall[] { @@ -420,6 +444,7 @@ export function extractMistralToolCallsFromOutput( for (const toolCall of toolCalls) { const toolFunction = isObject(toolCall.function) ? toolCall.function : {}; + const toolCallOutput = extractToolCallOutput(toolCall, toolFunction); extracted.push({ ...(typeof toolCall.id === "string" ? { id: toolCall.id } : {}), ...(typeof toolCall.index === "number" @@ -429,9 +454,10 @@ export function extractMistralToolCallsFromOutput( ...(typeof toolFunction.name === "string" ? { name: toolFunction.name } : {}), - ...(typeof toolFunction.arguments === "string" + ...(toolFunction.arguments !== undefined ? { arguments: toolFunction.arguments } : {}), + ...(toolCallOutput !== undefined ? { output: toolCallOutput } : {}), choiceIndex, }); } @@ -440,11 +466,15 @@ export function extractMistralToolCallsFromOutput( return extracted; } -function parseToolArguments(argumentsValue: string | undefined): unknown { - if (typeof argumentsValue !== "string") { +function parseToolArguments(argumentsValue: unknown): unknown { + if (argumentsValue === undefined) { return undefined; } + if (typeof argumentsValue !== "string") { + return argumentsValue; + } + try { return JSON.parse(argumentsValue); } catch { @@ -452,13 +482,37 @@ function parseToolArguments(argumentsValue: string | undefined): unknown { } } +function normalizeToolPayloadValue(value: unknown): unknown { + if (value === undefined || value === null) { + return value; + } + + if ( + typeof value === "string" || + typeof value === "number" || + typeof value === "boolean" + ) { + return value; + } + + try { + return JSON.parse(JSON.stringify(value)); + } catch { + return String(value); + } +} + function createMistralToolCallSpansFromOutput( parentSpan: Span, output: unknown, ): void { const toolCalls = extractMistralToolCallsFromOutput(output); for (const toolCall of toolCalls) { - const parsedToolArguments = parseToolArguments(toolCall.arguments); + const endTime = getCurrentUnixTimestamp(); + const parsedToolArguments = normalizeToolPayloadValue( + parseToolArguments(toolCall.arguments), + ); + const parsedToolOutput = normalizeToolPayloadValue(toolCall.output); const toolSpan = parentSpan.startSpan({ name: "mistral.tool", spanAttributes: { @@ -470,6 +524,10 @@ function createMistralToolCallSpansFromOutput( ...(parsedToolArguments !== undefined ? { input: parsedToolArguments } : {}), + ...(parsedToolOutput !== undefined ? { output: parsedToolOutput } : {}), + metrics: { + end: endTime, + }, metadata: { provider: "mistral", ...(toolCall.name ? { tool_name: toolCall.name } : {}), @@ -479,7 +537,7 @@ function createMistralToolCallSpansFromOutput( choice_index: toolCall.choiceIndex, }, }); - toolSpan.end(); + toolSpan.end({ endTime }); } } From 8fb51d63178a24b92a810c41c20efde9c8f51eb8 Mon Sep 17 00:00:00 2001 From: Luca Forstner Date: Fri, 3 Apr 2026 11:45:43 +0200 Subject: [PATCH 10/12] narrow down again --- .vscode/settings.json | 4 +- .../mistral-v1-10-0.log-payloads.json | 152 +++++++- .../mistral-v1-10-0.span-events.json | 82 ++-- .../mistral-v1-14-1.log-payloads.json | 152 +++++++- .../mistral-v1-14-1.span-events.json | 82 ++-- .../mistral-v1-15-1.log-payloads.json | 152 +++++++- .../mistral-v1-15-1.span-events.json | 82 ++-- .../mistral-v1-3-4.log-payloads.json | 153 +++++--- .../mistral-v1-3-4.span-events.json | 123 ++---- .../mistral-v1.log-payloads.json | 152 +++++++- .../__snapshots__/mistral-v1.span-events.json | 82 ++-- .../mistral-v2.log-payloads.json | 153 +++++++- .../__snapshots__/mistral-v2.span-events.json | 83 ++-- .../mistral-instrumentation/assertions.ts | 229 ++++++----- .../mistral-instrumentation/scenario.impl.mjs | 355 ++++++++++-------- .../plugins/mistral-plugin.test.ts | 82 ---- .../instrumentation/plugins/mistral-plugin.ts | 223 ----------- 17 files changed, 1271 insertions(+), 1070 deletions(-) diff --git a/.vscode/settings.json b/.vscode/settings.json index 52877f4d9..c0693da20 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -1,4 +1,4 @@ { - "typescript.tsdk": "js/node_modules/typescript/lib", - "typescript.enablePromptUseWorkspaceTsdk": false + "js/ts.tsdk.promptToUseWorkspaceVersion": false, + "js/ts.tsdk.path": "js/node_modules/typescript/lib" } diff --git a/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-10-0.log-payloads.json b/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-10-0.log-payloads.json index 5600434bb..0dee27210 100644 --- a/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-10-0.log-payloads.json +++ b/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-10-0.log-payloads.json @@ -136,15 +136,26 @@ }, { "has_input": true, - "has_output": false, + "has_output": true, "input": { + "keys": [ + "location" + ], "type": "object" }, "metadata": { "provider": "mistral" }, "metric_keys": [], - "output": null, + "output": { + "keys": [ + "choice_index", + "tool_call_id", + "tool_index", + "tool_name" + ], + "type": "object" + }, "span_id": "" }, { @@ -179,17 +190,109 @@ }, { "has_input": true, - "has_output": false, + "has_output": true, "input": { + "keys": [ + "from_currency", + "to_currency" + ], "type": "object" }, "metadata": { "provider": "mistral" }, "metric_keys": [], - "output": null, + "output": { + "keys": [ + "choice_index", + "tool_call_id", + "tool_index", + "tool_name" + ], + "type": "object" + }, "span_id": "" }, + { + "has_input": true, + "has_output": true, + "input": { + "item_count": 2, + "roles": [ + "system", + "user" + ], + "type": "messages" + }, + "metadata": { + "model": "mistral-small-latest", + "provider": "mistral" + }, + "metric_keys": [ + "completion_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "output": { + "choice_count": 1, + "finish_reason": "tool_calls", + "has_content": false, + "role": "assistant", + "tool_call_count": 2, + "type": "choices" + }, + "span_id": "" + }, + { + "has_input": true, + "has_output": true, + "input": { + "keys": [ + "location" + ], + "type": "object" + }, + "metadata": { + "provider": "mistral" + }, + "metric_keys": [], + "output": { + "keys": [ + "choice_index", + "tool_call_id", + "tool_index", + "tool_name" + ], + "type": "object" + }, + "span_id": "" + }, + { + "has_input": true, + "has_output": true, + "input": { + "keys": [ + "from_currency", + "to_currency" + ], + "type": "object" + }, + "metadata": { + "provider": "mistral" + }, + "metric_keys": [], + "output": { + "keys": [ + "choice_index", + "tool_call_id", + "tool_index", + "tool_name" + ], + "type": "object" + }, + "span_id": "" + }, { "has_input": false, "has_output": false, @@ -199,7 +302,7 @@ }, "metric_keys": [], "output": null, - "span_id": "" + "span_id": "" }, { "has_input": true, @@ -223,7 +326,7 @@ "tool_call_count": 0, "type": "choices" }, - "span_id": "" + "span_id": "" }, { "has_input": false, @@ -234,7 +337,7 @@ }, "metric_keys": [], "output": null, - "span_id": "" + "span_id": "" }, { "has_input": true, @@ -258,7 +361,7 @@ "tool_call_count": 0, "type": "choices" }, - "span_id": "" + "span_id": "" }, { "has_input": false, @@ -269,7 +372,7 @@ }, "metric_keys": [], "output": null, - "span_id": "" + "span_id": "" }, { "has_input": true, @@ -299,7 +402,7 @@ "tool_call_count": 0, "type": "choices" }, - "span_id": "" + "span_id": "" }, { "has_input": false, @@ -310,7 +413,7 @@ }, "metric_keys": [], "output": null, - "span_id": "" + "span_id": "" }, { "has_input": true, @@ -340,20 +443,31 @@ "tool_call_count": 1, "type": "choices" }, - "span_id": "" + "span_id": "" }, { "has_input": true, - "has_output": false, + "has_output": true, "input": { + "keys": [ + "city" + ], "type": "object" }, "metadata": { "provider": "mistral" }, "metric_keys": [], - "output": null, - "span_id": "" + "output": { + "keys": [ + "choice_index", + "tool_call_id", + "tool_index", + "tool_name" + ], + "type": "object" + }, + "span_id": "" }, { "has_input": false, @@ -364,7 +478,7 @@ }, "metric_keys": [], "output": null, - "span_id": "" + "span_id": "" }, { "has_input": true, @@ -394,7 +508,7 @@ "tool_call_count": 0, "type": "choices" }, - "span_id": "" + "span_id": "" }, { "has_input": false, @@ -405,7 +519,7 @@ }, "metric_keys": [], "output": null, - "span_id": "" + "span_id": "" }, { "has_input": true, @@ -424,6 +538,6 @@ "embedding_length": "", "type": "embedding" }, - "span_id": "" + "span_id": "" } ] diff --git a/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-10-0.span-events.json b/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-10-0.span-events.json index 749d92a35..b7d2919bc 100644 --- a/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-10-0.span-events.json +++ b/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-10-0.span-events.json @@ -143,33 +143,24 @@ }, { "has_input": true, - "has_output": false, + "has_output": true, "metadata": { + "model": "mistral-small-latest", "provider": "mistral" }, - "metric_keys": [], - "name": "mistral.tool", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" + "metric_keys": [ + "completion_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" ], - "type": "tool" - }, - { - "has_input": true, - "has_output": false, - "metadata": { - "provider": "mistral" - }, - "metric_keys": [], - "name": "mistral.tool", + "name": "mistral.chat.complete", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ - "" + "" ], - "type": "tool" + "type": "llm" }, { "has_input": false, @@ -180,7 +171,7 @@ "metric_keys": [], "name": "mistral-fim-complete-operation", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ "" ], @@ -201,9 +192,9 @@ ], "name": "mistral.fim.complete", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ - "" + "" ], "type": "llm" }, @@ -216,7 +207,7 @@ "metric_keys": [], "name": "mistral-fim-stream-operation", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ "" ], @@ -237,9 +228,9 @@ ], "name": "mistral.fim.stream", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ - "" + "" ], "type": "llm" }, @@ -252,7 +243,7 @@ "metric_keys": [], "name": "mistral-agents-complete-operation", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ "" ], @@ -273,9 +264,9 @@ ], "name": "mistral.agents.complete", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ - "" + "" ], "type": "llm" }, @@ -288,7 +279,7 @@ "metric_keys": [], "name": "mistral-agents-tool-call-operation", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ "" ], @@ -309,27 +300,12 @@ ], "name": "mistral.agents.complete", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ - "" + "" ], "type": "llm" }, - { - "has_input": true, - "has_output": false, - "metadata": { - "provider": "mistral" - }, - "metric_keys": [], - "name": "mistral.tool", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "tool" - }, { "has_input": false, "has_output": false, @@ -339,7 +315,7 @@ "metric_keys": [], "name": "mistral-agents-stream-operation", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ "" ], @@ -360,9 +336,9 @@ ], "name": "mistral.agents.stream", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ - "" + "" ], "type": "llm" }, @@ -375,7 +351,7 @@ "metric_keys": [], "name": "mistral-embeddings-operation", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ "" ], @@ -395,9 +371,9 @@ ], "name": "mistral.embeddings.create", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ - "" + "" ], "type": "llm" } diff --git a/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-14-1.log-payloads.json b/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-14-1.log-payloads.json index 5600434bb..0dee27210 100644 --- a/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-14-1.log-payloads.json +++ b/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-14-1.log-payloads.json @@ -136,15 +136,26 @@ }, { "has_input": true, - "has_output": false, + "has_output": true, "input": { + "keys": [ + "location" + ], "type": "object" }, "metadata": { "provider": "mistral" }, "metric_keys": [], - "output": null, + "output": { + "keys": [ + "choice_index", + "tool_call_id", + "tool_index", + "tool_name" + ], + "type": "object" + }, "span_id": "" }, { @@ -179,17 +190,109 @@ }, { "has_input": true, - "has_output": false, + "has_output": true, "input": { + "keys": [ + "from_currency", + "to_currency" + ], "type": "object" }, "metadata": { "provider": "mistral" }, "metric_keys": [], - "output": null, + "output": { + "keys": [ + "choice_index", + "tool_call_id", + "tool_index", + "tool_name" + ], + "type": "object" + }, "span_id": "" }, + { + "has_input": true, + "has_output": true, + "input": { + "item_count": 2, + "roles": [ + "system", + "user" + ], + "type": "messages" + }, + "metadata": { + "model": "mistral-small-latest", + "provider": "mistral" + }, + "metric_keys": [ + "completion_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "output": { + "choice_count": 1, + "finish_reason": "tool_calls", + "has_content": false, + "role": "assistant", + "tool_call_count": 2, + "type": "choices" + }, + "span_id": "" + }, + { + "has_input": true, + "has_output": true, + "input": { + "keys": [ + "location" + ], + "type": "object" + }, + "metadata": { + "provider": "mistral" + }, + "metric_keys": [], + "output": { + "keys": [ + "choice_index", + "tool_call_id", + "tool_index", + "tool_name" + ], + "type": "object" + }, + "span_id": "" + }, + { + "has_input": true, + "has_output": true, + "input": { + "keys": [ + "from_currency", + "to_currency" + ], + "type": "object" + }, + "metadata": { + "provider": "mistral" + }, + "metric_keys": [], + "output": { + "keys": [ + "choice_index", + "tool_call_id", + "tool_index", + "tool_name" + ], + "type": "object" + }, + "span_id": "" + }, { "has_input": false, "has_output": false, @@ -199,7 +302,7 @@ }, "metric_keys": [], "output": null, - "span_id": "" + "span_id": "" }, { "has_input": true, @@ -223,7 +326,7 @@ "tool_call_count": 0, "type": "choices" }, - "span_id": "" + "span_id": "" }, { "has_input": false, @@ -234,7 +337,7 @@ }, "metric_keys": [], "output": null, - "span_id": "" + "span_id": "" }, { "has_input": true, @@ -258,7 +361,7 @@ "tool_call_count": 0, "type": "choices" }, - "span_id": "" + "span_id": "" }, { "has_input": false, @@ -269,7 +372,7 @@ }, "metric_keys": [], "output": null, - "span_id": "" + "span_id": "" }, { "has_input": true, @@ -299,7 +402,7 @@ "tool_call_count": 0, "type": "choices" }, - "span_id": "" + "span_id": "" }, { "has_input": false, @@ -310,7 +413,7 @@ }, "metric_keys": [], "output": null, - "span_id": "" + "span_id": "" }, { "has_input": true, @@ -340,20 +443,31 @@ "tool_call_count": 1, "type": "choices" }, - "span_id": "" + "span_id": "" }, { "has_input": true, - "has_output": false, + "has_output": true, "input": { + "keys": [ + "city" + ], "type": "object" }, "metadata": { "provider": "mistral" }, "metric_keys": [], - "output": null, - "span_id": "" + "output": { + "keys": [ + "choice_index", + "tool_call_id", + "tool_index", + "tool_name" + ], + "type": "object" + }, + "span_id": "" }, { "has_input": false, @@ -364,7 +478,7 @@ }, "metric_keys": [], "output": null, - "span_id": "" + "span_id": "" }, { "has_input": true, @@ -394,7 +508,7 @@ "tool_call_count": 0, "type": "choices" }, - "span_id": "" + "span_id": "" }, { "has_input": false, @@ -405,7 +519,7 @@ }, "metric_keys": [], "output": null, - "span_id": "" + "span_id": "" }, { "has_input": true, @@ -424,6 +538,6 @@ "embedding_length": "", "type": "embedding" }, - "span_id": "" + "span_id": "" } ] diff --git a/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-14-1.span-events.json b/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-14-1.span-events.json index 749d92a35..b7d2919bc 100644 --- a/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-14-1.span-events.json +++ b/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-14-1.span-events.json @@ -143,33 +143,24 @@ }, { "has_input": true, - "has_output": false, + "has_output": true, "metadata": { + "model": "mistral-small-latest", "provider": "mistral" }, - "metric_keys": [], - "name": "mistral.tool", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" + "metric_keys": [ + "completion_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" ], - "type": "tool" - }, - { - "has_input": true, - "has_output": false, - "metadata": { - "provider": "mistral" - }, - "metric_keys": [], - "name": "mistral.tool", + "name": "mistral.chat.complete", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ - "" + "" ], - "type": "tool" + "type": "llm" }, { "has_input": false, @@ -180,7 +171,7 @@ "metric_keys": [], "name": "mistral-fim-complete-operation", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ "" ], @@ -201,9 +192,9 @@ ], "name": "mistral.fim.complete", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ - "" + "" ], "type": "llm" }, @@ -216,7 +207,7 @@ "metric_keys": [], "name": "mistral-fim-stream-operation", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ "" ], @@ -237,9 +228,9 @@ ], "name": "mistral.fim.stream", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ - "" + "" ], "type": "llm" }, @@ -252,7 +243,7 @@ "metric_keys": [], "name": "mistral-agents-complete-operation", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ "" ], @@ -273,9 +264,9 @@ ], "name": "mistral.agents.complete", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ - "" + "" ], "type": "llm" }, @@ -288,7 +279,7 @@ "metric_keys": [], "name": "mistral-agents-tool-call-operation", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ "" ], @@ -309,27 +300,12 @@ ], "name": "mistral.agents.complete", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ - "" + "" ], "type": "llm" }, - { - "has_input": true, - "has_output": false, - "metadata": { - "provider": "mistral" - }, - "metric_keys": [], - "name": "mistral.tool", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "tool" - }, { "has_input": false, "has_output": false, @@ -339,7 +315,7 @@ "metric_keys": [], "name": "mistral-agents-stream-operation", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ "" ], @@ -360,9 +336,9 @@ ], "name": "mistral.agents.stream", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ - "" + "" ], "type": "llm" }, @@ -375,7 +351,7 @@ "metric_keys": [], "name": "mistral-embeddings-operation", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ "" ], @@ -395,9 +371,9 @@ ], "name": "mistral.embeddings.create", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ - "" + "" ], "type": "llm" } diff --git a/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-15-1.log-payloads.json b/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-15-1.log-payloads.json index 5600434bb..0dee27210 100644 --- a/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-15-1.log-payloads.json +++ b/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-15-1.log-payloads.json @@ -136,15 +136,26 @@ }, { "has_input": true, - "has_output": false, + "has_output": true, "input": { + "keys": [ + "location" + ], "type": "object" }, "metadata": { "provider": "mistral" }, "metric_keys": [], - "output": null, + "output": { + "keys": [ + "choice_index", + "tool_call_id", + "tool_index", + "tool_name" + ], + "type": "object" + }, "span_id": "" }, { @@ -179,17 +190,109 @@ }, { "has_input": true, - "has_output": false, + "has_output": true, "input": { + "keys": [ + "from_currency", + "to_currency" + ], "type": "object" }, "metadata": { "provider": "mistral" }, "metric_keys": [], - "output": null, + "output": { + "keys": [ + "choice_index", + "tool_call_id", + "tool_index", + "tool_name" + ], + "type": "object" + }, "span_id": "" }, + { + "has_input": true, + "has_output": true, + "input": { + "item_count": 2, + "roles": [ + "system", + "user" + ], + "type": "messages" + }, + "metadata": { + "model": "mistral-small-latest", + "provider": "mistral" + }, + "metric_keys": [ + "completion_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "output": { + "choice_count": 1, + "finish_reason": "tool_calls", + "has_content": false, + "role": "assistant", + "tool_call_count": 2, + "type": "choices" + }, + "span_id": "" + }, + { + "has_input": true, + "has_output": true, + "input": { + "keys": [ + "location" + ], + "type": "object" + }, + "metadata": { + "provider": "mistral" + }, + "metric_keys": [], + "output": { + "keys": [ + "choice_index", + "tool_call_id", + "tool_index", + "tool_name" + ], + "type": "object" + }, + "span_id": "" + }, + { + "has_input": true, + "has_output": true, + "input": { + "keys": [ + "from_currency", + "to_currency" + ], + "type": "object" + }, + "metadata": { + "provider": "mistral" + }, + "metric_keys": [], + "output": { + "keys": [ + "choice_index", + "tool_call_id", + "tool_index", + "tool_name" + ], + "type": "object" + }, + "span_id": "" + }, { "has_input": false, "has_output": false, @@ -199,7 +302,7 @@ }, "metric_keys": [], "output": null, - "span_id": "" + "span_id": "" }, { "has_input": true, @@ -223,7 +326,7 @@ "tool_call_count": 0, "type": "choices" }, - "span_id": "" + "span_id": "" }, { "has_input": false, @@ -234,7 +337,7 @@ }, "metric_keys": [], "output": null, - "span_id": "" + "span_id": "" }, { "has_input": true, @@ -258,7 +361,7 @@ "tool_call_count": 0, "type": "choices" }, - "span_id": "" + "span_id": "" }, { "has_input": false, @@ -269,7 +372,7 @@ }, "metric_keys": [], "output": null, - "span_id": "" + "span_id": "" }, { "has_input": true, @@ -299,7 +402,7 @@ "tool_call_count": 0, "type": "choices" }, - "span_id": "" + "span_id": "" }, { "has_input": false, @@ -310,7 +413,7 @@ }, "metric_keys": [], "output": null, - "span_id": "" + "span_id": "" }, { "has_input": true, @@ -340,20 +443,31 @@ "tool_call_count": 1, "type": "choices" }, - "span_id": "" + "span_id": "" }, { "has_input": true, - "has_output": false, + "has_output": true, "input": { + "keys": [ + "city" + ], "type": "object" }, "metadata": { "provider": "mistral" }, "metric_keys": [], - "output": null, - "span_id": "" + "output": { + "keys": [ + "choice_index", + "tool_call_id", + "tool_index", + "tool_name" + ], + "type": "object" + }, + "span_id": "" }, { "has_input": false, @@ -364,7 +478,7 @@ }, "metric_keys": [], "output": null, - "span_id": "" + "span_id": "" }, { "has_input": true, @@ -394,7 +508,7 @@ "tool_call_count": 0, "type": "choices" }, - "span_id": "" + "span_id": "" }, { "has_input": false, @@ -405,7 +519,7 @@ }, "metric_keys": [], "output": null, - "span_id": "" + "span_id": "" }, { "has_input": true, @@ -424,6 +538,6 @@ "embedding_length": "", "type": "embedding" }, - "span_id": "" + "span_id": "" } ] diff --git a/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-15-1.span-events.json b/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-15-1.span-events.json index 749d92a35..b7d2919bc 100644 --- a/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-15-1.span-events.json +++ b/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-15-1.span-events.json @@ -143,33 +143,24 @@ }, { "has_input": true, - "has_output": false, + "has_output": true, "metadata": { + "model": "mistral-small-latest", "provider": "mistral" }, - "metric_keys": [], - "name": "mistral.tool", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" + "metric_keys": [ + "completion_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" ], - "type": "tool" - }, - { - "has_input": true, - "has_output": false, - "metadata": { - "provider": "mistral" - }, - "metric_keys": [], - "name": "mistral.tool", + "name": "mistral.chat.complete", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ - "" + "" ], - "type": "tool" + "type": "llm" }, { "has_input": false, @@ -180,7 +171,7 @@ "metric_keys": [], "name": "mistral-fim-complete-operation", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ "" ], @@ -201,9 +192,9 @@ ], "name": "mistral.fim.complete", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ - "" + "" ], "type": "llm" }, @@ -216,7 +207,7 @@ "metric_keys": [], "name": "mistral-fim-stream-operation", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ "" ], @@ -237,9 +228,9 @@ ], "name": "mistral.fim.stream", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ - "" + "" ], "type": "llm" }, @@ -252,7 +243,7 @@ "metric_keys": [], "name": "mistral-agents-complete-operation", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ "" ], @@ -273,9 +264,9 @@ ], "name": "mistral.agents.complete", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ - "" + "" ], "type": "llm" }, @@ -288,7 +279,7 @@ "metric_keys": [], "name": "mistral-agents-tool-call-operation", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ "" ], @@ -309,27 +300,12 @@ ], "name": "mistral.agents.complete", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ - "" + "" ], "type": "llm" }, - { - "has_input": true, - "has_output": false, - "metadata": { - "provider": "mistral" - }, - "metric_keys": [], - "name": "mistral.tool", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "tool" - }, { "has_input": false, "has_output": false, @@ -339,7 +315,7 @@ "metric_keys": [], "name": "mistral-agents-stream-operation", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ "" ], @@ -360,9 +336,9 @@ ], "name": "mistral.agents.stream", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ - "" + "" ], "type": "llm" }, @@ -375,7 +351,7 @@ "metric_keys": [], "name": "mistral-embeddings-operation", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ "" ], @@ -395,9 +371,9 @@ ], "name": "mistral.embeddings.create", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ - "" + "" ], "type": "llm" } diff --git a/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-3-4.log-payloads.json b/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-3-4.log-payloads.json index 0fe9355b4..465546854 100644 --- a/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-3-4.log-payloads.json +++ b/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-3-4.log-payloads.json @@ -106,7 +106,7 @@ }, { "has_input": true, - "has_output": false, + "has_output": true, "input": { "item_count": 1, "roles": [ @@ -118,10 +118,43 @@ "model": "mistral-small-latest", "provider": "mistral" }, - "metric_keys": [], - "output": null, + "metric_keys": [ + "completion_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "output": { + "choice_count": 1, + "finish_reason": "tool_calls", + "has_content": false, + "role": "assistant", + "tool_call_count": 1, + "type": "choices" + }, "span_id": "" }, + { + "has_input": true, + "has_output": true, + "input": { + "keys": [], + "type": "object" + }, + "metadata": { + "provider": "mistral" + }, + "metric_keys": [], + "output": { + "keys": [ + "choice_index", + "tool_call_id", + "tool_name" + ], + "type": "object" + }, + "span_id": "" + }, { "has_input": true, "has_output": true, @@ -150,45 +183,36 @@ "tool_call_count": 1, "type": "choices" }, - "span_id": "" + "span_id": "" }, { "has_input": true, - "has_output": false, + "has_output": true, "input": { + "keys": [], "type": "object" }, "metadata": { "provider": "mistral" }, "metric_keys": [], - "output": null, - "span_id": "" - }, - { - "has_input": true, - "has_output": false, - "input": { - "item_count": 1, - "roles": [ - "user" + "output": { + "keys": [ + "choice_index", + "tool_call_id", + "tool_name" ], - "type": "messages" - }, - "metadata": { - "model": "mistral-small-latest", - "provider": "mistral" + "type": "object" }, - "metric_keys": [], - "output": null, "span_id": "" }, { "has_input": true, "has_output": true, "input": { - "item_count": 1, + "item_count": 2, "roles": [ + "system", "user" ], "type": "messages" @@ -208,24 +232,58 @@ "finish_reason": "tool_calls", "has_content": false, "role": "assistant", - "tool_call_count": 1, + "tool_call_count": 2, "type": "choices" }, "span_id": "" }, { "has_input": true, - "has_output": false, + "has_output": true, "input": { + "keys": [ + "location" + ], "type": "object" }, "metadata": { "provider": "mistral" }, "metric_keys": [], - "output": null, + "output": { + "keys": [ + "choice_index", + "tool_call_id", + "tool_name" + ], + "type": "object" + }, "span_id": "" }, + { + "has_input": true, + "has_output": true, + "input": { + "keys": [ + "from_currency", + "to_currency" + ], + "type": "object" + }, + "metadata": { + "provider": "mistral" + }, + "metric_keys": [], + "output": { + "keys": [ + "choice_index", + "tool_call_id", + "tool_name" + ], + "type": "object" + }, + "span_id": "" + }, { "has_input": false, "has_output": false, @@ -235,7 +293,7 @@ }, "metric_keys": [], "output": null, - "span_id": "" + "span_id": "" }, { "has_input": true, @@ -259,7 +317,7 @@ "tool_call_count": 0, "type": "choices" }, - "span_id": "" + "span_id": "" }, { "has_input": false, @@ -270,7 +328,7 @@ }, "metric_keys": [], "output": null, - "span_id": "" + "span_id": "" }, { "has_input": true, @@ -291,7 +349,7 @@ "tool_call_count": 0, "type": "choices" }, - "span_id": "" + "span_id": "" }, { "has_input": false, @@ -302,7 +360,7 @@ }, "metric_keys": [], "output": null, - "span_id": "" + "span_id": "" }, { "has_input": true, @@ -332,7 +390,7 @@ "tool_call_count": 0, "type": "choices" }, - "span_id": "" + "span_id": "" }, { "has_input": false, @@ -343,23 +401,6 @@ }, "metric_keys": [], "output": null, - "span_id": "" - }, - { - "has_input": true, - "has_output": false, - "input": { - "item_count": 1, - "roles": [ - "user" - ], - "type": "messages" - }, - "metadata": { - "provider": "mistral" - }, - "metric_keys": [], - "output": null, "span_id": "" }, { @@ -394,15 +435,25 @@ }, { "has_input": true, - "has_output": false, + "has_output": true, "input": { + "keys": [ + "city" + ], "type": "object" }, "metadata": { "provider": "mistral" }, "metric_keys": [], - "output": null, + "output": { + "keys": [ + "choice_index", + "tool_call_id", + "tool_name" + ], + "type": "object" + }, "span_id": "" }, { diff --git a/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-3-4.span-events.json b/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-3-4.span-events.json index 341ca8762..909d73766 100644 --- a/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-3-4.span-events.json +++ b/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-3-4.span-events.json @@ -101,12 +101,17 @@ }, { "has_input": true, - "has_output": false, + "has_output": true, "metadata": { "model": "mistral-small-latest", "provider": "mistral" }, - "metric_keys": [], + "metric_keys": [ + "completion_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], "name": "mistral.chat.complete", "root_span_id": "", "span_id": "", @@ -136,22 +141,6 @@ ], "type": "llm" }, - { - "has_input": true, - "has_output": false, - "metadata": { - "model": "mistral-small-latest", - "provider": "mistral" - }, - "metric_keys": [], - "name": "mistral.chat.complete", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - }, { "has_input": true, "has_output": true, @@ -167,42 +156,12 @@ ], "name": "mistral.chat.complete", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ "" ], "type": "llm" }, - { - "has_input": true, - "has_output": false, - "metadata": { - "provider": "mistral" - }, - "metric_keys": [], - "name": "mistral.tool", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "tool" - }, - { - "has_input": true, - "has_output": false, - "metadata": { - "provider": "mistral" - }, - "metric_keys": [], - "name": "mistral.tool", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "tool" - }, { "has_input": false, "has_output": false, @@ -212,7 +171,7 @@ "metric_keys": [], "name": "mistral-fim-complete-operation", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ "" ], @@ -233,9 +192,9 @@ ], "name": "mistral.fim.complete", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ - "" + "" ], "type": "llm" }, @@ -248,7 +207,7 @@ "metric_keys": [], "name": "mistral-fim-stream-operation", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ "" ], @@ -266,9 +225,9 @@ ], "name": "mistral.fim.stream", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ - "" + "" ], "type": "llm" }, @@ -281,7 +240,7 @@ "metric_keys": [], "name": "mistral-agents-complete-operation", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ "" ], @@ -302,9 +261,9 @@ ], "name": "mistral.agents.complete", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ - "" + "" ], "type": "llm" }, @@ -317,27 +276,12 @@ "metric_keys": [], "name": "mistral-agents-tool-call-operation", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ "" ], "type": null }, - { - "has_input": true, - "has_output": false, - "metadata": { - "provider": "mistral" - }, - "metric_keys": [], - "name": "mistral.agents.complete", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - }, { "has_input": true, "has_output": true, @@ -353,27 +297,12 @@ ], "name": "mistral.agents.complete", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ - "" + "" ], "type": "llm" }, - { - "has_input": true, - "has_output": false, - "metadata": { - "provider": "mistral" - }, - "metric_keys": [], - "name": "mistral.tool", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "tool" - }, { "has_input": false, "has_output": false, @@ -383,7 +312,7 @@ "metric_keys": [], "name": "mistral-agents-stream-operation", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ "" ], @@ -404,9 +333,9 @@ ], "name": "mistral.agents.stream", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ - "" + "" ], "type": "llm" }, @@ -419,7 +348,7 @@ "metric_keys": [], "name": "mistral-embeddings-operation", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ "" ], @@ -439,9 +368,9 @@ ], "name": "mistral.embeddings.create", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ - "" + "" ], "type": "llm" } diff --git a/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1.log-payloads.json b/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1.log-payloads.json index 5600434bb..0dee27210 100644 --- a/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1.log-payloads.json +++ b/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1.log-payloads.json @@ -136,15 +136,26 @@ }, { "has_input": true, - "has_output": false, + "has_output": true, "input": { + "keys": [ + "location" + ], "type": "object" }, "metadata": { "provider": "mistral" }, "metric_keys": [], - "output": null, + "output": { + "keys": [ + "choice_index", + "tool_call_id", + "tool_index", + "tool_name" + ], + "type": "object" + }, "span_id": "" }, { @@ -179,17 +190,109 @@ }, { "has_input": true, - "has_output": false, + "has_output": true, "input": { + "keys": [ + "from_currency", + "to_currency" + ], "type": "object" }, "metadata": { "provider": "mistral" }, "metric_keys": [], - "output": null, + "output": { + "keys": [ + "choice_index", + "tool_call_id", + "tool_index", + "tool_name" + ], + "type": "object" + }, "span_id": "" }, + { + "has_input": true, + "has_output": true, + "input": { + "item_count": 2, + "roles": [ + "system", + "user" + ], + "type": "messages" + }, + "metadata": { + "model": "mistral-small-latest", + "provider": "mistral" + }, + "metric_keys": [ + "completion_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "output": { + "choice_count": 1, + "finish_reason": "tool_calls", + "has_content": false, + "role": "assistant", + "tool_call_count": 2, + "type": "choices" + }, + "span_id": "" + }, + { + "has_input": true, + "has_output": true, + "input": { + "keys": [ + "location" + ], + "type": "object" + }, + "metadata": { + "provider": "mistral" + }, + "metric_keys": [], + "output": { + "keys": [ + "choice_index", + "tool_call_id", + "tool_index", + "tool_name" + ], + "type": "object" + }, + "span_id": "" + }, + { + "has_input": true, + "has_output": true, + "input": { + "keys": [ + "from_currency", + "to_currency" + ], + "type": "object" + }, + "metadata": { + "provider": "mistral" + }, + "metric_keys": [], + "output": { + "keys": [ + "choice_index", + "tool_call_id", + "tool_index", + "tool_name" + ], + "type": "object" + }, + "span_id": "" + }, { "has_input": false, "has_output": false, @@ -199,7 +302,7 @@ }, "metric_keys": [], "output": null, - "span_id": "" + "span_id": "" }, { "has_input": true, @@ -223,7 +326,7 @@ "tool_call_count": 0, "type": "choices" }, - "span_id": "" + "span_id": "" }, { "has_input": false, @@ -234,7 +337,7 @@ }, "metric_keys": [], "output": null, - "span_id": "" + "span_id": "" }, { "has_input": true, @@ -258,7 +361,7 @@ "tool_call_count": 0, "type": "choices" }, - "span_id": "" + "span_id": "" }, { "has_input": false, @@ -269,7 +372,7 @@ }, "metric_keys": [], "output": null, - "span_id": "" + "span_id": "" }, { "has_input": true, @@ -299,7 +402,7 @@ "tool_call_count": 0, "type": "choices" }, - "span_id": "" + "span_id": "" }, { "has_input": false, @@ -310,7 +413,7 @@ }, "metric_keys": [], "output": null, - "span_id": "" + "span_id": "" }, { "has_input": true, @@ -340,20 +443,31 @@ "tool_call_count": 1, "type": "choices" }, - "span_id": "" + "span_id": "" }, { "has_input": true, - "has_output": false, + "has_output": true, "input": { + "keys": [ + "city" + ], "type": "object" }, "metadata": { "provider": "mistral" }, "metric_keys": [], - "output": null, - "span_id": "" + "output": { + "keys": [ + "choice_index", + "tool_call_id", + "tool_index", + "tool_name" + ], + "type": "object" + }, + "span_id": "" }, { "has_input": false, @@ -364,7 +478,7 @@ }, "metric_keys": [], "output": null, - "span_id": "" + "span_id": "" }, { "has_input": true, @@ -394,7 +508,7 @@ "tool_call_count": 0, "type": "choices" }, - "span_id": "" + "span_id": "" }, { "has_input": false, @@ -405,7 +519,7 @@ }, "metric_keys": [], "output": null, - "span_id": "" + "span_id": "" }, { "has_input": true, @@ -424,6 +538,6 @@ "embedding_length": "", "type": "embedding" }, - "span_id": "" + "span_id": "" } ] diff --git a/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1.span-events.json b/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1.span-events.json index 749d92a35..b7d2919bc 100644 --- a/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1.span-events.json +++ b/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1.span-events.json @@ -143,33 +143,24 @@ }, { "has_input": true, - "has_output": false, + "has_output": true, "metadata": { + "model": "mistral-small-latest", "provider": "mistral" }, - "metric_keys": [], - "name": "mistral.tool", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" + "metric_keys": [ + "completion_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" ], - "type": "tool" - }, - { - "has_input": true, - "has_output": false, - "metadata": { - "provider": "mistral" - }, - "metric_keys": [], - "name": "mistral.tool", + "name": "mistral.chat.complete", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ - "" + "" ], - "type": "tool" + "type": "llm" }, { "has_input": false, @@ -180,7 +171,7 @@ "metric_keys": [], "name": "mistral-fim-complete-operation", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ "" ], @@ -201,9 +192,9 @@ ], "name": "mistral.fim.complete", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ - "" + "" ], "type": "llm" }, @@ -216,7 +207,7 @@ "metric_keys": [], "name": "mistral-fim-stream-operation", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ "" ], @@ -237,9 +228,9 @@ ], "name": "mistral.fim.stream", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ - "" + "" ], "type": "llm" }, @@ -252,7 +243,7 @@ "metric_keys": [], "name": "mistral-agents-complete-operation", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ "" ], @@ -273,9 +264,9 @@ ], "name": "mistral.agents.complete", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ - "" + "" ], "type": "llm" }, @@ -288,7 +279,7 @@ "metric_keys": [], "name": "mistral-agents-tool-call-operation", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ "" ], @@ -309,27 +300,12 @@ ], "name": "mistral.agents.complete", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ - "" + "" ], "type": "llm" }, - { - "has_input": true, - "has_output": false, - "metadata": { - "provider": "mistral" - }, - "metric_keys": [], - "name": "mistral.tool", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "tool" - }, { "has_input": false, "has_output": false, @@ -339,7 +315,7 @@ "metric_keys": [], "name": "mistral-agents-stream-operation", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ "" ], @@ -360,9 +336,9 @@ ], "name": "mistral.agents.stream", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ - "" + "" ], "type": "llm" }, @@ -375,7 +351,7 @@ "metric_keys": [], "name": "mistral-embeddings-operation", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ "" ], @@ -395,9 +371,9 @@ ], "name": "mistral.embeddings.create", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ - "" + "" ], "type": "llm" } diff --git a/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v2.log-payloads.json b/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v2.log-payloads.json index 31b8bb5b4..abc7ba7b8 100644 --- a/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v2.log-payloads.json +++ b/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v2.log-payloads.json @@ -139,15 +139,26 @@ }, { "has_input": true, - "has_output": false, + "has_output": true, "input": { + "keys": [ + "location" + ], "type": "object" }, "metadata": { "provider": "mistral" }, "metric_keys": [], - "output": null, + "output": { + "keys": [ + "choice_index", + "tool_call_id", + "tool_index", + "tool_name" + ], + "type": "object" + }, "span_id": "" }, { @@ -183,17 +194,110 @@ }, { "has_input": true, - "has_output": false, + "has_output": true, "input": { + "keys": [ + "from_currency", + "to_currency" + ], "type": "object" }, "metadata": { "provider": "mistral" }, "metric_keys": [], - "output": null, + "output": { + "keys": [ + "choice_index", + "tool_call_id", + "tool_index", + "tool_name" + ], + "type": "object" + }, "span_id": "" }, + { + "has_input": true, + "has_output": true, + "input": { + "item_count": 2, + "roles": [ + "system", + "user" + ], + "type": "messages" + }, + "metadata": { + "model": "mistral-small-latest", + "provider": "mistral" + }, + "metric_keys": [ + "completion_tokens", + "prompt_cached_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "output": { + "choice_count": 1, + "finish_reason": "tool_calls", + "has_content": false, + "role": "assistant", + "tool_call_count": 2, + "type": "choices" + }, + "span_id": "" + }, + { + "has_input": true, + "has_output": true, + "input": { + "keys": [ + "location" + ], + "type": "object" + }, + "metadata": { + "provider": "mistral" + }, + "metric_keys": [], + "output": { + "keys": [ + "choice_index", + "tool_call_id", + "tool_index", + "tool_name" + ], + "type": "object" + }, + "span_id": "" + }, + { + "has_input": true, + "has_output": true, + "input": { + "keys": [ + "from_currency", + "to_currency" + ], + "type": "object" + }, + "metadata": { + "provider": "mistral" + }, + "metric_keys": [], + "output": { + "keys": [ + "choice_index", + "tool_call_id", + "tool_index", + "tool_name" + ], + "type": "object" + }, + "span_id": "" + }, { "has_input": false, "has_output": false, @@ -203,7 +307,7 @@ }, "metric_keys": [], "output": null, - "span_id": "" + "span_id": "" }, { "has_input": true, @@ -228,7 +332,7 @@ "tool_call_count": 0, "type": "choices" }, - "span_id": "" + "span_id": "" }, { "has_input": false, @@ -239,7 +343,7 @@ }, "metric_keys": [], "output": null, - "span_id": "" + "span_id": "" }, { "has_input": true, @@ -264,7 +368,7 @@ "tool_call_count": 0, "type": "choices" }, - "span_id": "" + "span_id": "" }, { "has_input": false, @@ -275,7 +379,7 @@ }, "metric_keys": [], "output": null, - "span_id": "" + "span_id": "" }, { "has_input": true, @@ -306,7 +410,7 @@ "tool_call_count": 0, "type": "choices" }, - "span_id": "" + "span_id": "" }, { "has_input": false, @@ -317,7 +421,7 @@ }, "metric_keys": [], "output": null, - "span_id": "" + "span_id": "" }, { "has_input": true, @@ -348,20 +452,31 @@ "tool_call_count": 1, "type": "choices" }, - "span_id": "" + "span_id": "" }, { "has_input": true, - "has_output": false, + "has_output": true, "input": { + "keys": [ + "city" + ], "type": "object" }, "metadata": { "provider": "mistral" }, "metric_keys": [], - "output": null, - "span_id": "" + "output": { + "keys": [ + "choice_index", + "tool_call_id", + "tool_index", + "tool_name" + ], + "type": "object" + }, + "span_id": "" }, { "has_input": false, @@ -372,7 +487,7 @@ }, "metric_keys": [], "output": null, - "span_id": "" + "span_id": "" }, { "has_input": true, @@ -403,7 +518,7 @@ "tool_call_count": 0, "type": "choices" }, - "span_id": "" + "span_id": "" }, { "has_input": false, @@ -414,7 +529,7 @@ }, "metric_keys": [], "output": null, - "span_id": "" + "span_id": "" }, { "has_input": true, @@ -433,6 +548,6 @@ "embedding_length": "", "type": "embedding" }, - "span_id": "" + "span_id": "" } ] diff --git a/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v2.span-events.json b/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v2.span-events.json index 6170b4952..6227c9374 100644 --- a/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v2.span-events.json +++ b/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v2.span-events.json @@ -147,33 +147,25 @@ }, { "has_input": true, - "has_output": false, + "has_output": true, "metadata": { + "model": "mistral-small-latest", "provider": "mistral" }, - "metric_keys": [], - "name": "mistral.tool", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" + "metric_keys": [ + "completion_tokens", + "prompt_cached_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" ], - "type": "tool" - }, - { - "has_input": true, - "has_output": false, - "metadata": { - "provider": "mistral" - }, - "metric_keys": [], - "name": "mistral.tool", + "name": "mistral.chat.complete", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ - "" + "" ], - "type": "tool" + "type": "llm" }, { "has_input": false, @@ -184,7 +176,7 @@ "metric_keys": [], "name": "mistral-fim-complete-operation", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ "" ], @@ -206,9 +198,9 @@ ], "name": "mistral.fim.complete", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ - "" + "" ], "type": "llm" }, @@ -221,7 +213,7 @@ "metric_keys": [], "name": "mistral-fim-stream-operation", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ "" ], @@ -243,9 +235,9 @@ ], "name": "mistral.fim.stream", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ - "" + "" ], "type": "llm" }, @@ -258,7 +250,7 @@ "metric_keys": [], "name": "mistral-agents-complete-operation", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ "" ], @@ -280,9 +272,9 @@ ], "name": "mistral.agents.complete", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ - "" + "" ], "type": "llm" }, @@ -295,7 +287,7 @@ "metric_keys": [], "name": "mistral-agents-tool-call-operation", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ "" ], @@ -317,27 +309,12 @@ ], "name": "mistral.agents.complete", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ - "" + "" ], "type": "llm" }, - { - "has_input": true, - "has_output": false, - "metadata": { - "provider": "mistral" - }, - "metric_keys": [], - "name": "mistral.tool", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "tool" - }, { "has_input": false, "has_output": false, @@ -347,7 +324,7 @@ "metric_keys": [], "name": "mistral-agents-stream-operation", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ "" ], @@ -369,9 +346,9 @@ ], "name": "mistral.agents.stream", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ - "" + "" ], "type": "llm" }, @@ -384,7 +361,7 @@ "metric_keys": [], "name": "mistral-embeddings-operation", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ "" ], @@ -404,9 +381,9 @@ ], "name": "mistral.embeddings.create", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ - "" + "" ], "type": "llm" } diff --git a/e2e/scenarios/mistral-instrumentation/assertions.ts b/e2e/scenarios/mistral-instrumentation/assertions.ts index c4925ca0b..da757b9f9 100644 --- a/e2e/scenarios/mistral-instrumentation/assertions.ts +++ b/e2e/scenarios/mistral-instrumentation/assertions.ts @@ -176,24 +176,17 @@ function summarizeOutput(output: unknown): Json { }; } -function summarizePayloadRow( - row: CapturedLogRow, - options?: { spanName?: string }, -): Json { +function summarizePayloadRow(row: CapturedLogRow): Json { const metrics = typeof row.metrics === "object" && row.metrics !== null && !Array.isArray(row.metrics) ? (row.metrics as Record) : {}; - const omitObjectKeysForInput = options?.spanName === "mistral.tool"; - return { has_input: row.input !== undefined && row.input !== null, has_output: row.output !== undefined && row.output !== null, - input: summarizeInput(row.input, { - omitObjectKeys: omitObjectKeysForInput, - }), + input: summarizeInput(row.input), metadata: pickMetadata( row.metadata as Record | undefined, ["model", "operation", "provider", "scenario"], @@ -324,6 +317,13 @@ function mergePayloadRows(rows: CapturedLogRow[]): CapturedLogRow[] { .filter((row): row is CapturedLogRow => row !== undefined); } +function pickOutputSpans(spans: T[]): T[] { + const spansWithOutput = spans.filter( + (span) => span.output !== undefined && span.output !== null, + ); + return spansWithOutput.length > 0 ? spansWithOutput : spans; +} + function buildSpanSummary( events: CapturedLogEvent[], snapshotName: string, @@ -345,9 +345,7 @@ function buildSpanSummary( "mistral.chat.complete", chatToolCallOperation?.span.id, ); - const chatToolSpans = chatToolCallSpans.flatMap((chatToolCallSpan) => - findChildSpans(events, "mistral.tool", chatToolCallSpan.span.id), - ); + const selectedChatToolCallSpans = pickOutputSpans(chatToolCallSpans); const fimCompleteOperation = findLatestSpan( events, "mistral-fim-complete-operation", @@ -369,9 +367,7 @@ function buildSpanSummary( "mistral.agents.complete", agentsToolCallOperation?.span.id, ); - const agentsToolSpans = agentsToolCallSpans.flatMap((agentsToolCallSpan) => - findChildSpans(events, "mistral.tool", agentsToolCallSpan.span.id), - ); + const selectedAgentsToolCallSpans = pickOutputSpans(agentsToolCallSpans); const agentsStreamOperation = findLatestSpan( events, "mistral-agents-stream-operation", @@ -393,8 +389,7 @@ function buildSpanSummary( "mistral.chat.stream", ]), chatToolCallOperation, - ...chatToolCallSpans, - ...chatToolSpans, + ...selectedChatToolCallSpans, fimCompleteOperation, findMistralSpan(events, fimCompleteOperation?.span.id, [ "mistral.fim.complete", @@ -408,8 +403,7 @@ function buildSpanSummary( "mistral.agents.complete", ]), agentsToolCallOperation, - ...agentsToolCallSpans, - ...agentsToolSpans, + ...selectedAgentsToolCallSpans, agentsStreamOperation, findMistralSpan(events, agentsStreamOperation?.span.id, [ "mistral.agents.stream", @@ -437,12 +431,19 @@ function buildPayloadSummary( payloads: CapturedLogPayload[], snapshotName: string, ): Json { + const parentIdBySpanId = new Map(); const spanNameById = new Map(); for (const event of events) { if ( typeof event?.span?.id === "string" && typeof event?.span?.name === "string" ) { + const parentId = + Array.isArray(event.span.parentIds) && + typeof event.span.parentIds[0] === "string" + ? event.span.parentIds[0] + : ""; + parentIdBySpanId.set(event.span.id, parentId); spanNameById.set(event.span.id, event.span.name); } } @@ -450,15 +451,60 @@ function buildPayloadSummary( const root = findLatestSpan(events, ROOT_NAME); const payloadRows = payloadRowsForRootSpan(payloads, root?.span.id); const mergedRows = mergePayloadRows(payloadRows); + const llmSpanNames = new Set([ + "mistral.chat.complete", + "mistral.chat.stream", + "mistral.fim.complete", + "mistral.fim.stream", + "mistral.agents.complete", + "mistral.agents.stream", + "mistral.embeddings.create", + ]); + const parentAndNameWithOutput = new Set(); + + for (const row of mergedRows) { + if (typeof row.span_id !== "string") { + continue; + } + const spanName = spanNameById.get(row.span_id); + if (!spanName || !llmSpanNames.has(spanName)) { + continue; + } + if (row.output === undefined || row.output === null) { + continue; + } + + const parentId = parentIdBySpanId.get(row.span_id) || ""; + parentAndNameWithOutput.add(`${parentId}:${spanName}`); + } + + const stableRows = mergedRows.filter((row) => { + if (typeof row.span_id !== "string") { + return true; + } + + const spanName = spanNameById.get(row.span_id); + if (!spanName || !llmSpanNames.has(spanName)) { + return true; + } + + if (row.output !== undefined && row.output !== null) { + return true; + } + + const parentId = parentIdBySpanId.get(row.span_id) || ""; + return !parentAndNameWithOutput.has(`${parentId}:${spanName}`); + }); + return normalizeForSnapshot( - mergedRows.map((row) => { + stableRows.map((row) => { const spanName = typeof row.span_id === "string" ? spanNameById.get(row.span_id) : undefined; return normalizeLegacyV134PayloadSummaryRow( - summarizePayloadRow(row, { spanName }), + summarizePayloadRow(row), snapshotName, spanName, ); @@ -557,29 +603,31 @@ export function defineMistralInstrumentationAssertions(options: { "mistral.chat.complete", operation?.span.id, ); - const spanIds = new Set(spans.map((span) => span.span.id)); - const toolSpans = spans.flatMap((span) => - findChildSpans(events, "mistral.tool", span.span.id), + const selectedSpans = pickOutputSpans(spans); + const toolCallCountBySpanId = new Map( + selectedSpans.map((span) => { + const output = span.output as + | Array<{ + message?: { + toolCalls?: unknown; + tool_calls?: unknown; + }; + }> + | undefined; + const firstChoice = Array.isArray(output) ? output[0] : undefined; + const toolCalls = + (Array.isArray(firstChoice?.message?.tool_calls) && + firstChoice.message.tool_calls) || + (Array.isArray(firstChoice?.message?.toolCalls) && + firstChoice.message.toolCalls) || + []; + return [span.span.id, toolCalls.length]; + }), ); - const spansWithToolCalls = spans.filter((span) => { - const output = span.output as - | Array<{ - message?: { - toolCalls?: unknown; - tool_calls?: unknown; - }; - }> - | undefined; - const firstChoice = Array.isArray(output) ? output[0] : undefined; - const toolCalls = - (Array.isArray(firstChoice?.message?.tool_calls) && - firstChoice.message.tool_calls) || - (Array.isArray(firstChoice?.message?.toolCalls) && - firstChoice.message.toolCalls) || - []; - return toolCalls.length > 0; + const spansWithToolCalls = selectedSpans.filter((span) => { + return (toolCallCountBySpanId.get(span.span.id) || 0) > 0; }); - const finishReasons = spans + const finishReasons = selectedSpans .map((span) => { const output = span.output as | Array<{ @@ -597,28 +645,54 @@ export function defineMistralInstrumentationAssertions(options: { return undefined; }) .filter((value): value is string => typeof value === "string"); - const toolParentIds = new Set( - toolSpans.flatMap((toolSpan) => - toolSpan.span.parentIds.filter((parentId) => spanIds.has(parentId)), - ), - ); const toolNames = new Set( - toolSpans - .map( - (toolSpan) => - (toolSpan.row.metadata as { tool_name?: unknown } | undefined) - ?.tool_name, - ) - .filter((name): name is string => typeof name === "string"), + selectedSpans.flatMap((span) => { + const output = span.output as + | Array<{ + message?: { + toolCalls?: unknown; + tool_calls?: unknown; + }; + }> + | undefined; + const firstChoice = Array.isArray(output) ? output[0] : undefined; + const toolCalls = + (Array.isArray(firstChoice?.message?.tool_calls) && + firstChoice.message.tool_calls) || + (Array.isArray(firstChoice?.message?.toolCalls) && + firstChoice.message.toolCalls) || + []; + + return toolCalls + .map((toolCall) => { + if ( + !toolCall || + typeof toolCall !== "object" || + Array.isArray(toolCall) + ) { + return undefined; + } + const toolFunction = ( + toolCall as { function?: { name?: unknown } } + ).function; + return typeof toolFunction?.name === "string" + ? toolFunction.name + : undefined; + }) + .filter((name): name is string => typeof name === "string"); + }), ); expect(operation).toBeDefined(); expect(operation?.span.parentIds).toEqual([root?.span.id ?? ""]); - expect(spans.length).toBeGreaterThanOrEqual(2); + expect(selectedSpans.length).toBeGreaterThanOrEqual(2); expect(spansWithToolCalls.length).toBeGreaterThanOrEqual(2); + expect( + Array.from(toolCallCountBySpanId.values()).some((count) => count >= 2), + ).toBe(true); expect(finishReasons.length).toBeGreaterThanOrEqual(2); - for (const span of spans) { + for (const span of selectedSpans) { expect(span.span.type).toBe("llm"); expect(span.row.metadata).toMatchObject({ model: CHAT_MODEL, @@ -626,22 +700,6 @@ export function defineMistralInstrumentationAssertions(options: { }); } - expect(toolSpans.length).toBeGreaterThanOrEqual(2); - expect(toolParentIds.size).toBeGreaterThanOrEqual(2); - for (const toolSpan of toolSpans) { - expect(toolSpan.span.type).toBe("tool"); - expect(toolSpan.row.metadata).toMatchObject({ - provider: "mistral", - tool_name: expect.any(String), - }); - expect( - toolSpan.span.parentIds.some((parentId) => spanIds.has(parentId)), - ).toBe(true); - expect( - (toolSpan.row.metrics as { end?: unknown } | undefined)?.end, - ).toEqual(expect.any(Number)); - expect(toolSpan.input).toBeDefined(); - } expect(toolNames.has("get_weather")).toBe(true); expect(toolNames.has("get_exchange_rate")).toBe(true); }); @@ -734,11 +792,8 @@ export function defineMistralInstrumentationAssertions(options: { "mistral.agents.complete", operation?.span.id, ); - const spanIds = new Set(spans.map((span) => span.span.id)); - const toolSpans = spans.flatMap((span) => - findChildSpans(events, "mistral.tool", span.span.id), - ); - const spansWithToolCalls = spans.filter((span) => { + const selectedSpans = pickOutputSpans(spans); + const spansWithToolCalls = selectedSpans.filter((span) => { const output = span.output as | Array<{ message?: { @@ -759,31 +814,15 @@ export function defineMistralInstrumentationAssertions(options: { expect(operation).toBeDefined(); expect(operation?.span.parentIds).toEqual([root?.span.id ?? ""]); - expect(spans.length).toBeGreaterThanOrEqual(1); + expect(selectedSpans.length).toBeGreaterThanOrEqual(1); expect(spansWithToolCalls.length).toBeGreaterThanOrEqual(1); - for (const span of spans) { + for (const span of selectedSpans) { expect(span.span.type).toBe("llm"); expect(span.row.metadata).toMatchObject({ provider: "mistral", }); } - - expect(toolSpans.length).toBeGreaterThanOrEqual(1); - for (const toolSpan of toolSpans) { - expect(toolSpan.span.type).toBe("tool"); - expect(toolSpan.row.metadata).toMatchObject({ - provider: "mistral", - tool_name: expect.any(String), - }); - expect( - toolSpan.span.parentIds.some((parentId) => spanIds.has(parentId)), - ).toBe(true); - expect( - (toolSpan.row.metrics as { end?: unknown } | undefined)?.end, - ).toEqual(expect.any(Number)); - expect(toolSpan.input).toBeDefined(); - } }, ); diff --git a/e2e/scenarios/mistral-instrumentation/scenario.impl.mjs b/e2e/scenarios/mistral-instrumentation/scenario.impl.mjs index a19718fac..082e8333f 100644 --- a/e2e/scenarios/mistral-instrumentation/scenario.impl.mjs +++ b/e2e/scenarios/mistral-instrumentation/scenario.impl.mjs @@ -15,6 +15,11 @@ import { export const MISTRAL_SCENARIO_TIMEOUT_MS = 240_000; const TEST_TOOL_DELAY_MS = 50; +const MISTRAL_REQUEST_RETRY_OPTIONS = { + attempts: 5, + delayMs: 2_000, + maxDelayMs: 10_000, +}; export const MISTRAL_SCENARIO_SPECS = [ { autoEntry: "scenario.mistral-v1-3-4.mjs", @@ -195,7 +200,10 @@ function getAgentCreatePayload() { }; } -async function withRetry(callback, { attempts = 3, delayMs = 1_000 } = {}) { +async function withRetry( + callback, + { attempts = 3, delayMs = 1_000, maxDelayMs = Number.POSITIVE_INFINITY } = {}, +) { let lastError; for (let attempt = 1; attempt <= attempts; attempt++) { try { @@ -205,7 +213,8 @@ async function withRetry(callback, { attempts = 3, delayMs = 1_000 } = {}) { if (attempt === attempts) { throw error; } - await new Promise((resolve) => setTimeout(resolve, delayMs * attempt)); + const retryDelayMs = Math.min(delayMs * attempt, maxDelayMs); + await new Promise((resolve) => setTimeout(resolve, retryDelayMs)); } } @@ -228,7 +237,7 @@ async function createAgentViaHttp(client, apiKey) { }, method: "POST", }), - { attempts: 3, delayMs: 1_000 }, + MISTRAL_REQUEST_RETRY_OPTIONS, ); const responseBody = await response.text(); if (!response.ok) { @@ -270,7 +279,7 @@ async function createAgentViaSdk(client, apiKey) { const created = await withRetry( async () => createAgent.call(agentManager, getAgentCreatePayload()), - { attempts: 3, delayMs: 1_000 }, + MISTRAL_REQUEST_RETRY_OPTIONS, ); const createdAgentId = getAgentId(created); if (!createdAgentId) { @@ -370,7 +379,7 @@ async function runMistralInstrumentationScenario( maxTokens: 16, temperature: 0, }), - { attempts: 3, delayMs: 1_000 }, + MISTRAL_REQUEST_RETRY_OPTIONS, ); }, ); @@ -379,24 +388,21 @@ async function runMistralInstrumentationScenario( "mistral-chat-stream-operation", "chat-stream", async () => { - await withRetry( - async () => { - const stream = await client.chat.stream({ - model: CHAT_MODEL, - messages: [ - { - role: "user", - content: "Reply with exactly: streamed output", - }, - ], - maxTokens: 24, - stream: true, - temperature: 0, - }); - await collectAsync(stream); - }, - { attempts: 3, delayMs: 1_000 }, - ); + await withRetry(async () => { + const stream = await client.chat.stream({ + model: CHAT_MODEL, + messages: [ + { + role: "user", + content: "Reply with exactly: streamed output", + }, + ], + maxTokens: 24, + stream: true, + temperature: 0, + }); + await collectAsync(stream); + }, MISTRAL_REQUEST_RETRY_OPTIONS); }, ); @@ -404,76 +410,114 @@ async function runMistralInstrumentationScenario( "mistral-chat-tool-call-operation", "chat-tool-call", async () => { - await withRetry( - async () => { - const request = { - model: CHAT_MODEL, - messages: [ - { - role: "user", - content: - "Call the get_weather tool for Vienna. Do not answer with plain text.", - }, - ], - toolChoice: "required", - maxTokens: 48, - temperature: 0, - }; - - try { - return await client.chat.complete({ - ...request, - tools: [getWeatherToolDefinition()], - }); - } catch (error) { - if (!isMistralInputValidationError(error)) { - throw error; - } - - return await client.chat.complete({ - ...request, - tools: [getWeatherToolDefinition({ legacy: true })], - }); + await withRetry(async () => { + const request = { + model: CHAT_MODEL, + messages: [ + { + role: "user", + content: + "Call the get_weather tool for Vienna. Do not answer with plain text.", + }, + ], + toolChoice: "required", + maxTokens: 48, + temperature: 0, + }; + + try { + return await client.chat.complete({ + ...request, + tools: [getWeatherToolDefinition()], + }); + } catch (error) { + if (!isMistralInputValidationError(error)) { + throw error; } - }, - { attempts: 3, delayMs: 1_000 }, - ); + + return await client.chat.complete({ + ...request, + tools: [getWeatherToolDefinition({ legacy: true })], + }); + } + }, MISTRAL_REQUEST_RETRY_OPTIONS); await simulateToolExecutionDelay(); - await withRetry( - async () => { - const request = { - model: CHAT_MODEL, - messages: [ - { - role: "user", - content: - "Call the get_exchange_rate tool for USD to EUR. Do not answer with plain text.", - }, + await withRetry(async () => { + const request = { + model: CHAT_MODEL, + messages: [ + { + role: "user", + content: + "Call the get_exchange_rate tool for USD to EUR. Do not answer with plain text.", + }, + ], + toolChoice: "required", + maxTokens: 48, + temperature: 0, + }; + + try { + return await client.chat.complete({ + ...request, + tools: [getExchangeRateToolDefinition()], + }); + } catch (error) { + if (!isMistralInputValidationError(error)) { + throw error; + } + + return await client.chat.complete({ + ...request, + tools: [getExchangeRateToolDefinition({ legacy: true })], + }); + } + }, MISTRAL_REQUEST_RETRY_OPTIONS); + await simulateToolExecutionDelay(); + + await withRetry(async () => { + const request = { + model: CHAT_MODEL, + messages: [ + { + role: "system", + content: + "You must return only tool calls and no plain text.", + }, + { + role: "user", + content: + "In a single assistant response, call exactly two tools: get_weather with location Vienna and get_exchange_rate with from_currency USD and to_currency EUR.", + }, + ], + toolChoice: "required", + maxTokens: 96, + temperature: 0, + }; + + try { + return await client.chat.complete({ + ...request, + tools: [ + getWeatherToolDefinition(), + getExchangeRateToolDefinition(), ], - toolChoice: "required", - maxTokens: 48, - temperature: 0, - }; - - try { - return await client.chat.complete({ - ...request, - tools: [getExchangeRateToolDefinition()], - }); - } catch (error) { - if (!isMistralInputValidationError(error)) { - throw error; - } - - return await client.chat.complete({ - ...request, - tools: [getExchangeRateToolDefinition({ legacy: true })], - }); + }); + } catch (error) { + if (!isMistralInputValidationError(error)) { + throw error; } - }, - { attempts: 3, delayMs: 1_000 }, - ); + + return await client.chat.complete({ + ...request, + tools: [ + getWeatherToolDefinition({ legacy: true }), + getExchangeRateToolDefinition({ legacy: true }), + ], + }); + } + }, MISTRAL_REQUEST_RETRY_OPTIONS); await simulateToolExecutionDelay(); }, ); @@ -491,7 +535,7 @@ async function runMistralInstrumentationScenario( maxTokens: 24, temperature: 0, }), - { attempts: 3, delayMs: 1_000 }, + MISTRAL_REQUEST_RETRY_OPTIONS, ); }, ); @@ -500,20 +544,17 @@ async function runMistralInstrumentationScenario( "mistral-fim-stream-operation", "fim-stream", async () => { - await withRetry( - async () => { - const stream = await client.fim.stream({ - model: FIM_MODEL, - prompt: "const project = ", - suffix: ";", - maxTokens: 16, - stream: true, - temperature: 0, - }); - await collectAsync(stream); - }, - { attempts: 3, delayMs: 1_000 }, - ); + await withRetry(async () => { + const stream = await client.fim.stream({ + model: FIM_MODEL, + prompt: "const project = ", + suffix: ";", + maxTokens: 16, + stream: true, + temperature: 0, + }); + await collectAsync(stream); + }, MISTRAL_REQUEST_RETRY_OPTIONS); }, ); @@ -537,7 +578,7 @@ async function runMistralInstrumentationScenario( maxTokens: 12, temperature: 0, }), - { attempts: 3, delayMs: 1_000 }, + MISTRAL_REQUEST_RETRY_OPTIONS, ); }, ); @@ -546,43 +587,40 @@ async function runMistralInstrumentationScenario( "mistral-agents-tool-call-operation", "agents-tool-call", async () => { - await withRetry( - async () => { - const request = { - agentId, - messages: [ - { - role: "user", - content: - "Call the get_time_in_city tool for Vienna. Do not answer with plain text.", - }, - ], - responseFormat: { - type: "text", + await withRetry(async () => { + const request = { + agentId, + messages: [ + { + role: "user", + content: + "Call the get_time_in_city tool for Vienna. Do not answer with plain text.", }, - toolChoice: "required", - maxTokens: 32, - temperature: 0, - }; - - try { - return await client.agents.complete({ - ...request, - tools: [getAgentTimeToolDefinition()], - }); - } catch (error) { - if (!isMistralInputValidationError(error)) { - throw error; - } - - return await client.agents.complete({ - ...request, - tools: [getAgentTimeToolDefinition({ legacy: true })], - }); + ], + responseFormat: { + type: "text", + }, + toolChoice: "required", + maxTokens: 32, + temperature: 0, + }; + + try { + return await client.agents.complete({ + ...request, + tools: [getAgentTimeToolDefinition()], + }); + } catch (error) { + if (!isMistralInputValidationError(error)) { + throw error; } - }, - { attempts: 3, delayMs: 1_000 }, - ); + + return await client.agents.complete({ + ...request, + tools: [getAgentTimeToolDefinition({ legacy: true })], + }); + } + }, MISTRAL_REQUEST_RETRY_OPTIONS); await simulateToolExecutionDelay(); }, ); @@ -591,27 +629,24 @@ async function runMistralInstrumentationScenario( "mistral-agents-stream-operation", "agents-stream", async () => { - await withRetry( - async () => { - const stream = await client.agents.stream({ - agentId, - messages: [ - { - role: "user", - content: "Reply with exactly: agent stream", - }, - ], - responseFormat: { - type: "text", + await withRetry(async () => { + const stream = await client.agents.stream({ + agentId, + messages: [ + { + role: "user", + content: "Reply with exactly: agent stream", }, - maxTokens: 12, - stream: true, - temperature: 0, - }); - await collectAsync(stream); - }, - { attempts: 3, delayMs: 1_000 }, - ); + ], + responseFormat: { + type: "text", + }, + maxTokens: 12, + stream: true, + temperature: 0, + }); + await collectAsync(stream); + }, MISTRAL_REQUEST_RETRY_OPTIONS); }, ); @@ -625,7 +660,7 @@ async function runMistralInstrumentationScenario( model: EMBEDDING_MODEL, inputs: "braintrust mistral instrumentation", }), - { attempts: 3, delayMs: 1_000 }, + MISTRAL_REQUEST_RETRY_OPTIONS, ); }, ); diff --git a/js/src/instrumentation/plugins/mistral-plugin.test.ts b/js/src/instrumentation/plugins/mistral-plugin.test.ts index 69f37eace..e73d63076 100644 --- a/js/src/instrumentation/plugins/mistral-plugin.test.ts +++ b/js/src/instrumentation/plugins/mistral-plugin.test.ts @@ -3,7 +3,6 @@ import { aggregateMistralStreamChunks, extractMistralRequestMetadata, extractMistralResponseMetadata, - extractMistralToolCallsFromOutput, parseMistralMetricsFromUsage, } from "./mistral-plugin"; @@ -70,87 +69,6 @@ describe("extractMistralResponseMetadata", () => { }); }); -describe("extractMistralToolCallsFromOutput", () => { - it("extracts tool calls from snake_case and camelCase output", () => { - expect( - extractMistralToolCallsFromOutput([ - { - index: 0, - message: { - tool_calls: [ - { - id: "call_1", - index: 0, - type: "function", - function: { - name: "lookup_weather", - arguments: '{"city":"Vienna"}', - }, - }, - ], - }, - }, - { - index: 1, - message: { - toolCalls: [ - { - id: "call_2", - function: { - name: "lookup_time", - arguments: '{"timezone":"Europe/Vienna"}', - }, - }, - ], - }, - }, - { - index: 2, - message: { - tool_calls: [ - { - id: "call_3", - function: { - name: "lookup_clock", - arguments: { city: "Vienna" }, - output: { current_time: "09:00" }, - }, - }, - ], - }, - }, - ]), - ).toEqual([ - { - choiceIndex: 0, - id: "call_1", - index: 0, - type: "function", - name: "lookup_weather", - arguments: '{"city":"Vienna"}', - }, - { - choiceIndex: 1, - id: "call_2", - name: "lookup_time", - arguments: '{"timezone":"Europe/Vienna"}', - }, - { - choiceIndex: 2, - id: "call_3", - name: "lookup_clock", - arguments: { city: "Vienna" }, - output: { current_time: "09:00" }, - }, - ]); - }); - - it("returns empty list for non-choice output", () => { - expect(extractMistralToolCallsFromOutput(undefined)).toEqual([]); - expect(extractMistralToolCallsFromOutput({})).toEqual([]); - }); -}); - describe("parseMistralMetricsFromUsage", () => { it("returns empty metrics for missing usage", () => { expect(parseMistralMetricsFromUsage(undefined)).toEqual({}); diff --git a/js/src/instrumentation/plugins/mistral-plugin.ts b/js/src/instrumentation/plugins/mistral-plugin.ts index 91c3baeef..88ae25f46 100644 --- a/js/src/instrumentation/plugins/mistral-plugin.ts +++ b/js/src/instrumentation/plugins/mistral-plugin.ts @@ -5,7 +5,6 @@ import { unsubscribeAll, } from "../core/channel-tracing"; import { SpanTypeAttribute, isObject } from "../../../util/index"; -import type { Span } from "../../logger"; import { processInputAttachments } from "../../wrappers/attachment-utils"; import { getCurrentUnixTimestamp } from "../../util"; import { mistralChannels } from "./mistral-channels"; @@ -38,11 +37,6 @@ export class MistralPlugin extends BasePlugin { extractMetadata: (result) => extractMistralResponseMetadata(result), extractMetrics: (result, startTime) => extractMistralMetrics(result?.usage, startTime), - onComplete: ({ output, result, span }) => - createMistralToolCallSpans(span, { - output, - result, - }), }), ); @@ -56,11 +50,6 @@ export class MistralPlugin extends BasePlugin { extractMetrics: (result, startTime) => extractMistralStreamingMetrics(result, startTime), aggregateChunks: aggregateMistralStreamChunks, - onComplete: ({ output, result, span }) => - createMistralToolCallSpans(span, { - output, - result, - }), }), ); @@ -91,11 +80,6 @@ export class MistralPlugin extends BasePlugin { extractMetadata: (result) => extractMistralResponseMetadata(result), extractMetrics: (result, startTime) => extractMistralMetrics(result?.usage, startTime), - onComplete: ({ output, result, span }) => - createMistralToolCallSpans(span, { - output, - result, - }), }), ); @@ -123,11 +107,6 @@ export class MistralPlugin extends BasePlugin { extractMetadata: (result) => extractMistralResponseMetadata(result), extractMetrics: (result, startTime) => extractMistralMetrics(result?.usage, startTime), - onComplete: ({ output, result, span }) => - createMistralToolCallSpans(span, { - output, - result, - }), }), ); @@ -141,11 +120,6 @@ export class MistralPlugin extends BasePlugin { extractMetrics: (result, startTime) => extractMistralStreamingMetrics(result, startTime), aggregateChunks: aggregateMistralStreamChunks, - onComplete: ({ output, result, span }) => - createMistralToolCallSpans(span, { - output, - result, - }), }), ); } @@ -368,203 +342,6 @@ function extractMistralStreamOutput(result: unknown): unknown { return isObject(result) ? result.choices : undefined; } -function getMessageToolCalls( - message: Record | undefined, -): Record[] { - if (!message) { - return []; - } - - const rawToolCalls = - (Array.isArray(message.toolCalls) && message.toolCalls) || - (Array.isArray(message.tool_calls) && message.tool_calls) || - []; - - return rawToolCalls.filter((toolCall) => isObject(toolCall)); -} - -type MistralExtractedToolCall = { - arguments?: unknown; - choiceIndex: number; - id?: string; - index?: number; - name?: string; - output?: unknown; - type?: string; -}; - -function extractToolCallOutput( - toolCall: Record, - toolFunction: Record, -): unknown { - const candidates = [ - toolFunction.output, - toolFunction.result, - toolCall.output, - toolCall.result, - toolCall.response, - toolCall.toolOutput, - toolCall.tool_output, - ]; - - for (const candidate of candidates) { - if (candidate !== undefined) { - return candidate; - } - } - - return undefined; -} - -export function extractMistralToolCallsFromOutput( - output: unknown, -): MistralExtractedToolCall[] { - const choices = Array.isArray(output) - ? output - : isArrayLike(output) - ? Array.from(output) - : undefined; - - if (!choices) { - return []; - } - - const extracted: MistralExtractedToolCall[] = []; - - for (const rawChoice of choices) { - if (!isObject(rawChoice) || !isObject(rawChoice.message)) { - continue; - } - - const choiceIndex = - typeof rawChoice.index === "number" && rawChoice.index >= 0 - ? rawChoice.index - : 0; - const toolCalls = getMessageToolCalls(rawChoice.message); - - for (const toolCall of toolCalls) { - const toolFunction = isObject(toolCall.function) ? toolCall.function : {}; - const toolCallOutput = extractToolCallOutput(toolCall, toolFunction); - extracted.push({ - ...(typeof toolCall.id === "string" ? { id: toolCall.id } : {}), - ...(typeof toolCall.index === "number" - ? { index: toolCall.index } - : {}), - ...(typeof toolCall.type === "string" ? { type: toolCall.type } : {}), - ...(typeof toolFunction.name === "string" - ? { name: toolFunction.name } - : {}), - ...(toolFunction.arguments !== undefined - ? { arguments: toolFunction.arguments } - : {}), - ...(toolCallOutput !== undefined ? { output: toolCallOutput } : {}), - choiceIndex, - }); - } - } - - return extracted; -} - -function parseToolArguments(argumentsValue: unknown): unknown { - if (argumentsValue === undefined) { - return undefined; - } - - if (typeof argumentsValue !== "string") { - return argumentsValue; - } - - try { - return JSON.parse(argumentsValue); - } catch { - return argumentsValue; - } -} - -function normalizeToolPayloadValue(value: unknown): unknown { - if (value === undefined || value === null) { - return value; - } - - if ( - typeof value === "string" || - typeof value === "number" || - typeof value === "boolean" - ) { - return value; - } - - try { - return JSON.parse(JSON.stringify(value)); - } catch { - return String(value); - } -} - -function createMistralToolCallSpansFromOutput( - parentSpan: Span, - output: unknown, -): void { - const toolCalls = extractMistralToolCallsFromOutput(output); - for (const toolCall of toolCalls) { - const endTime = getCurrentUnixTimestamp(); - const parsedToolArguments = normalizeToolPayloadValue( - parseToolArguments(toolCall.arguments), - ); - const parsedToolOutput = normalizeToolPayloadValue(toolCall.output); - const toolSpan = parentSpan.startSpan({ - name: "mistral.tool", - spanAttributes: { - type: SpanTypeAttribute.TOOL, - }, - }); - - toolSpan.log({ - ...(parsedToolArguments !== undefined - ? { input: parsedToolArguments } - : {}), - ...(parsedToolOutput !== undefined ? { output: parsedToolOutput } : {}), - metrics: { - end: endTime, - }, - metadata: { - provider: "mistral", - ...(toolCall.name ? { tool_name: toolCall.name } : {}), - ...(toolCall.id ? { tool_call_id: toolCall.id } : {}), - ...(toolCall.index !== undefined ? { tool_index: toolCall.index } : {}), - ...(toolCall.type ? { tool_type: toolCall.type } : {}), - choice_index: toolCall.choiceIndex, - }, - }); - toolSpan.end({ endTime }); - } -} - -function createMistralToolCallSpans( - parentSpan: Span, - args: { - output: unknown; - result: unknown; - }, -): void { - const outputToolCalls = extractMistralToolCallsFromOutput(args.output); - if (outputToolCalls.length > 0) { - createMistralToolCallSpansFromOutput(parentSpan, args.output); - return; - } - - if (!isObject(args.result)) { - return; - } - - const resultChoices = args.result.choices; - const resultToolCalls = extractMistralToolCallsFromOutput(resultChoices); - if (resultToolCalls.length > 0) { - createMistralToolCallSpansFromOutput(parentSpan, resultChoices); - } -} - function extractMistralStreamingMetrics( result: unknown, startTime?: number, From 793ce51280483b7e3c37734f855477426f268409 Mon Sep 17 00:00:00 2001 From: Luca Forstner Date: Fri, 3 Apr 2026 13:10:39 +0200 Subject: [PATCH 11/12] update snapshots --- .../mistral-v1-10-0.log-payloads.json | 150 ++---------------- .../mistral-v1-14-1.log-payloads.json | 150 ++---------------- .../mistral-v1-15-1.log-payloads.json | 150 ++---------------- .../mistral-v1-3-4.log-payloads.json | 140 ++-------------- .../mistral-v1.log-payloads.json | 150 ++---------------- .../mistral-v2.log-payloads.json | 150 ++---------------- 6 files changed, 84 insertions(+), 806 deletions(-) diff --git a/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-10-0.log-payloads.json b/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-10-0.log-payloads.json index 0dee27210..c435f5788 100644 --- a/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-10-0.log-payloads.json +++ b/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-10-0.log-payloads.json @@ -134,30 +134,6 @@ }, "span_id": "" }, - { - "has_input": true, - "has_output": true, - "input": { - "keys": [ - "location" - ], - "type": "object" - }, - "metadata": { - "provider": "mistral" - }, - "metric_keys": [], - "output": { - "keys": [ - "choice_index", - "tool_call_id", - "tool_index", - "tool_name" - ], - "type": "object" - }, - "span_id": "" - }, { "has_input": true, "has_output": true, @@ -186,32 +162,7 @@ "tool_call_count": 1, "type": "choices" }, - "span_id": "" - }, - { - "has_input": true, - "has_output": true, - "input": { - "keys": [ - "from_currency", - "to_currency" - ], - "type": "object" - }, - "metadata": { - "provider": "mistral" - }, - "metric_keys": [], - "output": { - "keys": [ - "choice_index", - "tool_call_id", - "tool_index", - "tool_name" - ], - "type": "object" - }, - "span_id": "" + "span_id": "" }, { "has_input": true, @@ -242,56 +193,7 @@ "tool_call_count": 2, "type": "choices" }, - "span_id": "" - }, - { - "has_input": true, - "has_output": true, - "input": { - "keys": [ - "location" - ], - "type": "object" - }, - "metadata": { - "provider": "mistral" - }, - "metric_keys": [], - "output": { - "keys": [ - "choice_index", - "tool_call_id", - "tool_index", - "tool_name" - ], - "type": "object" - }, - "span_id": "" - }, - { - "has_input": true, - "has_output": true, - "input": { - "keys": [ - "from_currency", - "to_currency" - ], - "type": "object" - }, - "metadata": { - "provider": "mistral" - }, - "metric_keys": [], - "output": { - "keys": [ - "choice_index", - "tool_call_id", - "tool_index", - "tool_name" - ], - "type": "object" - }, - "span_id": "" + "span_id": "" }, { "has_input": false, @@ -302,7 +204,7 @@ }, "metric_keys": [], "output": null, - "span_id": "" + "span_id": "" }, { "has_input": true, @@ -326,7 +228,7 @@ "tool_call_count": 0, "type": "choices" }, - "span_id": "" + "span_id": "" }, { "has_input": false, @@ -337,7 +239,7 @@ }, "metric_keys": [], "output": null, - "span_id": "" + "span_id": "" }, { "has_input": true, @@ -361,7 +263,7 @@ "tool_call_count": 0, "type": "choices" }, - "span_id": "" + "span_id": "" }, { "has_input": false, @@ -372,7 +274,7 @@ }, "metric_keys": [], "output": null, - "span_id": "" + "span_id": "" }, { "has_input": true, @@ -402,7 +304,7 @@ "tool_call_count": 0, "type": "choices" }, - "span_id": "" + "span_id": "" }, { "has_input": false, @@ -413,7 +315,7 @@ }, "metric_keys": [], "output": null, - "span_id": "" + "span_id": "" }, { "has_input": true, @@ -443,31 +345,7 @@ "tool_call_count": 1, "type": "choices" }, - "span_id": "" - }, - { - "has_input": true, - "has_output": true, - "input": { - "keys": [ - "city" - ], - "type": "object" - }, - "metadata": { - "provider": "mistral" - }, - "metric_keys": [], - "output": { - "keys": [ - "choice_index", - "tool_call_id", - "tool_index", - "tool_name" - ], - "type": "object" - }, - "span_id": "" + "span_id": "" }, { "has_input": false, @@ -478,7 +356,7 @@ }, "metric_keys": [], "output": null, - "span_id": "" + "span_id": "" }, { "has_input": true, @@ -508,7 +386,7 @@ "tool_call_count": 0, "type": "choices" }, - "span_id": "" + "span_id": "" }, { "has_input": false, @@ -519,7 +397,7 @@ }, "metric_keys": [], "output": null, - "span_id": "" + "span_id": "" }, { "has_input": true, @@ -538,6 +416,6 @@ "embedding_length": "", "type": "embedding" }, - "span_id": "" + "span_id": "" } ] diff --git a/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-14-1.log-payloads.json b/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-14-1.log-payloads.json index 0dee27210..c435f5788 100644 --- a/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-14-1.log-payloads.json +++ b/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-14-1.log-payloads.json @@ -134,30 +134,6 @@ }, "span_id": "" }, - { - "has_input": true, - "has_output": true, - "input": { - "keys": [ - "location" - ], - "type": "object" - }, - "metadata": { - "provider": "mistral" - }, - "metric_keys": [], - "output": { - "keys": [ - "choice_index", - "tool_call_id", - "tool_index", - "tool_name" - ], - "type": "object" - }, - "span_id": "" - }, { "has_input": true, "has_output": true, @@ -186,32 +162,7 @@ "tool_call_count": 1, "type": "choices" }, - "span_id": "" - }, - { - "has_input": true, - "has_output": true, - "input": { - "keys": [ - "from_currency", - "to_currency" - ], - "type": "object" - }, - "metadata": { - "provider": "mistral" - }, - "metric_keys": [], - "output": { - "keys": [ - "choice_index", - "tool_call_id", - "tool_index", - "tool_name" - ], - "type": "object" - }, - "span_id": "" + "span_id": "" }, { "has_input": true, @@ -242,56 +193,7 @@ "tool_call_count": 2, "type": "choices" }, - "span_id": "" - }, - { - "has_input": true, - "has_output": true, - "input": { - "keys": [ - "location" - ], - "type": "object" - }, - "metadata": { - "provider": "mistral" - }, - "metric_keys": [], - "output": { - "keys": [ - "choice_index", - "tool_call_id", - "tool_index", - "tool_name" - ], - "type": "object" - }, - "span_id": "" - }, - { - "has_input": true, - "has_output": true, - "input": { - "keys": [ - "from_currency", - "to_currency" - ], - "type": "object" - }, - "metadata": { - "provider": "mistral" - }, - "metric_keys": [], - "output": { - "keys": [ - "choice_index", - "tool_call_id", - "tool_index", - "tool_name" - ], - "type": "object" - }, - "span_id": "" + "span_id": "" }, { "has_input": false, @@ -302,7 +204,7 @@ }, "metric_keys": [], "output": null, - "span_id": "" + "span_id": "" }, { "has_input": true, @@ -326,7 +228,7 @@ "tool_call_count": 0, "type": "choices" }, - "span_id": "" + "span_id": "" }, { "has_input": false, @@ -337,7 +239,7 @@ }, "metric_keys": [], "output": null, - "span_id": "" + "span_id": "" }, { "has_input": true, @@ -361,7 +263,7 @@ "tool_call_count": 0, "type": "choices" }, - "span_id": "" + "span_id": "" }, { "has_input": false, @@ -372,7 +274,7 @@ }, "metric_keys": [], "output": null, - "span_id": "" + "span_id": "" }, { "has_input": true, @@ -402,7 +304,7 @@ "tool_call_count": 0, "type": "choices" }, - "span_id": "" + "span_id": "" }, { "has_input": false, @@ -413,7 +315,7 @@ }, "metric_keys": [], "output": null, - "span_id": "" + "span_id": "" }, { "has_input": true, @@ -443,31 +345,7 @@ "tool_call_count": 1, "type": "choices" }, - "span_id": "" - }, - { - "has_input": true, - "has_output": true, - "input": { - "keys": [ - "city" - ], - "type": "object" - }, - "metadata": { - "provider": "mistral" - }, - "metric_keys": [], - "output": { - "keys": [ - "choice_index", - "tool_call_id", - "tool_index", - "tool_name" - ], - "type": "object" - }, - "span_id": "" + "span_id": "" }, { "has_input": false, @@ -478,7 +356,7 @@ }, "metric_keys": [], "output": null, - "span_id": "" + "span_id": "" }, { "has_input": true, @@ -508,7 +386,7 @@ "tool_call_count": 0, "type": "choices" }, - "span_id": "" + "span_id": "" }, { "has_input": false, @@ -519,7 +397,7 @@ }, "metric_keys": [], "output": null, - "span_id": "" + "span_id": "" }, { "has_input": true, @@ -538,6 +416,6 @@ "embedding_length": "", "type": "embedding" }, - "span_id": "" + "span_id": "" } ] diff --git a/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-15-1.log-payloads.json b/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-15-1.log-payloads.json index 0dee27210..c435f5788 100644 --- a/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-15-1.log-payloads.json +++ b/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-15-1.log-payloads.json @@ -134,30 +134,6 @@ }, "span_id": "" }, - { - "has_input": true, - "has_output": true, - "input": { - "keys": [ - "location" - ], - "type": "object" - }, - "metadata": { - "provider": "mistral" - }, - "metric_keys": [], - "output": { - "keys": [ - "choice_index", - "tool_call_id", - "tool_index", - "tool_name" - ], - "type": "object" - }, - "span_id": "" - }, { "has_input": true, "has_output": true, @@ -186,32 +162,7 @@ "tool_call_count": 1, "type": "choices" }, - "span_id": "" - }, - { - "has_input": true, - "has_output": true, - "input": { - "keys": [ - "from_currency", - "to_currency" - ], - "type": "object" - }, - "metadata": { - "provider": "mistral" - }, - "metric_keys": [], - "output": { - "keys": [ - "choice_index", - "tool_call_id", - "tool_index", - "tool_name" - ], - "type": "object" - }, - "span_id": "" + "span_id": "" }, { "has_input": true, @@ -242,56 +193,7 @@ "tool_call_count": 2, "type": "choices" }, - "span_id": "" - }, - { - "has_input": true, - "has_output": true, - "input": { - "keys": [ - "location" - ], - "type": "object" - }, - "metadata": { - "provider": "mistral" - }, - "metric_keys": [], - "output": { - "keys": [ - "choice_index", - "tool_call_id", - "tool_index", - "tool_name" - ], - "type": "object" - }, - "span_id": "" - }, - { - "has_input": true, - "has_output": true, - "input": { - "keys": [ - "from_currency", - "to_currency" - ], - "type": "object" - }, - "metadata": { - "provider": "mistral" - }, - "metric_keys": [], - "output": { - "keys": [ - "choice_index", - "tool_call_id", - "tool_index", - "tool_name" - ], - "type": "object" - }, - "span_id": "" + "span_id": "" }, { "has_input": false, @@ -302,7 +204,7 @@ }, "metric_keys": [], "output": null, - "span_id": "" + "span_id": "" }, { "has_input": true, @@ -326,7 +228,7 @@ "tool_call_count": 0, "type": "choices" }, - "span_id": "" + "span_id": "" }, { "has_input": false, @@ -337,7 +239,7 @@ }, "metric_keys": [], "output": null, - "span_id": "" + "span_id": "" }, { "has_input": true, @@ -361,7 +263,7 @@ "tool_call_count": 0, "type": "choices" }, - "span_id": "" + "span_id": "" }, { "has_input": false, @@ -372,7 +274,7 @@ }, "metric_keys": [], "output": null, - "span_id": "" + "span_id": "" }, { "has_input": true, @@ -402,7 +304,7 @@ "tool_call_count": 0, "type": "choices" }, - "span_id": "" + "span_id": "" }, { "has_input": false, @@ -413,7 +315,7 @@ }, "metric_keys": [], "output": null, - "span_id": "" + "span_id": "" }, { "has_input": true, @@ -443,31 +345,7 @@ "tool_call_count": 1, "type": "choices" }, - "span_id": "" - }, - { - "has_input": true, - "has_output": true, - "input": { - "keys": [ - "city" - ], - "type": "object" - }, - "metadata": { - "provider": "mistral" - }, - "metric_keys": [], - "output": { - "keys": [ - "choice_index", - "tool_call_id", - "tool_index", - "tool_name" - ], - "type": "object" - }, - "span_id": "" + "span_id": "" }, { "has_input": false, @@ -478,7 +356,7 @@ }, "metric_keys": [], "output": null, - "span_id": "" + "span_id": "" }, { "has_input": true, @@ -508,7 +386,7 @@ "tool_call_count": 0, "type": "choices" }, - "span_id": "" + "span_id": "" }, { "has_input": false, @@ -519,7 +397,7 @@ }, "metric_keys": [], "output": null, - "span_id": "" + "span_id": "" }, { "has_input": true, @@ -538,6 +416,6 @@ "embedding_length": "", "type": "embedding" }, - "span_id": "" + "span_id": "" } ] diff --git a/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-3-4.log-payloads.json b/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-3-4.log-payloads.json index 465546854..cf15e9eec 100644 --- a/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-3-4.log-payloads.json +++ b/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-3-4.log-payloads.json @@ -134,27 +134,6 @@ }, "span_id": "" }, - { - "has_input": true, - "has_output": true, - "input": { - "keys": [], - "type": "object" - }, - "metadata": { - "provider": "mistral" - }, - "metric_keys": [], - "output": { - "keys": [ - "choice_index", - "tool_call_id", - "tool_name" - ], - "type": "object" - }, - "span_id": "" - }, { "has_input": true, "has_output": true, @@ -183,28 +162,7 @@ "tool_call_count": 1, "type": "choices" }, - "span_id": "" - }, - { - "has_input": true, - "has_output": true, - "input": { - "keys": [], - "type": "object" - }, - "metadata": { - "provider": "mistral" - }, - "metric_keys": [], - "output": { - "keys": [ - "choice_index", - "tool_call_id", - "tool_name" - ], - "type": "object" - }, - "span_id": "" + "span_id": "" }, { "has_input": true, @@ -235,54 +193,7 @@ "tool_call_count": 2, "type": "choices" }, - "span_id": "" - }, - { - "has_input": true, - "has_output": true, - "input": { - "keys": [ - "location" - ], - "type": "object" - }, - "metadata": { - "provider": "mistral" - }, - "metric_keys": [], - "output": { - "keys": [ - "choice_index", - "tool_call_id", - "tool_name" - ], - "type": "object" - }, - "span_id": "" - }, - { - "has_input": true, - "has_output": true, - "input": { - "keys": [ - "from_currency", - "to_currency" - ], - "type": "object" - }, - "metadata": { - "provider": "mistral" - }, - "metric_keys": [], - "output": { - "keys": [ - "choice_index", - "tool_call_id", - "tool_name" - ], - "type": "object" - }, - "span_id": "" + "span_id": "" }, { "has_input": false, @@ -293,7 +204,7 @@ }, "metric_keys": [], "output": null, - "span_id": "" + "span_id": "" }, { "has_input": true, @@ -317,7 +228,7 @@ "tool_call_count": 0, "type": "choices" }, - "span_id": "" + "span_id": "" }, { "has_input": false, @@ -328,7 +239,7 @@ }, "metric_keys": [], "output": null, - "span_id": "" + "span_id": "" }, { "has_input": true, @@ -349,7 +260,7 @@ "tool_call_count": 0, "type": "choices" }, - "span_id": "" + "span_id": "" }, { "has_input": false, @@ -360,7 +271,7 @@ }, "metric_keys": [], "output": null, - "span_id": "" + "span_id": "" }, { "has_input": true, @@ -390,7 +301,7 @@ "tool_call_count": 0, "type": "choices" }, - "span_id": "" + "span_id": "" }, { "has_input": false, @@ -401,7 +312,7 @@ }, "metric_keys": [], "output": null, - "span_id": "" + "span_id": "" }, { "has_input": true, @@ -431,30 +342,7 @@ "tool_call_count": 1, "type": "choices" }, - "span_id": "" - }, - { - "has_input": true, - "has_output": true, - "input": { - "keys": [ - "city" - ], - "type": "object" - }, - "metadata": { - "provider": "mistral" - }, - "metric_keys": [], - "output": { - "keys": [ - "choice_index", - "tool_call_id", - "tool_name" - ], - "type": "object" - }, - "span_id": "" + "span_id": "" }, { "has_input": false, @@ -465,7 +353,7 @@ }, "metric_keys": [], "output": null, - "span_id": "" + "span_id": "" }, { "has_input": true, @@ -495,7 +383,7 @@ "tool_call_count": 0, "type": "choices" }, - "span_id": "" + "span_id": "" }, { "has_input": false, @@ -506,7 +394,7 @@ }, "metric_keys": [], "output": null, - "span_id": "" + "span_id": "" }, { "has_input": true, @@ -525,6 +413,6 @@ "embedding_length": "", "type": "embedding" }, - "span_id": "" + "span_id": "" } ] diff --git a/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1.log-payloads.json b/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1.log-payloads.json index 0dee27210..c435f5788 100644 --- a/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1.log-payloads.json +++ b/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1.log-payloads.json @@ -134,30 +134,6 @@ }, "span_id": "" }, - { - "has_input": true, - "has_output": true, - "input": { - "keys": [ - "location" - ], - "type": "object" - }, - "metadata": { - "provider": "mistral" - }, - "metric_keys": [], - "output": { - "keys": [ - "choice_index", - "tool_call_id", - "tool_index", - "tool_name" - ], - "type": "object" - }, - "span_id": "" - }, { "has_input": true, "has_output": true, @@ -186,32 +162,7 @@ "tool_call_count": 1, "type": "choices" }, - "span_id": "" - }, - { - "has_input": true, - "has_output": true, - "input": { - "keys": [ - "from_currency", - "to_currency" - ], - "type": "object" - }, - "metadata": { - "provider": "mistral" - }, - "metric_keys": [], - "output": { - "keys": [ - "choice_index", - "tool_call_id", - "tool_index", - "tool_name" - ], - "type": "object" - }, - "span_id": "" + "span_id": "" }, { "has_input": true, @@ -242,56 +193,7 @@ "tool_call_count": 2, "type": "choices" }, - "span_id": "" - }, - { - "has_input": true, - "has_output": true, - "input": { - "keys": [ - "location" - ], - "type": "object" - }, - "metadata": { - "provider": "mistral" - }, - "metric_keys": [], - "output": { - "keys": [ - "choice_index", - "tool_call_id", - "tool_index", - "tool_name" - ], - "type": "object" - }, - "span_id": "" - }, - { - "has_input": true, - "has_output": true, - "input": { - "keys": [ - "from_currency", - "to_currency" - ], - "type": "object" - }, - "metadata": { - "provider": "mistral" - }, - "metric_keys": [], - "output": { - "keys": [ - "choice_index", - "tool_call_id", - "tool_index", - "tool_name" - ], - "type": "object" - }, - "span_id": "" + "span_id": "" }, { "has_input": false, @@ -302,7 +204,7 @@ }, "metric_keys": [], "output": null, - "span_id": "" + "span_id": "" }, { "has_input": true, @@ -326,7 +228,7 @@ "tool_call_count": 0, "type": "choices" }, - "span_id": "" + "span_id": "" }, { "has_input": false, @@ -337,7 +239,7 @@ }, "metric_keys": [], "output": null, - "span_id": "" + "span_id": "" }, { "has_input": true, @@ -361,7 +263,7 @@ "tool_call_count": 0, "type": "choices" }, - "span_id": "" + "span_id": "" }, { "has_input": false, @@ -372,7 +274,7 @@ }, "metric_keys": [], "output": null, - "span_id": "" + "span_id": "" }, { "has_input": true, @@ -402,7 +304,7 @@ "tool_call_count": 0, "type": "choices" }, - "span_id": "" + "span_id": "" }, { "has_input": false, @@ -413,7 +315,7 @@ }, "metric_keys": [], "output": null, - "span_id": "" + "span_id": "" }, { "has_input": true, @@ -443,31 +345,7 @@ "tool_call_count": 1, "type": "choices" }, - "span_id": "" - }, - { - "has_input": true, - "has_output": true, - "input": { - "keys": [ - "city" - ], - "type": "object" - }, - "metadata": { - "provider": "mistral" - }, - "metric_keys": [], - "output": { - "keys": [ - "choice_index", - "tool_call_id", - "tool_index", - "tool_name" - ], - "type": "object" - }, - "span_id": "" + "span_id": "" }, { "has_input": false, @@ -478,7 +356,7 @@ }, "metric_keys": [], "output": null, - "span_id": "" + "span_id": "" }, { "has_input": true, @@ -508,7 +386,7 @@ "tool_call_count": 0, "type": "choices" }, - "span_id": "" + "span_id": "" }, { "has_input": false, @@ -519,7 +397,7 @@ }, "metric_keys": [], "output": null, - "span_id": "" + "span_id": "" }, { "has_input": true, @@ -538,6 +416,6 @@ "embedding_length": "", "type": "embedding" }, - "span_id": "" + "span_id": "" } ] diff --git a/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v2.log-payloads.json b/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v2.log-payloads.json index abc7ba7b8..85b50c53d 100644 --- a/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v2.log-payloads.json +++ b/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v2.log-payloads.json @@ -137,30 +137,6 @@ }, "span_id": "" }, - { - "has_input": true, - "has_output": true, - "input": { - "keys": [ - "location" - ], - "type": "object" - }, - "metadata": { - "provider": "mistral" - }, - "metric_keys": [], - "output": { - "keys": [ - "choice_index", - "tool_call_id", - "tool_index", - "tool_name" - ], - "type": "object" - }, - "span_id": "" - }, { "has_input": true, "has_output": true, @@ -190,32 +166,7 @@ "tool_call_count": 1, "type": "choices" }, - "span_id": "" - }, - { - "has_input": true, - "has_output": true, - "input": { - "keys": [ - "from_currency", - "to_currency" - ], - "type": "object" - }, - "metadata": { - "provider": "mistral" - }, - "metric_keys": [], - "output": { - "keys": [ - "choice_index", - "tool_call_id", - "tool_index", - "tool_name" - ], - "type": "object" - }, - "span_id": "" + "span_id": "" }, { "has_input": true, @@ -247,56 +198,7 @@ "tool_call_count": 2, "type": "choices" }, - "span_id": "" - }, - { - "has_input": true, - "has_output": true, - "input": { - "keys": [ - "location" - ], - "type": "object" - }, - "metadata": { - "provider": "mistral" - }, - "metric_keys": [], - "output": { - "keys": [ - "choice_index", - "tool_call_id", - "tool_index", - "tool_name" - ], - "type": "object" - }, - "span_id": "" - }, - { - "has_input": true, - "has_output": true, - "input": { - "keys": [ - "from_currency", - "to_currency" - ], - "type": "object" - }, - "metadata": { - "provider": "mistral" - }, - "metric_keys": [], - "output": { - "keys": [ - "choice_index", - "tool_call_id", - "tool_index", - "tool_name" - ], - "type": "object" - }, - "span_id": "" + "span_id": "" }, { "has_input": false, @@ -307,7 +209,7 @@ }, "metric_keys": [], "output": null, - "span_id": "" + "span_id": "" }, { "has_input": true, @@ -332,7 +234,7 @@ "tool_call_count": 0, "type": "choices" }, - "span_id": "" + "span_id": "" }, { "has_input": false, @@ -343,7 +245,7 @@ }, "metric_keys": [], "output": null, - "span_id": "" + "span_id": "" }, { "has_input": true, @@ -368,7 +270,7 @@ "tool_call_count": 0, "type": "choices" }, - "span_id": "" + "span_id": "" }, { "has_input": false, @@ -379,7 +281,7 @@ }, "metric_keys": [], "output": null, - "span_id": "" + "span_id": "" }, { "has_input": true, @@ -410,7 +312,7 @@ "tool_call_count": 0, "type": "choices" }, - "span_id": "" + "span_id": "" }, { "has_input": false, @@ -421,7 +323,7 @@ }, "metric_keys": [], "output": null, - "span_id": "" + "span_id": "" }, { "has_input": true, @@ -452,31 +354,7 @@ "tool_call_count": 1, "type": "choices" }, - "span_id": "" - }, - { - "has_input": true, - "has_output": true, - "input": { - "keys": [ - "city" - ], - "type": "object" - }, - "metadata": { - "provider": "mistral" - }, - "metric_keys": [], - "output": { - "keys": [ - "choice_index", - "tool_call_id", - "tool_index", - "tool_name" - ], - "type": "object" - }, - "span_id": "" + "span_id": "" }, { "has_input": false, @@ -487,7 +365,7 @@ }, "metric_keys": [], "output": null, - "span_id": "" + "span_id": "" }, { "has_input": true, @@ -518,7 +396,7 @@ "tool_call_count": 0, "type": "choices" }, - "span_id": "" + "span_id": "" }, { "has_input": false, @@ -529,7 +407,7 @@ }, "metric_keys": [], "output": null, - "span_id": "" + "span_id": "" }, { "has_input": true, @@ -548,6 +426,6 @@ "embedding_length": "", "type": "embedding" }, - "span_id": "" + "span_id": "" } ] From 21098caf36f62130f7f847f9b7dc190aad6fb42e Mon Sep 17 00:00:00 2001 From: Luca Forstner Date: Fri, 3 Apr 2026 15:17:43 +0200 Subject: [PATCH 12/12] fix cjs --- .../mistral-instrumentation/scenario.impl.mjs | 6 ---- .../mistral-instrumentation/scenario.test.ts | 32 +++++++++---------- .../auto-instrumentations/loader/cjs-patch.ts | 27 ++++++++++++---- .../auto-instrumentations/loader/esm-hook.mts | 13 +++++--- .../loader/get-package-version.ts | 30 +++++++++++++++-- 5 files changed, 72 insertions(+), 36 deletions(-) diff --git a/e2e/scenarios/mistral-instrumentation/scenario.impl.mjs b/e2e/scenarios/mistral-instrumentation/scenario.impl.mjs index 082e8333f..ee61b661c 100644 --- a/e2e/scenarios/mistral-instrumentation/scenario.impl.mjs +++ b/e2e/scenarios/mistral-instrumentation/scenario.impl.mjs @@ -25,42 +25,36 @@ export const MISTRAL_SCENARIO_SPECS = [ autoEntry: "scenario.mistral-v1-3-4.mjs", dependencyName: "mistral-sdk-v1-3-4", snapshotName: "mistral-v1-3-4", - supportsAutoHook: false, wrapperEntry: "scenario.mistral-v1-3-4.ts", }, { autoEntry: "scenario.mistral-v1-10-0.mjs", dependencyName: "mistral-sdk-v1-10-0", snapshotName: "mistral-v1-10-0", - supportsAutoHook: false, wrapperEntry: "scenario.mistral-v1-10-0.ts", }, { autoEntry: "scenario.mistral-v1-14-1.mjs", dependencyName: "mistral-sdk-v1-14-1", snapshotName: "mistral-v1-14-1", - supportsAutoHook: false, wrapperEntry: "scenario.mistral-v1-14-1.ts", }, { autoEntry: "scenario.mistral-v1-15-1.mjs", dependencyName: "mistral-sdk-v1-15-1", snapshotName: "mistral-v1-15-1", - supportsAutoHook: false, wrapperEntry: "scenario.mistral-v1-15-1.ts", }, { autoEntry: "scenario.mistral-v1.mjs", dependencyName: "mistral-sdk-v1", snapshotName: "mistral-v1", - supportsAutoHook: false, wrapperEntry: "scenario.mistral-v1.ts", }, { autoEntry: "scenario.mjs", dependencyName: "mistral-sdk-v2", snapshotName: "mistral-v2", - supportsAutoHook: true, wrapperEntry: "scenario.ts", }, ]; diff --git a/e2e/scenarios/mistral-instrumentation/scenario.test.ts b/e2e/scenarios/mistral-instrumentation/scenario.test.ts index 54868e61b..77b1da4f6 100644 --- a/e2e/scenarios/mistral-instrumentation/scenario.test.ts +++ b/e2e/scenarios/mistral-instrumentation/scenario.test.ts @@ -41,22 +41,20 @@ for (const scenario of mistralScenarios) { timeoutMs: MISTRAL_SCENARIO_TIMEOUT_MS, }); - if (scenario.supportsAutoHook) { - defineMistralInstrumentationAssertions({ - name: "auto-hook instrumentation", - runScenario: async ({ runNodeScenarioDir }) => { - await runNodeScenarioDir({ - entry: scenario.autoEntry, - nodeArgs: ["--import", "braintrust/hook.mjs"], - runContext: { variantKey: scenario.snapshotName }, - scenarioDir, - timeoutMs: MISTRAL_SCENARIO_TIMEOUT_MS, - }); - }, - snapshotName: scenario.snapshotName, - testFileUrl: import.meta.url, - timeoutMs: MISTRAL_SCENARIO_TIMEOUT_MS, - }); - } + defineMistralInstrumentationAssertions({ + name: "auto-hook instrumentation", + runScenario: async ({ runNodeScenarioDir }) => { + await runNodeScenarioDir({ + entry: scenario.autoEntry, + nodeArgs: ["--import", "braintrust/hook.mjs"], + runContext: { variantKey: scenario.snapshotName }, + scenarioDir, + timeoutMs: MISTRAL_SCENARIO_TIMEOUT_MS, + }); + }, + snapshotName: scenario.snapshotName, + testFileUrl: import.meta.url, + timeoutMs: MISTRAL_SCENARIO_TIMEOUT_MS, + }); }); } diff --git a/js/src/auto-instrumentations/loader/cjs-patch.ts b/js/src/auto-instrumentations/loader/cjs-patch.ts index a29014e8e..127b5ef5b 100644 --- a/js/src/auto-instrumentations/loader/cjs-patch.ts +++ b/js/src/auto-instrumentations/loader/cjs-patch.ts @@ -7,7 +7,7 @@ import { create, type InstrumentationConfig, } from "@apm-js-collab/code-transformer"; -import * as Module from "node:module"; +import * as NodeModule from "node:module"; import { sep } from "node:path"; import moduleDetailsFromPath from "module-details-from-path"; import { getPackageName, getPackageVersion } from "./get-package-version.js"; @@ -15,14 +15,18 @@ import { getPackageName, getPackageVersion } from "./get-package-version.js"; export class ModulePatch { private packages: Set; private instrumentator: any; + private modulePrototype: { _compile?: (...args: any[]) => unknown }; private originalCompile: any; constructor({ instrumentations = [], }: { instrumentations?: InstrumentationConfig[] } = {}) { + const modulePrototype = resolveModulePrototype() as any; + this.packages = new Set(instrumentations.map((i) => i.module.name)); this.instrumentator = create(instrumentations); - this.originalCompile = (Module.prototype as any)._compile; + this.modulePrototype = modulePrototype; + this.originalCompile = modulePrototype._compile; } /** @@ -32,9 +36,7 @@ export class ModulePatch { */ patch() { const self = this; - (Module.prototype as any)._compile = function wrappedCompile( - ...args: any[] - ) { + this.modulePrototype._compile = function wrappedCompile(...args: any[]) { const [content, filename] = args; // Normalize path to platform-specific separator for module-details-from-path @@ -82,6 +84,19 @@ export class ModulePatch { * **Note**: This is intended to be used in testing only. */ unpatch() { - (Module.prototype as any)._compile = this.originalCompile; + this.modulePrototype._compile = this.originalCompile; } } + +function resolveModulePrototype(): + | { _compile?: (...args: any[]) => unknown } + | undefined { + const moduleCtor = (NodeModule as any).Module; + if (moduleCtor && typeof moduleCtor === "function") { + return moduleCtor.prototype as { _compile?: (...args: any[]) => unknown }; + } + + return (NodeModule as any).prototype as + | { _compile?: (...args: any[]) => unknown } + | undefined; +} diff --git a/js/src/auto-instrumentations/loader/esm-hook.mts b/js/src/auto-instrumentations/loader/esm-hook.mts index fceeff60d..4f6d16a28 100644 --- a/js/src/auto-instrumentations/loader/esm-hook.mts +++ b/js/src/auto-instrumentations/loader/esm-hook.mts @@ -11,7 +11,7 @@ import { type InstrumentationConfig, } from "@apm-js-collab/code-transformer"; import moduleDetailsFromPath from "module-details-from-path"; -import { getPackageVersion } from "./get-package-version.js"; +import { getPackageName, getPackageVersion } from "./get-package-version.js"; let instrumentator: any; let packages: Set; @@ -44,14 +44,19 @@ export async function resolve( const resolvedModule = moduleDetailsFromPath(normalizedForPlatform); - if (resolvedModule && packages?.has(resolvedModule.name)) { - const version = getPackageVersion(resolvedModule.basedir); + if (resolvedModule) { + const packageName = + getPackageName(resolvedModule.basedir) ?? resolvedModule.name; + if (!packages?.has(packageName)) { + return url; + } + const version = getPackageVersion(resolvedModule.basedir); // Normalize module path for WASM transformer (expects forward slashes) const normalizedModulePath = resolvedModule.path.replace(/\\/g, "/"); const transformer = instrumentator.getTransformer( - resolvedModule.name, + packageName, version, normalizedModulePath, ); diff --git a/js/src/auto-instrumentations/loader/get-package-version.ts b/js/src/auto-instrumentations/loader/get-package-version.ts index 25effb8ad..d2470060b 100644 --- a/js/src/auto-instrumentations/loader/get-package-version.ts +++ b/js/src/auto-instrumentations/loader/get-package-version.ts @@ -3,7 +3,7 @@ * If the package.json file cannot be read, it defaults to the Node.js version. */ -import { readFileSync } from "node:fs"; +import { readFileSync, realpathSync } from "node:fs"; import { join } from "node:path"; const packageVersions = new Map(); @@ -19,12 +19,36 @@ function readPackageJson(baseDir: string): Record | undefined { } } +function resolvePackageBaseDir(baseDir: string): string { + try { + return realpathSync(baseDir); + } catch { + return baseDir; + } +} + +function readPackageJsonWithFallback( + baseDir: string, +): Record | undefined { + const packageJson = readPackageJson(baseDir); + if (packageJson) { + return packageJson; + } + + const resolvedBaseDir = resolvePackageBaseDir(baseDir); + if (resolvedBaseDir === baseDir) { + return undefined; + } + + return readPackageJson(resolvedBaseDir); +} + export function getPackageVersion(baseDir: string): string { if (packageVersions.has(baseDir)) { return packageVersions.get(baseDir)!; } - const packageJson = readPackageJson(baseDir); + const packageJson = readPackageJsonWithFallback(baseDir); if (typeof packageJson?.version === "string") { packageVersions.set(baseDir, packageJson.version); return packageJson.version; @@ -38,7 +62,7 @@ export function getPackageName(baseDir: string): string | undefined { return packageNames.get(baseDir); } - const packageJson = readPackageJson(baseDir); + const packageJson = readPackageJsonWithFallback(baseDir); if (typeof packageJson?.name === "string") { packageNames.set(baseDir, packageJson.name); return packageJson.name;