From ce6367775a89d7c8a17ff879c4ccc04fe2ef3870 Mon Sep 17 00:00:00 2001 From: C P Date: Mon, 27 Jul 2026 17:15:46 -0400 Subject: [PATCH 1/4] docs: document OpenAI SDK auto-instrumentation Add Python and JavaScript setup for supported OpenAI APIs. Explain initialization, content capture, exporter configuration, flushing, and verification. Clarify how provider instrumentation relates to PromptLayer traces and request logs. --- docs.json | 14 +- features/auto-instrumentation/openai.mdx | 213 +++++++++++++++ features/auto-instrumentation/overview.mdx | 17 ++ features/integrations.mdx | 4 +- features/observability.mdx | 45 ++- features/opentelemetry.mdx | 27 +- running-requests/traces.mdx | 303 ++++++--------------- sdks/javascript.mdx | 4 + sdks/python.mdx | 4 + 9 files changed, 364 insertions(+), 267 deletions(-) create mode 100644 features/auto-instrumentation/openai.mdx create mode 100644 features/auto-instrumentation/overview.mdx diff --git a/docs.json b/docs.json index a9eec372..e4f9d5d5 100644 --- a/docs.json +++ b/docs.json @@ -90,11 +90,19 @@ "icon": "chart-column", "pages": [ "features/observability", - "why-promptlayer/advanced-search", - "why-promptlayer/analytics", "running-requests/traces", - "features/opentelemetry", + { + "group": "SDK Auto-Instrumentation", + "icon": "wave-pulse", + "pages": [ + "features/auto-instrumentation/overview", + "features/auto-instrumentation/openai" + ] + }, "features/integrations", + "features/opentelemetry", + "why-promptlayer/advanced-search", + "why-promptlayer/analytics", { "group": "Advanced Logging", "icon": "cassette-tape", diff --git a/features/auto-instrumentation/openai.mdx b/features/auto-instrumentation/openai.mdx new file mode 100644 index 00000000..88d32a20 --- /dev/null +++ b/features/auto-instrumentation/openai.mdx @@ -0,0 +1,213 @@ +--- +title: "OpenAI SDK" +description: "Automatically trace supported direct OpenAI SDK calls with PromptLayer." +icon: "robot" +--- + +PromptLayer can auto-instrument the official OpenAI SDK and export supported calls as OpenTelemetry spans. Each supported direct SDK call appears in PromptLayer as both a trace span and an associated request log without changing how you create or use the OpenAI client. + + +This guide covers the OpenAI model SDK. If you use the OpenAI Agents SDK, follow the [OpenAI Agents SDK integration](/features/integrations#openai-agents-sdk) instead. + + +## Supported APIs + +| OpenAI API | Python | JavaScript | +| --- | --- | --- | +| Chat Completions (`chat.completions.create`) | Supported | Supported | +| Responses (`responses.create`) | Not supported | Supported | +| Embeddings (`embeddings.create`) | Supported | Supported | + +Only the API surfaces in this table are auto-instrumented. Other OpenAI SDK calls continue to work normally, but this integration does not automatically create PromptLayer traces or request logs for them. + +## Prerequisites + +- A [PromptLayer API key](/quickstart#prerequisites) for the workspace that should receive the traces +- An OpenAI API key +- Python 3.10 or later for the Python integration, or Node.js 20 or later for the JavaScript integration + +Export the variables for your language before the application starts. The API keys are required for the setup in this guide. The content-capture setting is required if request logs should include prompts, responses, and tool arguments. + + +```bash Python +export PROMPTLAYER_API_KEY="pl_..." +export OPENAI_API_KEY="sk_..." +export OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT="span_only" +``` + +```bash JavaScript +export PROMPTLAYER_API_KEY="pl_..." +export OPENAI_API_KEY="sk_..." +export OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT="true" +``` + + +Omit `OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT` if you want metadata-only telemetry without message contents. + +## Python + +### 1. Install the SDKs + +Install PromptLayer with the OpenAI tracing extra: + +```bash +pip install "promptlayer[otel-genai-instrumentation]" openai +``` + +### 2. Initialize instrumentation + +Call `instrument_openai()` before the first OpenAI request. It configures the OpenAI instrumentor and an authenticated OTLP exporter for PromptLayer. + +```python +from openai import OpenAI +from promptlayer import instrument_openai + +tracer_provider = instrument_openai() +client = OpenAI() + +try: + completion = client.chat.completions.create( + model="gpt-4.1-mini", + messages=[ + {"role": "user", "content": "Explain distributed tracing in one sentence."} + ], + ) + print(completion.choices[0].message.content) + + embedding = client.embeddings.create( + model="text-embedding-3-small", + input="Distributed tracing connects work across services.", + ) + print(len(embedding.data[0].embedding)) +finally: + # Flush pending spans before a short-lived process exits. + tracer_provider.force_flush() +``` + +`instrument_openai()` is idempotent when called again with the same tracer provider. If your application already owns an OpenTelemetry SDK tracer provider, pass it with `tracer_provider=`. + + +If your application already creates a PromptLayer client, `enable_tracing=True` configures the same OpenAI auto-instrumentation when the tracing extra is installed: + +```python +from openai import OpenAI +from promptlayer import PromptLayer + +promptlayer_client = PromptLayer(enable_tracing=True) +client = OpenAI() + +completion = client.chat.completions.create( + model="gpt-4.1-mini", + messages=[{"role": "user", "content": "Say hello."}], +) + +promptlayer_client.tracer_provider.force_flush() +``` + +Use either this setup or `instrument_openai()` for the same tracer provider; you do not need both. + + +## JavaScript + +### 1. Install the SDKs + +```bash +npm install promptlayer openai +``` + +### 2. Preload PromptLayer instrumentation + +Start Node.js with the `promptlayer/register` preload. The preload must run before your application imports `openai`. + +```bash +node --import promptlayer/register app.mjs +``` + +For a deployment command that you cannot edit directly, add the preload through `NODE_OPTIONS`: + +```bash +NODE_OPTIONS="--import promptlayer/register" node app.mjs +``` + +### 3. Use the OpenAI SDK normally + +```javascript +import OpenAI from "openai"; +import { shutdownTracing } from "promptlayer"; + +const client = new OpenAI(); + +try { + const completion = await client.chat.completions.create({ + model: "gpt-4.1-mini", + messages: [ + { + role: "user", + content: "Explain distributed tracing in one sentence.", + }, + ], + }); + console.log(completion.choices[0]?.message.content); + + const response = await client.responses.create({ + model: "gpt-4.1-mini", + input: "Explain distributed tracing in one sentence.", + }); + console.log(response.output_text); + + const embedding = await client.embeddings.create({ + model: "text-embedding-3-small", + input: "Distributed tracing connects work across services.", + }); + console.log(embedding.data[0]?.embedding.length); +} finally { + // Flush pending spans and stop the PromptLayer-owned tracing provider. + await shutdownTracing(); +} +``` + +Call `shutdownTracing()` when a short-lived process finishes, not after every request in a long-running server. + +## Capture Prompts and Responses + +Prompt and response content is disabled by default because it can contain sensitive data. Model names, timing, token usage when available, and other non-content telemetry are still recorded. + +The complete export blocks in [Prerequisites](#prerequisites) enable content capture. Use the language-specific value shown there, set it before `instrument_openai()` or the `promptlayer/register` preload runs, and restart an already-running process after changing it. + + +Content capture can send user prompts, model responses, and tool arguments to PromptLayer. Review your privacy, retention, and compliance requirements before enabling it. + + +## Configuration Reference + +| Setting | Required | Description | +| --- | --- | --- | +| `PROMPTLAYER_API_KEY` | Yes | Authenticates trace export and selects the PromptLayer workspace. Python can instead pass `api_key=` to `instrument_openai()`. | +| `OPENAI_API_KEY` | Yes | Authenticates OpenAI SDK requests. It is read by OpenAI and is not sent to PromptLayer. | +| `PROMPTLAYER_BASE_URL` | No | Overrides the PromptLayer API root. The tracing endpoint defaults to `/v1/traces`. | +| `PROMPTLAYER_OTLP_TRACES_ENDPOINT` | No | Overrides the complete OTLP/HTTP trace endpoint and takes precedence over `PROMPTLAYER_BASE_URL`. | +| `OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT` | For content | Includes prompts, responses, and tool arguments. Use `span_only` for Python and `true` for JavaScript. | + +Python also accepts `api_key`, `base_url`, `endpoint`, and `tracer_provider` keyword arguments: + +```python +tracer_provider = instrument_openai( + api_key="pl_...", + endpoint="https://api.promptlayer.com/v1/traces", + tracer_provider=application_tracer_provider, +) +``` + +Configure a tracer provider only once and reuse it. If the OpenAI SDK is already instrumented with a different provider, PromptLayer rejects the mismatch instead of silently exporting incomplete traces. + +## Verify the Integration + +Run one supported OpenAI request, flush tracing, and open [Traces](/running-requests/traces) in PromptLayer. The OpenAI span should have an associated request log. If the call runs inside `PromptLayer.run()`, PromptLayer links the provider span to the existing run request log instead of creating a duplicate. + +If no span appears: + +- Confirm the initialization or JavaScript preload runs before the first OpenAI call. +- Confirm the call uses an API surface listed in [Supported APIs](#supported-apis). +- Confirm `PROMPTLAYER_API_KEY` belongs to the workspace you are checking. +- Flush or shut down tracing before a short-lived process exits. +- If only prompt or response content is missing, check the language-specific content-capture value and restart the process. diff --git a/features/auto-instrumentation/overview.mdx b/features/auto-instrumentation/overview.mdx new file mode 100644 index 00000000..20c14e79 --- /dev/null +++ b/features/auto-instrumentation/overview.mdx @@ -0,0 +1,17 @@ +--- +title: "SDK Auto-Instrumentation" +description: "Trace calls made through model provider SDKs without wrapping each request." +icon: "wave-pulse" +--- + +Use SDK auto-instrumentation when your application code creates and calls a supported model provider's native client. PromptLayer registers the provider-specific instrumentor and trace exporter, so supported calls become OpenTelemetry spans and linked request logs without replacing the provider client or logging each request manually. + +## Provider Guides + +| Provider | Languages | Guide | +| --- | --- | --- | +| OpenAI | Python and JavaScript | [Auto-Instrument the OpenAI SDK](/features/auto-instrumentation/openai) | + +The provider guide is the source of truth for supported API surfaces, installation, initialization order, content capture, exporter settings, flushing, and verification. Coverage and configuration can differ between languages. + +If a framework or agent SDK creates the provider client for you, use [Telemetry Integrations](/features/integrations). If no provider guide applies or you already own the telemetry pipeline, use [OpenTelemetry](/features/opentelemetry). See the [Observability overview](/features/observability#choose-a-tracing-page) for the complete routing guide. diff --git a/features/integrations.mdx b/features/integrations.mdx index 7ff06fd3..cad67ea3 100644 --- a/features/integrations.mdx +++ b/features/integrations.mdx @@ -4,9 +4,9 @@ description: "Send traces, spans, LLM calls, tool calls, and agent telemetry int icon: 'handshake' --- -Telemetry integrations send observability data from LLM frameworks, agent SDKs, and model routers into PromptLayer. Use these setup paths when you want PromptLayer to capture traces, spans, LLM calls, tool calls, prompts, completions, token usage, and model metadata from tools you already use. +Use this page when an LLM framework, agent SDK, or model router makes calls or produces telemetry for your application. Each section is the setup guide for that framework or tool. -Don't see your framework listed? You can send traces from **any** OpenTelemetry-compatible SDK or Collector using the [OpenTelemetry](/features/opentelemetry) page, or [email us](mailto:hello@promptlayer.com). +If your tool is not listed, use [OpenTelemetry](/features/opentelemetry) or [email us](mailto:hello@promptlayer.com). If your code calls a model provider SDK directly, use [SDK Auto-Instrumentation](/features/auto-instrumentation/overview). The [Observability overview](/features/observability#choose-a-tracing-page) compares all tracing paths. ## LiteLLM diff --git a/features/observability.mdx b/features/observability.mdx index fe3e60e6..db7a19c1 100644 --- a/features/observability.mdx +++ b/features/observability.mdx @@ -1,30 +1,36 @@ --- title: "Overview" -description: "Use Observability to analyze app behavior, review PromptLayer usage, and turn useful history into datasets." +description: "Understand request logs and traces, then choose how to send telemetry to PromptLayer." icon: "book" --- -Observability helps you understand how your AI applications behave in production, testing, and development. Use it to optimize behavior, inspect workflows, and track cost and latency. +Observability helps you inspect model calls and follow the execution of AI applications across prompts, agents, tools, and workflows. -It also shows how your team uses PromptLayer: which users, workspaces, and applications are active over time. +## Request Logs and Traces -**Request logs** and **traces** are the core artifacts. Request logs capture model calls, inputs, outputs, timing, tokens, cost, status, tags, metadata, scores, and prompt associations. Traces show span-level context for workflows, agents, tools, and multi-step logic. You can use both to create datasets for evaluations, backtests, and automation. +| Artifact | Use it to | +| --- | --- | +| **Request log** | Inspect one model call, including its input, output, model, timing, tokens, cost, status, metadata, and prompt association. | +| **Trace** | Follow one end-to-end operation as a hierarchy of LLM calls, agents, tools, retrieval steps, and custom application spans. | -## What you can do +A trace can contain multiple model calls and therefore link to multiple request logs. Supported direct GenAI spans create request logs, while spans managed by `PromptLayer.run()` can link to the run's existing request log. -- Analyze application behavior with request logs and traces (spans, inputs, outputs, latency, cost, token usage, and more) -- Turn requests and traces into datasets for evaluations and regression tests. -- Review usage across workspaces, users, and environments. +## Choose a Tracing Page -## How it fits together +All tracing setup paths send spans to the same PromptLayer trace view. Start with the page that matches the code producing your telemetry: -1. Log requests and traces with the PromptLayer SDK, REST API, custom logging, or OpenTelemetry. -2. Add metadata, tags, scores, and prompt associations so you can find the right runs later. -3. Use logs, traces, and analytics to understand application behavior across prompts, users, sessions, models, workflows, and environments. -4. Use the same data to understand how your team uses PromptLayer. -5. Convert useful history into datasets for evaluations and automated feedback loops. +| Page | Use it when | What it covers | +| --- | --- | --- | +| [Traces](/running-requests/traces) | You use `PromptLayer.run()` or want to add custom application and tool spans with the PromptLayer SDK. | The trace hierarchy and UI, `traceable`, `wrapWithSpan`, `traceTool`, nesting, and filtering. | +| [SDK Auto-Instrumentation](/features/auto-instrumentation/overview) | Your application calls a supported model provider's SDK directly. | Provider-specific setup that traces SDK calls without replacing the provider client. | +| [Telemetry Integrations](/features/integrations) | A framework, agent SDK, or model router makes the calls. | Integration-specific setup for tools such as OpenAI Agents, Claude Code, Vercel AI SDK, and Pydantic AI. | +| [OpenTelemetry](/features/opentelemetry) | You already have an OpenTelemetry pipeline, use a Collector, or need to instrument an unsupported library. | PromptLayer's OTLP endpoint, GenAI semantic conventions, custom attributes, and Collector configuration. | -## Viewing logs + +Use the most specific integration available for the library making the model call. Use the generic OpenTelemetry path when you already own the telemetry pipeline or no dedicated integration applies. These paths create the same kind of PromptLayer traces; they are different setup methods, not different trace products. + + +## View Request Logs Click **Logs** in the sidebar to see request history. You can filter by prompt, search by content, inspect errors, and review request details. @@ -43,7 +49,7 @@ You can also view logs for a specific prompt by clicking **Analytics & Logs** in From the logs table, select historical requests and add them to a dataset when you want to backtest a prompt change. -## Next steps +## Explore and Use Observability Data Track cost, latency, request volume, token usage, models, prompts, tags, and metadata. - - Inspect span hierarchies, timing, inputs, outputs, errors, and linked request logs. - -If you're using a supported framework like the [Vercel AI SDK](/features/integrations#vercel-ai-sdk), [OpenAI Agents SDK](/features/integrations#openai-agents-sdk), or [Claude Code](/features/integrations#claude-code), see the [Telemetry Integrations](/features/integrations) page for framework-specific setup — those integrations handle the OTEL configuration for you. - - -## How It Works +## OTLP Endpoint PromptLayer exposes an [OTLP/HTTP endpoint](/reference/otlp-ingest-traces) at: @@ -25,7 +18,7 @@ https://api.promptlayer.com/v1/traces Any OpenTelemetry SDK or Collector can export traces to this endpoint. Spans that include [GenAI semantic convention](https://opentelemetry.io/docs/specs/semconv/gen-ai/) attributes are automatically converted into PromptLayer request logs. -## Setup +## Configure an SDK Configure your OpenTelemetry SDK to export traces to PromptLayer using the OTLP/HTTP exporter. @@ -249,12 +242,6 @@ service: This lets you fan out traces to PromptLayer alongside your existing observability backends (Datadog, New Relic, Jaeger, etc.) without changing your application code. -## Content Types - -The endpoint accepts both binary protobuf (`application/x-protobuf`, recommended) and JSON (`application/json`) encodings. Both support `Content-Encoding: gzip`. - -## Next Steps +## API Reference -- [OTLP Ingest Traces API Reference](/reference/otlp-ingest-traces) — full endpoint documentation -- [Telemetry Integrations](/features/integrations) — framework-specific setups (Vercel AI SDK, OpenAI Agents, Claude Code) -- [Traces](/running-requests/traces) — PromptLayer SDK native tracing with `@traceable` and `wrapWithSpan` +See [Ingest Traces (OTLP)](/reference/otlp-ingest-traces) for authentication, accepted encodings, compression, request schemas, responses, and errors. diff --git a/running-requests/traces.mdx b/running-requests/traces.mdx index 529dc982..8b268126 100644 --- a/running-requests/traces.mdx +++ b/running-requests/traces.mdx @@ -1,286 +1,151 @@ --- title: "Traces" +description: "Understand PromptLayer traces and add custom spans with the PromptLayer SDK." icon: "diagram-project" --- -Traces are a powerful feature in PromptLayer that allow you to monitor and analyze the execution flow of your applications, including LLM requests. Built on OpenTelemetry, Traces provide detailed insights into function calls, their durations, inputs, and outputs. +A trace represents one end-to-end operation in your application. It contains a hierarchy of **spans**, where each span records one timed step such as an LLM call, agent run, tool call, retrieval step, or application function. + +Supported LLM spans link to PromptLayer request logs, so you can move from the full execution path to the input, output, model, tokens, cost, and metadata for an individual model call. -This page covers tracing with the **PromptLayer SDK** (`@traceable`, `wrapWithSpan`). If you want to send traces from any OpenTelemetry SDK or Collector without using the PromptLayer SDK, see the [OpenTelemetry](/features/opentelemetry) page. For framework-specific integrations (Vercel AI SDK, OpenAI Agents, Claude Code), see [Telemetry Integrations](/features/integrations). To collect runner traces inside SDK evaluations (Eval spans, flush, and Trace import), see [Agent tracing for evals](/sdks/evals/agent-tracing). +This page explains the trace hierarchy and PromptLayer SDK span helpers. To choose between provider auto-instrumentation, a framework integration, and a custom OpenTelemetry pipeline, start with the [Observability overview](/features/observability#choose-a-tracing-page). -## Overview - -Traces in PromptLayer offer a comprehensive view of your application's performance and behavior. They allow you to: - -- Visualize the execution flow of your functions -- Track LLM requests and their associated metadata -- Measure function durations and identify performance bottlenecks -- Inspect function inputs and outputs for debugging +## Inspect a Trace -**Note:** The left menu in the PromptLayer UI only shows root spans, which represent the entry function of your program. While your program is running, you might not see all spans in the UI immediately, even though child spans are being sent to the backend. The root span, along with all its child spans, will only appear in the UI once the program completes. This behavior is particularly noticeable in long-running programs or those with complex execution flows. +The trace view shows parent-child relationships, duration, status, attributes, inputs, outputs, and linked request logs. Root spans represent the full operation; child spans represent the work performed inside it. ![Traces Overview](/images/traces/trace-details.png) -## Automatic LLM Request Tracing + +The trace list shows root spans. For a long-running operation, child spans can reach PromptLayer before the root span finishes, but the complete trace appears in the list after the root span ends. + -When you initialize the PromptLayer class with `enable_tracing` set to `True`, PromptLayer will automatically track any LLM calls made using the PromptLayer library. This allows you to capture detailed information about your LLM requests, including: +## Trace PromptLayer SDK Runs -- Model used -- Input prompts -- Generated responses -- Request duration -- Associated metadata +Enable tracing when you create a PromptLayer client. Calls made through `run()` then participate in the active trace. ```python Python from promptlayer import PromptLayer -# Initialize PromptLayer with tracing enabled -pl_client = PromptLayer(enable_tracing=True) +pl = PromptLayer(enable_tracing=True) + +result = pl.run( + prompt_name="simple-greeting", + input_variables={"name": "Alice"}, +) ``` ```javascript JavaScript import { PromptLayer } from "promptlayer"; -// Initialize PromptLayer with tracing enabled -const promptlayer = new PromptLayer({ - apiKey: process.env.PROMPTLAYER_API_KEY, - enableTracing: true, +const pl = new PromptLayer({ + apiKey: process.env.PROMPTLAYER_API_KEY, + enableTracing: true, +}); + +const result = await pl.run({ + promptName: "simple-greeting", + inputVariables: { name: "Alice" }, }); ``` -Once PromptLayer is initialized with tracing enabled, you can use the `run()` method to execute prompts. All LLM calls made through this method will be automatically traced, providing detailed insights into your prompt executions. +For the full `run()` interface, see the [Python SDK](/sdks/python#using-the-run-method-recommended) or [JavaScript SDK](/sdks/javascript#using-the-run-method-recommended). + + +This setting enables tracing for the PromptLayer client. To trace calls made directly through a provider client such as `OpenAI()`, use [SDK Auto-Instrumentation](/features/auto-instrumentation/overview). + + +## Add Custom Spans + +Use `traceable` in Python or `wrapWithSpan` in JavaScript to record application functions that are not traced by an integration. Give important spans a descriptive name so they are easy to identify in the trace view. ```python Python -response = pl_client.run( - prompt_name="simple-greeting", - input_variables={ - "name": "Alice" - }, - metadata={ - "user_id": "12345" - } +@pl.traceable( + name="calculate-total", + attributes={"component": "billing"}, ) - -print(response) +def calculate_total(items): + return sum(item["price"] for item in items) ``` ```javascript JavaScript -async function runPrompt() { - try { - const response = await promptlayer.run({ - promptName: "simple-greeting", - inputVariables: { - name: "Alice" - }, - metadata: { - user_id: "12345" - } - }); - - console.log(response); - } catch (error) { - console.error("Error running prompt:", error); - } -} - -runPrompt(); +const calculateTotal = pl.wrapWithSpan( + "calculate-total", + (items) => items.reduce((total, item) => total + item.price, 0) +); ``` -## Custom Function Tracing +If you omit the Python `name`, PromptLayer uses the function name. JavaScript takes the span name as the first argument to `wrapWithSpan`. -In addition to automatic LLM request tracing, you can also use the `traceable` decorator (for Python) or `wrapWithSpan` (for JavaScript) to explicitly track span data for additional functions. This allows you to gather detailed information about function executions. +## Nest Spans + +Traced functions called inside another active span become children of that span. When an in-process provider or framework integration preserves the active OpenTelemetry context, its spans also appear as children. This allows one trace to combine application, tool, and LLM spans. ```python Python -# Use the @pl_client.traceable() decorator to trace a function -@pl_client.traceable() -def greet(name): - return f"Hello, {name}!" - -# Use the decorator with custom attributes -@pl_client.traceable(attributes={"function_type": "math"}) -def calculate_sum(a, b): - return a + b - -result1 = greet("Alice") -print(result1) - -result2 = calculate_sum(5, 3) -print(result2) +@pl.traceable() +def retrieve_context(question): + return ["Relevant context"] + +@pl.traceable(name="answer-question") +def answer_question(question): + context = retrieve_context(question) + return {"question": question, "context": context} ``` ```javascript JavaScript -// Define and wrap a function with PromptLayer tracing -const greet = promptlayer.wrapWithSpan('greet', (name: string): string => { - return `Hello, ${name}!`; -}); - -const result = greet("Alice"); -console.log(result); +const retrieveContext = pl.wrapWithSpan( + "retrieve-context", + async (question) => ["Relevant context"] +); + +const answerQuestion = pl.wrapWithSpan( + "answer-question", + async (question) => ({ + question, + context: await retrieveContext(question), + }) +); ``` -## Tracing Tools +![Group nests spans](/images/traces/nested-spans.png) + +## Trace Tools -Agent tool calls have their own helper: `traceTool` (Python `@pl_client.traceTool` / JavaScript `pl.traceTool`). It is a thin specialization of `traceable` that names the span `Tool: ` and tags it with `node_type=CODE_EXECUTION` plus the tool's identity. Wrap each tool handler you want to record: +Use `traceTool` for tool handlers in a custom agent that does not already have a dedicated [Telemetry Integration](/features/integrations). PromptLayer names the span `Tool: ` and marks it as a tool call. ```python Python -pl_client = PromptLayer(enable_tracing=True) - -# name defaults to the function name -@pl_client.traceTool(name="get_weather") +@pl.traceTool(name="get_weather") def get_weather(city: str) -> str: return f"{city} is 72F and sunny." ``` ```javascript JavaScript -const pl = new PromptLayer({ enableTracing: true }); - -// traceTool(name, fn) returns a wrapped function -const getWeather = pl.traceTool("get_weather", async (city) => `${city} is 72F and sunny.`); +const getWeather = pl.traceTool( + "get_weather", + async (city) => `${city} is 72F and sunny.` +); ``` -Like `traceable`, `traceTool` only records when tracing is enabled on that client (`enable_tracing=True` / `enableTracing: true`); on a non-tracing client it is a no-op that calls the function unchanged. +`traceTool` records only when tracing is enabled on the client. The tool name is also used by tool-aware features such as the [Trajectory scorer](/sdks/evals/scorers/overview#trajectory). -The `Tool: ` span name is a contract that PromptLayer's tool-aware features read: the Trace UI surfaces these spans as tool calls, and the SDK evals [Trajectory scorer](/sdks/evals/scorers/overview#trajectory) scores the sequence of tool names against your accepted scenarios. Most framework integrations (OpenAI Agents, Claude, Vercel AI, and the others in [Telemetry Integrations](/features/integrations)) emit these spans for you, so you only reach for `traceTool` on a custom agent that has no helper. +## Filter Traces -## Setting Custom Span Names +The trace list can be filtered by metadata and resource attribute values. Filters search the entire span hierarchy: a trace appears if any root or child span has a matching attribute. -When tracing functions, you may want to set custom names for your spans to make them more descriptive. Both Python and JavaScript implementations of PromptLayer allow you to set custom span names. +In the span detail panel, hover over a top-level string, number, or boolean attribute and select the filter button to add that value to the trace list filters. Nested objects and arrays can be inspected but cannot be used as direct filter values. -### Python +## Analyze Traces -In Python, you can set a custom span name by passing the `name` parameter to the `traceable` decorator: +Trace analytics can aggregate whole traces or individual spans. Use [Analytics](/why-promptlayer/analytics) for charts and the PromptLayer AI assistant. For the exact trace- and span-level fields available through the public API, see [Trace Analytics - Custom Queries](/reference/trace-analytics-custom-analytics). -```python -@pl_client.traceable(name="CustomGreeting") -def greet(name): - return f"Hello, {name}!" - -result = greet("Alice") -print(result) -``` - -If you don't provide a name parameter, the span will use the function's name by default. - -### JavaScript - -In JavaScript, you can set a custom span name by passing it as the first argument to the wrapWithSpan function: - -```javascript JavaScript -const greet = promptlayer.wrapWithSpan('CustomGreeting', (name) => { - return `Hello, ${name}!`; -}); - -const result = greet("Alice"); -console.log(result); -``` - -If you want to use the function's name as the span name, you can simply pass the function name as a string: - -```javascript JavaScript -const greet = promptlayer.wrapWithSpan('greet', (name) => { - return `Hello, ${name}!`; -}); -``` - -## Creating Parent Spans and Grouping Function Calls - -To create a parent span and group multiple function calls within it, you can use the traceable decorator on a main function that calls other traced functions. -Here's an example that demonstrates this concept: - -```python -from promptlayer import PromptLayer - -# Initialize PromptLayer with tracing enabled -pl_client = PromptLayer(enable_tracing=True) - -@pl_client.traceable(name="custom-span") -def main(): - # This function will be the parent span - openai_call() - anthropic_call() - custom_function() - run_prompt() - -@pl_client.traceable() -def openai_call(): - response = pl_client.run( - prompt_name="simple-greeting", - input_variables={} - ) - print("OpenAI response:", response["prompt_blueprint"]["prompt_template"]["messages"][-1]) - -@pl_client.traceable() -def anthropic_call(): - response = pl_client.run( - prompt_name="simple-greeting", - input_variables={}, - provider="anthropic", - model="claude-sonnet-4-20250514" - ) - print("Anthropic response:", response["prompt_blueprint"]["prompt_template"]["messages"][-1]) - -@pl_client.traceable() -def custom_function(): - # This is a custom function that will be traced - result = "Custom function executed" - print(result) - return result - -@pl_client.traceable() -def run_prompt(): - response = pl_client.run( - prompt_name="simple-greeting", - input_variables={} - ) - print("Prompt response:", response["prompt_blueprint"]["prompt_template"]["messages"][-1]) - -if __name__ == "__main__": - main() -``` - -![Group nests spans](/images/traces/nested-spans.png) - -## Filtering Traces by Span Attributes - -The trace list can be filtered by metadata and resource attribute values. Filters search across the **entire span hierarchy** — a trace appears in results if any span within it (root or child) carries a matching attribute. - -For example, if only a nested LLM call span has `{"environment": "production"}` in its attributes, filtering the trace list by `environment = production` will still surface the parent trace in results, even if the root span itself does not carry that attribute. - -### Adding Filters from the Span Detail Panel - -When you open a span in the trace detail view, hovering over a scalar metadata or resource value reveals a filter button. Clicking it adds the key-value pair as an active filter on the trace list. The filter button appears only for top-level scalar values (string, number, or boolean). Nested objects and arrays are displayed for inspection but cannot be used as direct filter targets. - -## Trace Analytics - -Custom analytics charts can aggregate over traces at two levels: whole traces or the individual spans inside them. Charts are available from the analytics dashboard, through the PromptLayer AI assistant (e.g. "show me the slowest tools across my `agent-turn` traces this week"), and via the public API — see [Trace Analytics — Custom Queries](/reference/trace-analytics-custom-analytics). - -### Trace-level charts - -Group and measure whole traces. Group-by fields: `trace_status`, `trace_name`, `trace_models_used`, `trace_prompt_ids`, `trace_workflow_ids`, `trace_tool_names`. Metric fields: `trace_duration_ms`, `trace_total_cost_usd`, `trace_total_tokens`, `trace_input_tokens`, `trace_output_tokens`, `trace_span_count`, `trace_depth`. - -### Span-level charts - -Group and measure the individual spans inside matching traces — trace-level filters (like `trace_name` or a time range) select which traces participate, then the aggregation runs across every span within them. - -- **Group-by fields:** `span_tool_name`, `span_name`, `span_type`, `span_kind`, `span_status` -- **Metric fields:** `span_duration_ms`, `span_cost_usd`, `span_tokens`, `span_input_tokens`, `span_output_tokens` - -All standard metrics apply (`count`, `sum`, `avg`, `min`, `max`, `percentile`). Span durations are returned in seconds. Example questions this answers: - -- Which tool has the highest max/p95 latency across my agent traces? (`groupByField: span_tool_name`, `metric: max`, `metricField: span_duration_ms`) -- How many times was each tool called? (`groupByField: span_tool_name`, `metric: count`) -- What's the distribution of span durations? (`chartType: histogram`, `histogramField: span_duration_ms`) -- Which span types account for the most cost? (`groupByField: span_type`, `metric: sum`, `metricField: span_cost_usd`) - - -A chart must stay at one level: span-level fields cannot be mixed with trace-level fields in the same chart, and span-level charts do not support `timeSeries` or metadata-key breakdowns. - +To collect traces from an agent runner inside an SDK evaluation, see [Agent Tracing for Evals](/sdks/evals/agent-tracing). diff --git a/sdks/javascript.mdx b/sdks/javascript.mdx index 4b6d1e7d..69c56c6b 100644 --- a/sdks/javascript.mdx +++ b/sdks/javascript.mdx @@ -16,6 +16,10 @@ title: "JavaScript" npm install promptlayer ``` +## OpenAI SDK Auto-Instrumentation + +PromptLayer can automatically trace supported direct OpenAI SDK calls. See [OpenAI SDK Auto-Instrumentation](/features/auto-instrumentation/openai) for the preload command, supported APIs, content capture, and flushing. + ## SDK evals Define evals with `evaluate(...)`, then run them with `promptlayer eval run `. Start from the [Quickstart](/sdks/evals/quickstart) or the [SDK Evals overview](/sdks/evals/overview). diff --git a/sdks/python.mdx b/sdks/python.mdx index dbdfccdd..c6e59fac 100644 --- a/sdks/python.mdx +++ b/sdks/python.mdx @@ -16,6 +16,10 @@ title: 'Python' pip install promptlayer ``` +## OpenAI SDK Auto-Instrumentation + +PromptLayer can automatically trace supported direct OpenAI SDK calls. See [OpenAI SDK Auto-Instrumentation](/features/auto-instrumentation/openai) for the tracing extra, initialization order, supported APIs, content capture, and flushing. + ## SDK evals Define evals with `evaluate(...)`, then run them with `promptlayer eval run `. Start from the [Quickstart](/sdks/evals/quickstart) or the [SDK Evals overview](/sdks/evals/overview). From 94be5551aa643e9852805bcf030aa5fbc30719b0 Mon Sep 17 00:00:00 2001 From: C P Date: Mon, 27 Jul 2026 17:57:40 -0400 Subject: [PATCH 2/4] docs: reorganize observability by artifact Group request-log configuration under Request Logs and tracing configuration under Traces. Move Search and Analytics into Core Concepts and remove redundant Advanced hubs. Update canonical paths, backlinks, and redirects from pages published on master. --- docs.json | 110 +++++++++++++--- features/faq.mdx | 6 +- features/image-generation.mdx | 4 +- features/observability.mdx | 83 ------------ features/observability/overview.mdx | 72 ++++++++++ features/observability/request-logs.mdx | 57 ++++++++ .../request-logs}/custom-logging.mdx | 2 + .../request-logs}/metadata.mdx | 8 +- .../request-logs/request-ids.mdx} | 7 +- .../request-logs/scores.mdx} | 5 +- .../request-logs/structured-outputs.mdx} | 10 +- .../request-logs/tags.mdx} | 4 +- .../request-logs}/tracking-templates.mdx | 4 +- .../observability}/traces.mdx | 10 +- .../traces}/auto-instrumentation/openai.mdx | 4 +- .../traces}/auto-instrumentation/overview.mdx | 8 +- .../observability/traces/configuration.mdx | 18 +++ .../traces}/integrations.mdx | 2 +- .../traces}/opentelemetry.mdx | 2 +- .../dynamic-release-labels.mdx | 2 +- features/prompt-registry/release-labels.mdx | 2 +- features/search-and-analytics/analytics.mdx | 60 +++++++++ .../search-data-model.mdx | 8 +- features/search-and-analytics/search.mdx | 63 +++++++++ onboarding-guides/agentic-workflows.mdx | 2 +- onboarding-guides/getting-started.mdx | 2 +- onboarding-guides/observability.mdx | 6 +- overview.mdx | 8 +- reference/close-trace.mdx | 2 +- reference/get-request.mdx | 2 +- reference/get-trace.mdx | 4 +- reference/log-request.mdx | 4 +- reference/otlp-ingest-traces.mdx | 6 +- .../request-analytics-custom-analytics.mdx | 2 +- reference/request-analytics.mdx | 2 +- reference/search-request-logs.mdx | 4 +- reference/search-request-suggestions.mdx | 4 +- reference/spans-bulk.mdx | 4 +- .../trace-analytics-custom-analytics.mdx | 2 +- reference/track-metadata.mdx | 2 +- reference/track-prompt.mdx | 2 +- sdks/evals/agent-tracing.mdx | 6 +- sdks/evals/building-an-eval.mdx | 2 +- sdks/evals/quickstart.mdx | 20 +-- sdks/evals/scorers/overview.mdx | 2 +- sdks/javascript.mdx | 4 +- sdks/python.mdx | 6 +- why-promptlayer/advanced-search.mdx | 123 ------------------ why-promptlayer/analytics.mdx | 34 ----- why-promptlayer/fine-tuning.mdx | 2 +- why-promptlayer/voice-agents.mdx | 8 +- 51 files changed, 465 insertions(+), 351 deletions(-) delete mode 100644 features/observability.mdx create mode 100644 features/observability/overview.mdx create mode 100644 features/observability/request-logs.mdx rename features/{prompt-history => observability/request-logs}/custom-logging.mdx (98%) rename features/{prompt-history => observability/request-logs}/metadata.mdx (83%) rename features/{prompt-history/request-id.mdx => observability/request-logs/request-ids.mdx} (82%) rename features/{prompt-history/scoring-requests.mdx => observability/request-logs/scores.mdx} (84%) rename features/{prompt-history/structured-output-logging.mdx => observability/request-logs/structured-outputs.mdx} (95%) rename features/{prompt-history/tagging-requests.mdx => observability/request-logs/tags.mdx} (85%) rename features/{prompt-history => observability/request-logs}/tracking-templates.mdx (92%) rename {running-requests => features/observability}/traces.mdx (88%) rename features/{ => observability/traces}/auto-instrumentation/openai.mdx (95%) rename features/{ => observability/traces}/auto-instrumentation/overview.mdx (64%) create mode 100644 features/observability/traces/configuration.mdx rename features/{ => observability/traces}/integrations.mdx (96%) rename features/{ => observability/traces}/opentelemetry.mdx (95%) create mode 100644 features/search-and-analytics/analytics.mdx rename features/{prompt-history => search-and-analytics}/search-data-model.mdx (95%) create mode 100644 features/search-and-analytics/search.mdx delete mode 100644 why-promptlayer/advanced-search.mdx delete mode 100644 why-promptlayer/analytics.mdx diff --git a/docs.json b/docs.json index e4f9d5d5..07221f76 100644 --- a/docs.json +++ b/docs.json @@ -89,36 +89,50 @@ "group": "Observability", "icon": "chart-column", "pages": [ - "features/observability", - "running-requests/traces", + "features/observability/overview", { - "group": "SDK Auto-Instrumentation", - "icon": "wave-pulse", + "group": "Request Logs", + "icon": "rectangle-list", "pages": [ - "features/auto-instrumentation/overview", - "features/auto-instrumentation/openai" + "features/observability/request-logs", + "features/observability/request-logs/request-ids", + "features/observability/request-logs/tags", + "features/observability/request-logs/metadata", + "features/observability/request-logs/scores", + "features/observability/request-logs/tracking-templates", + "features/observability/request-logs/custom-logging", + "features/observability/request-logs/structured-outputs" ] }, - "features/integrations", - "features/opentelemetry", - "why-promptlayer/advanced-search", - "why-promptlayer/analytics", { - "group": "Advanced Logging", - "icon": "cassette-tape", + "group": "Traces", + "icon": "diagram-project", "pages": [ - "features/prompt-history/request-id", - "features/prompt-history/tagging-requests", - "features/prompt-history/metadata", - "features/prompt-history/scoring-requests", - "features/prompt-history/tracking-templates", - "features/prompt-history/custom-logging", - "features/prompt-history/structured-output-logging", - "features/prompt-history/search-data-model" + "features/observability/traces", + "features/observability/traces/configuration", + { + "group": "SDK Auto-Instrumentation", + "icon": "code", + "pages": [ + "features/observability/traces/auto-instrumentation/overview", + "features/observability/traces/auto-instrumentation/openai" + ] + }, + "features/observability/traces/integrations", + "features/observability/traces/opentelemetry" ] } ] }, + { + "group": "Search & Analytics", + "icon": "chart-pie-simple", + "pages": [ + "features/search-and-analytics/search", + "features/search-and-analytics/analytics", + "features/search-and-analytics/search-data-model" + ] + }, "why-promptlayer/workflows", { "group": "Tool Registry", @@ -520,6 +534,62 @@ } }, "redirects": [ + { + "source": "/features/observability", + "destination": "/features/observability/overview" + }, + { + "source": "/running-requests/traces", + "destination": "/features/observability/traces" + }, + { + "source": "/features/integrations", + "destination": "/features/observability/traces/integrations" + }, + { + "source": "/features/opentelemetry", + "destination": "/features/observability/traces/opentelemetry" + }, + { + "source": "/features/prompt-history/request-id", + "destination": "/features/observability/request-logs/request-ids" + }, + { + "source": "/features/prompt-history/tagging-requests", + "destination": "/features/observability/request-logs/tags" + }, + { + "source": "/features/prompt-history/metadata", + "destination": "/features/observability/request-logs/metadata" + }, + { + "source": "/features/prompt-history/scoring-requests", + "destination": "/features/observability/request-logs/scores" + }, + { + "source": "/features/prompt-history/tracking-templates", + "destination": "/features/observability/request-logs/tracking-templates" + }, + { + "source": "/features/prompt-history/custom-logging", + "destination": "/features/observability/request-logs/custom-logging" + }, + { + "source": "/features/prompt-history/structured-output-logging", + "destination": "/features/observability/request-logs/structured-outputs" + }, + { + "source": "/features/prompt-history/search-data-model", + "destination": "/features/search-and-analytics/search-data-model" + }, + { + "source": "/why-promptlayer/advanced-search", + "destination": "/features/search-and-analytics/search" + }, + { + "source": "/why-promptlayer/analytics", + "destination": "/features/search-and-analytics/analytics" + }, { "source": "/sdks/evals/setup-with-ai", "destination": "/sdks/evals/quickstart" diff --git a/features/faq.mdx b/features/faq.mdx index e971bd17..7133f8aa 100644 --- a/features/faq.mdx +++ b/features/faq.mdx @@ -15,7 +15,7 @@ Yes, PromptLayer supports multi-modal image models, including `gpt-4-vision-prev To use `gpt-4-vision-preview` with PromptLayer, follow these steps: 1. Ensure you have the PromptLayer and OpenAI Python libraries installed. -2. Use the [`run()` method](/sdks/python#using-the-run-method-recommended) to execute prompts, or use [`log_request`](/features/prompt-history/custom-logging) to log requests made with your own client. +2. Use the [`run()` method](/sdks/python#using-the-run-method-recommended) to execute prompts, or use [`log_request`](/features/observability/request-logs/custom-logging) to log requests made with your own client. 3. Make your request to `gpt-4-vision-preview` with the necessary image inputs, either through image URLs or base64 encoded images. 4. Check the PromptLayer dashboard to see your request logged! @@ -25,7 +25,7 @@ Multi-modal models are also supported in the Prompt Registry, Playground, and Ev ## Do you support OpenAI function calling? -Yes, we take great pride in staying up to date. PromptLayer supports [function calling](https://platform.openai.com/docs/guides/function-calling) through the `run()` method and via [custom logging](/features/prompt-history/custom-logging). You can also configure tool calling directly in the [Prompt Registry](/features/prompt-registry/tool-calling). +Yes, we take great pride in staying up to date. PromptLayer supports [function calling](https://platform.openai.com/docs/guides/function-calling) through the `run()` method and via [custom logging](/features/observability/request-logs/custom-logging). You can also configure tool calling directly in the [Prompt Registry](/features/prompt-registry/tool-calling). ## Does PromptLayer support streaming? @@ -135,7 +135,7 @@ PromptLayer provides out-of-the-box support for Mistral in our logs, playground, ## What's the difference between tags and metadata? -Both [tags](/features/prompt-history/tagging-requests) and [metadata](/features/prompt-history/metadata) enable the addition of supplementary information to your request logs, yet they serve distinct purposes. Tags are ideal for classifying requests into a limited number of predefined categories, such as "prod" or "dev". Conversely, metadata is tailored for capturing unique, request-specific details like user IDs or session IDs. +Both [tags](/features/observability/request-logs/tags) and [metadata](/features/observability/request-logs/metadata) enable the addition of supplementary information to your request logs, yet they serve distinct purposes. Tags are ideal for classifying requests into a limited number of predefined categories, such as "prod" or "dev". Conversely, metadata is tailored for capturing unique, request-specific details like user IDs or session IDs. ## Why do I see extra input variables in my prompt template? Parsing does not seem to be working. diff --git a/features/image-generation.mdx b/features/image-generation.mdx index 92064a7b..f9b2b9d6 100644 --- a/features/image-generation.mdx +++ b/features/image-generation.mdx @@ -337,7 +337,7 @@ PromptLayer automatically handles image storage for generated images: ## Logging Image Generation Requests -If you're making image generation calls with your own client, you can log them to PromptLayer using [`log_request`](/features/prompt-history/custom-logging). PromptLayer recognizes the following function names for image generation: +If you're making image generation calls with your own client, you can log them to PromptLayer using [`log_request`](/features/observability/request-logs/custom-logging). PromptLayer recognizes the following function names for image generation: - `openai.images.generate` - `openai.OpenAI.images.generate` @@ -354,4 +354,4 @@ Image generation outputs work with PromptLayer's evaluation system. Generated im - [Tool Calling (Built-in Tools)](/features/prompt-registry/tool-calling) - [Python SDK Run Method](/sdks/python#using-the-run-method-recommended) - [JavaScript SDK Run Method](/sdks/javascript#using-the-run-method-recommended) -- [Custom Logging](/features/prompt-history/custom-logging) +- [Custom Logging](/features/observability/request-logs/custom-logging) diff --git a/features/observability.mdx b/features/observability.mdx deleted file mode 100644 index db7a19c1..00000000 --- a/features/observability.mdx +++ /dev/null @@ -1,83 +0,0 @@ ---- -title: "Overview" -description: "Understand request logs and traces, then choose how to send telemetry to PromptLayer." -icon: "book" ---- - -Observability helps you inspect model calls and follow the execution of AI applications across prompts, agents, tools, and workflows. - -## Request Logs and Traces - -| Artifact | Use it to | -| --- | --- | -| **Request log** | Inspect one model call, including its input, output, model, timing, tokens, cost, status, metadata, and prompt association. | -| **Trace** | Follow one end-to-end operation as a hierarchy of LLM calls, agents, tools, retrieval steps, and custom application spans. | - -A trace can contain multiple model calls and therefore link to multiple request logs. Supported direct GenAI spans create request logs, while spans managed by `PromptLayer.run()` can link to the run's existing request log. - -## Choose a Tracing Page - -All tracing setup paths send spans to the same PromptLayer trace view. Start with the page that matches the code producing your telemetry: - -| Page | Use it when | What it covers | -| --- | --- | --- | -| [Traces](/running-requests/traces) | You use `PromptLayer.run()` or want to add custom application and tool spans with the PromptLayer SDK. | The trace hierarchy and UI, `traceable`, `wrapWithSpan`, `traceTool`, nesting, and filtering. | -| [SDK Auto-Instrumentation](/features/auto-instrumentation/overview) | Your application calls a supported model provider's SDK directly. | Provider-specific setup that traces SDK calls without replacing the provider client. | -| [Telemetry Integrations](/features/integrations) | A framework, agent SDK, or model router makes the calls. | Integration-specific setup for tools such as OpenAI Agents, Claude Code, Vercel AI SDK, and Pydantic AI. | -| [OpenTelemetry](/features/opentelemetry) | You already have an OpenTelemetry pipeline, use a Collector, or need to instrument an unsupported library. | PromptLayer's OTLP endpoint, GenAI semantic conventions, custom attributes, and Collector configuration. | - - -Use the most specific integration available for the library making the model call. Use the generic OpenTelemetry path when you already own the telemetry pipeline or no dedicated integration applies. These paths create the same kind of PromptLayer traces; they are different setup methods, not different trace products. - - -## View Request Logs - -Click **Logs** in the sidebar to see request history. You can filter by prompt, search by content, inspect errors, and review request details. - - - Request logs - - -You can also view logs for a specific prompt by clicking **Analytics & Logs** in the prompt editor. - - - Logs filtered by prompt - - -From the logs table, select historical requests and add them to a dataset when you want to backtest a prompt change. - -## Explore and Use Observability Data - - - - Track cost, latency, request volume, token usage, models, prompts, tags, and metadata. - - - Find logs by request content, metadata, tags, scores, status, model, prompt, and usage fields. - - - Build evaluation and backtesting Tables from filtered request history and production examples. - - - Add request IDs, metadata, tags, scores, prompt associations, and custom logs from code. - - diff --git a/features/observability/overview.mdx b/features/observability/overview.mdx new file mode 100644 index 00000000..88a0b522 --- /dev/null +++ b/features/observability/overview.mdx @@ -0,0 +1,72 @@ +--- +title: "Overview" +description: "Understand the request logs and traces that make up PromptLayer observability." +icon: "book" +--- + +Observability helps you inspect model calls and follow the execution of AI applications across prompts, agents, tools, and workflows. + +PromptLayer organizes observability data into two artifacts: + +| Artifact | Use it to | +| --- | --- | +| [**Request log**](/features/observability/request-logs) | Inspect one model call, including its input, output, model, timing, tokens, cost, status, metadata, and prompt association. | +| [**Trace**](/features/observability/traces) | Follow one end-to-end operation as a hierarchy of LLM calls, agents, tools, retrieval steps, and custom application spans. | + +A trace can contain multiple model calls and therefore link to multiple request logs. Supported direct GenAI spans create request logs, while spans managed by `PromptLayer.run()` can link to the run's existing request log. + +## Where to Go Next + + + + Review individual model calls, errors, inputs, outputs, usage, and cost. + + + Inspect the parent-child execution path across model calls, tools, agents, and application code. + + + Find request logs or traces with content queries, structured filters, and date ranges. + + + Aggregate the current request-log or trace query into charts. + + + +## Configure Observability + +Once you know which artifact you need, follow the setup path for the code that produces it: + + + + Choose how requests are captured, then add IDs, tags, metadata, scores, and prompt associations. + + + Choose PromptLayer SDK spans, provider auto-instrumentation, a telemetry integration, or OpenTelemetry. + + + +For indexing details, see the [Search Data Model](/features/search-and-analytics/search-data-model). diff --git a/features/observability/request-logs.mdx b/features/observability/request-logs.mdx new file mode 100644 index 00000000..c42a6fc0 --- /dev/null +++ b/features/observability/request-logs.mdx @@ -0,0 +1,57 @@ +--- +title: "Overview" +description: "Inspect individual model calls, including their content, status, usage, cost, and metadata." +icon: "rectangle-list" +--- + +A request log records one model call. Use it to inspect exactly what your application sent and received, diagnose a failure, review latency and token usage, or connect production behavior to a prompt template. + +Each request log can include: + +- Model input and output +- Provider and model +- Start time, end time, and latency +- Input and output tokens +- Estimated cost +- Status and error details +- Prompt template association +- Tags, metadata, scores, and a PromptLayer request ID + +## View Request Logs + +Open **Request Logs** in the PromptLayer sidebar. The Requests table shows the model, prompt, status, timing, token usage, cost, and other fields for each call. Select a row to open its full details. + + + PromptLayer request logs table + + +To review logs for one prompt, open that prompt and select **Analytics & Logs**. + + + Request logs filtered to one prompt + + +## Find a Request + +Use the date range, free-text query, and structured filters above the Requests table. You can filter by fields such as prompt, model, provider, status, tags, metadata, content, latency, tokens, and cost. + +See [Search](/features/search-and-analytics/search) for the dashboard workflow, supported filter categories, and older-data behavior. + +## Request Logs and Traces + +A request log describes one model call. A [trace](/features/observability/traces) describes the complete operation around that call and can contain multiple request logs. + +When PromptLayer receives a supported GenAI span, it creates a request log for the model call and links it to the trace. If `PromptLayer.run()` already created the request log, PromptLayer links the span to that log instead of creating a duplicate. + +## Configure Request Logs + +Use [tags](/features/observability/request-logs/tags), [metadata](/features/observability/request-logs/metadata), and [scores](/features/observability/request-logs/scores) to make requests easier to find and compare. Every log also has a [PromptLayer request ID](/features/observability/request-logs/request-ids) that you can store or use with tracking methods. + +Use [Tracking Templates](/features/observability/request-logs/tracking-templates) to associate provider calls with PromptLayer prompts. For manual provider calls or unsupported models, use [Custom Logging](/features/observability/request-logs/custom-logging). If a provider returns JSON, see [Structured Outputs](/features/observability/request-logs/structured-outputs). + +## Reuse Production Data + +Select request logs from the table and add them to a [PromptLayer Table](/features/tables/overview#import-data). This lets you turn representative production examples or failures into evaluation and backtesting data. diff --git a/features/prompt-history/custom-logging.mdx b/features/observability/request-logs/custom-logging.mdx similarity index 98% rename from features/prompt-history/custom-logging.mdx rename to features/observability/request-logs/custom-logging.mdx index 893bce6f..a080c03f 100644 --- a/features/prompt-history/custom-logging.mdx +++ b/features/observability/request-logs/custom-logging.mdx @@ -4,6 +4,8 @@ icon: "brackets-curly" --- +Custom logging creates [request logs](/features/observability/request-logs) for calls that PromptLayer does not capture automatically. Return to [Configure Request Logging](/features/observability/request-logs) to compare capture and enrichment options. + ## When to Use Custom Logging Use the `log_request` method when: diff --git a/features/prompt-history/metadata.mdx b/features/observability/request-logs/metadata.mdx similarity index 83% rename from features/prompt-history/metadata.mdx rename to features/observability/request-logs/metadata.mdx index 04666e6e..36351506 100644 --- a/features/prompt-history/metadata.mdx +++ b/features/observability/request-logs/metadata.mdx @@ -5,7 +5,9 @@ icon: "brackets-curly" PromptLayer allows you to attach multiple key value pairs as metadata to a request. In the dashboard, you can look up requests and analyze analytics using metadata. -We recommend using this for things like session IDs, user IDs, or error messages. Metadata is useful to help you use the [advanced search](/why-promptlayer/advanced-search) or understand the Analytics page. +Metadata enriches [request logs](/features/observability/request-logs). Return to [Configure Request Logging](/features/observability/request-logs) for the other enrichment options. + +We recommend using this for things like session IDs, user IDs, or error messages. Metadata is useful for [searching request logs](/features/search-and-analytics/search) and analyzing grouped results. ## Add metadata when running a prompt @@ -89,10 +91,10 @@ Once metadata is added, you will then be able to see it in the web UI. ![score](/images/metadata-ui.png) -Metadata is optimized for high-cardinality, request-specific values such as user IDs, session IDs, and error messages. For a smaller set of categories, such as environment, app, feature, or pipeline stage, use [tags](/features/prompt-history/tagging-requests) instead. +Metadata is optimized for high-cardinality, request-specific values such as user IDs, session IDs, and error messages. For a smaller set of categories, such as environment, app, feature, or pipeline stage, use [tags](/features/observability/request-logs/tags) instead. ## Attaching metadata via OpenTelemetry If you instrument your app with OpenTelemetry, you can attach metadata directly from span attributes — no `track.metadata()` call required. PromptLayer automatically maps standard attributes like `user.id` and `gen_ai.conversation.id`, and also reads arbitrary `promptlayer.metadata.` attributes. -See [Attaching User Identity & Metadata](/features/opentelemetry#attaching-user-identity-%26-metadata) in the OpenTelemetry guide for details. +See [Attaching User Identity & Metadata](/features/observability/traces/opentelemetry#attaching-user-identity-%26-metadata) in the OpenTelemetry guide for details. diff --git a/features/prompt-history/request-id.mdx b/features/observability/request-logs/request-ids.mdx similarity index 82% rename from features/prompt-history/request-id.mdx rename to features/observability/request-logs/request-ids.mdx index 225995fb..798397b4 100644 --- a/features/prompt-history/request-id.mdx +++ b/features/observability/request-logs/request-ids.mdx @@ -3,9 +3,11 @@ title: "Request IDs" icon: "id-card" --- -Every PromptLayer log has a unique PromptLayer Request ID (`pl_id`). +Every PromptLayer log has a unique PromptLayer Request ID (`pl_id`). -All tracking in PromptLayer is based on the `pl_request_id`. This identifier is needed to enrich logs with [metadata](/features/prompt-history/metadata), [scores](/features/prompt-history/scoring-requests), [associated prompt templates](/features/prompt-history/tracking-templates), and more. You can also use it to [retrieve the full request payload](/reference/get-request) as a prompt blueprint. +See [Request Logs](/features/observability/request-logs) for the record this ID identifies, or return to [Configure Request Logging](/features/observability/request-logs). + +All tracking in PromptLayer is based on the `pl_request_id`. This identifier is needed to enrich logs with [metadata](/features/observability/request-logs/metadata), [scores](/features/observability/request-logs/scores), [associated prompt templates](/features/observability/request-logs/tracking-templates), and more. You can also use it to [retrieve the full request payload](/reference/get-request) as a prompt blueprint. You can quickly grab a request ID from the web UI as shown below. @@ -91,4 +93,3 @@ const plRequestId = result.request_id; ``` - diff --git a/features/prompt-history/scoring-requests.mdx b/features/observability/request-logs/scores.mdx similarity index 84% rename from features/prompt-history/scoring-requests.mdx rename to features/observability/request-logs/scores.mdx index 5781363f..27a8b934 100644 --- a/features/prompt-history/scoring-requests.mdx +++ b/features/observability/request-logs/scores.mdx @@ -1,10 +1,12 @@ --- -title: "Score Requests" +title: "Scores" icon: "star" --- Every PromptLayer request can be given an integer score 0-100. +Scores enrich [request logs](/features/observability/request-logs). Return to [Configure Request Logging](/features/observability/request-logs) for the other enrichment options. + ![score](/images/score.png) To associate a score with a prompt, you can either do this visually from the dashboard or programmatically. @@ -49,4 +51,3 @@ curl --request POST \ ``` - diff --git a/features/prompt-history/structured-output-logging.mdx b/features/observability/request-logs/structured-outputs.mdx similarity index 95% rename from features/prompt-history/structured-output-logging.mdx rename to features/observability/request-logs/structured-outputs.mdx index 4db3ff29..1911c409 100644 --- a/features/prompt-history/structured-output-logging.mdx +++ b/features/observability/request-logs/structured-outputs.mdx @@ -1,8 +1,10 @@ --- -title: "Logging Structured Outputs" +title: "Structured Outputs" icon: "brackets-curly" --- +Structured-output configuration appears on the resulting [request log](/features/observability/request-logs). Return to [Configure Request Logging](/features/observability/request-logs) for the other logging options. + ## Overview When logging requests that use structured outputs (JSON schemas), you need to include the schema configuration in the `parameters` field of your `/log-request` call. This allows PromptLayer to properly track and display your structured output configurations alongside your request history. @@ -380,7 +382,7 @@ Check your provider's documentation for specific schema requirements. ## See Also - [Structured Outputs in Prompt Registry](/features/prompt-registry/structured-outputs) - Creating prompts with structured outputs -- [Custom Logging Guide](/features/prompt-history/custom-logging) - General guide to logging requests +- [Custom Logging Guide](/features/observability/request-logs/custom-logging) - General guide to logging requests - [Log Request API Reference](/reference/log-request) - Full API specification -- [Metadata Documentation](/features/prompt-history/metadata) - Using metadata for tracking -- [Tagging Requests](/features/prompt-history/tagging-requests) - Organizing requests with tags +- [Metadata Documentation](/features/observability/request-logs/metadata) - Using metadata for tracking +- [Tagging Requests](/features/observability/request-logs/tags) - Organizing requests with tags diff --git a/features/prompt-history/tagging-requests.mdx b/features/observability/request-logs/tags.mdx similarity index 85% rename from features/prompt-history/tagging-requests.mdx rename to features/observability/request-logs/tags.mdx index 83d64b8c..fbd3a855 100644 --- a/features/prompt-history/tagging-requests.mdx +++ b/features/observability/request-logs/tags.mdx @@ -5,6 +5,8 @@ icon: "tag" While using PromptLayer, over time the number of logs will grow, making it difficult to find what you are looking for. Tags are a great way to help keep things organized. +Tags enrich [request logs](/features/observability/request-logs). Return to [Configure Request Logging](/features/observability/request-logs) for the other enrichment options. + Tags can be used for whatever you want, but the top 2 ways are to: 1. Keep track of which application you are working on @@ -50,4 +52,4 @@ And can be filtered by clicking on the tags button by the search-bar: ## -Please note that tags are optimized for categorization based on a small number of predefined options. For request enrichments with n > 1000 options, please use [metadata](/features/prompt-history/metadata) instead. \ No newline at end of file +Please note that tags are optimized for categorization based on a small number of predefined options. For request enrichments with n > 1000 options, please use [metadata](/features/observability/request-logs/metadata) instead. diff --git a/features/prompt-history/tracking-templates.mdx b/features/observability/request-logs/tracking-templates.mdx similarity index 92% rename from features/prompt-history/tracking-templates.mdx rename to features/observability/request-logs/tracking-templates.mdx index aab9ec25..f6e97a85 100644 --- a/features/prompt-history/tracking-templates.mdx +++ b/features/observability/request-logs/tracking-templates.mdx @@ -5,6 +5,8 @@ icon: "chart-scatter-bubble" PromptLayer allows you to track prompt template usage, latency, cost, and more. This is done by associating a request with a prompt template as shown below. +The association appears on the [request log](/features/observability/request-logs). Return to [Configure Request Logging](/features/observability/request-logs) for the other enrichment options. + [Endpoint Reference](/reference/track-prompt) To associate requests with a prompt from the prompt registry, run the code @@ -85,4 +87,4 @@ curl --request POST \ }' ``` - \ No newline at end of file + diff --git a/running-requests/traces.mdx b/features/observability/traces.mdx similarity index 88% rename from running-requests/traces.mdx rename to features/observability/traces.mdx index 8b268126..45b2620d 100644 --- a/running-requests/traces.mdx +++ b/features/observability/traces.mdx @@ -1,5 +1,5 @@ --- -title: "Traces" +title: "Overview" description: "Understand PromptLayer traces and add custom spans with the PromptLayer SDK." icon: "diagram-project" --- @@ -9,7 +9,7 @@ A trace represents one end-to-end operation in your application. It contains a h Supported LLM spans link to PromptLayer request logs, so you can move from the full execution path to the input, output, model, tokens, cost, and metadata for an individual model call. -This page explains the trace hierarchy and PromptLayer SDK span helpers. To choose between provider auto-instrumentation, a framework integration, and a custom OpenTelemetry pipeline, start with the [Observability overview](/features/observability#choose-a-tracing-page). +This page explains the trace hierarchy and PromptLayer SDK span helpers. To choose between provider auto-instrumentation, a framework integration, and a custom OpenTelemetry pipeline, see [Configuration](/features/observability/traces/configuration). ## Inspect a Trace @@ -56,7 +56,7 @@ const result = await pl.run({ For the full `run()` interface, see the [Python SDK](/sdks/python#using-the-run-method-recommended) or [JavaScript SDK](/sdks/javascript#using-the-run-method-recommended). -This setting enables tracing for the PromptLayer client. To trace calls made directly through a provider client such as `OpenAI()`, use [SDK Auto-Instrumentation](/features/auto-instrumentation/overview). +This setting enables tracing for the PromptLayer client. To trace calls made directly through a provider client such as `OpenAI()`, use [SDK Auto-Instrumentation](/features/observability/traces/auto-instrumentation/overview). ## Add Custom Spans @@ -119,7 +119,7 @@ const answerQuestion = pl.wrapWithSpan( ## Trace Tools -Use `traceTool` for tool handlers in a custom agent that does not already have a dedicated [Telemetry Integration](/features/integrations). PromptLayer names the span `Tool: ` and marks it as a tool call. +Use `traceTool` for tool handlers in a custom agent that does not already have a dedicated [Telemetry Integration](/features/observability/traces/integrations). PromptLayer names the span `Tool: ` and marks it as a tool call. ```python Python @@ -146,6 +146,6 @@ In the span detail panel, hover over a top-level string, number, or boolean attr ## Analyze Traces -Trace analytics can aggregate whole traces or individual spans. Use [Analytics](/why-promptlayer/analytics) for charts and the PromptLayer AI assistant. For the exact trace- and span-level fields available through the public API, see [Trace Analytics - Custom Queries](/reference/trace-analytics-custom-analytics). +Trace analytics can aggregate whole traces or individual spans. Use [Analytics](/features/search-and-analytics/analytics) for charts and the PromptLayer AI assistant. For the exact trace- and span-level fields available through the public API, see [Trace Analytics - Custom Queries](/reference/trace-analytics-custom-analytics). To collect traces from an agent runner inside an SDK evaluation, see [Agent Tracing for Evals](/sdks/evals/agent-tracing). diff --git a/features/auto-instrumentation/openai.mdx b/features/observability/traces/auto-instrumentation/openai.mdx similarity index 95% rename from features/auto-instrumentation/openai.mdx rename to features/observability/traces/auto-instrumentation/openai.mdx index 88d32a20..7ecbadca 100644 --- a/features/auto-instrumentation/openai.mdx +++ b/features/observability/traces/auto-instrumentation/openai.mdx @@ -7,7 +7,7 @@ icon: "robot" PromptLayer can auto-instrument the official OpenAI SDK and export supported calls as OpenTelemetry spans. Each supported direct SDK call appears in PromptLayer as both a trace span and an associated request log without changing how you create or use the OpenAI client. -This guide covers the OpenAI model SDK. If you use the OpenAI Agents SDK, follow the [OpenAI Agents SDK integration](/features/integrations#openai-agents-sdk) instead. +This guide covers the OpenAI model SDK. If you use the OpenAI Agents SDK, follow the [OpenAI Agents SDK integration](/features/observability/traces/integrations#openai-agents-sdk) instead. ## Supported APIs @@ -202,7 +202,7 @@ Configure a tracer provider only once and reuse it. If the OpenAI SDK is already ## Verify the Integration -Run one supported OpenAI request, flush tracing, and open [Traces](/running-requests/traces) in PromptLayer. The OpenAI span should have an associated request log. If the call runs inside `PromptLayer.run()`, PromptLayer links the provider span to the existing run request log instead of creating a duplicate. +Run one supported OpenAI request, flush tracing, and open [Traces](/features/observability/traces) in PromptLayer. The OpenAI span should have an associated request log. If the call runs inside `PromptLayer.run()`, PromptLayer links the provider span to the existing run request log instead of creating a duplicate. If no span appears: diff --git a/features/auto-instrumentation/overview.mdx b/features/observability/traces/auto-instrumentation/overview.mdx similarity index 64% rename from features/auto-instrumentation/overview.mdx rename to features/observability/traces/auto-instrumentation/overview.mdx index 20c14e79..6ac5d118 100644 --- a/features/auto-instrumentation/overview.mdx +++ b/features/observability/traces/auto-instrumentation/overview.mdx @@ -1,17 +1,19 @@ --- -title: "SDK Auto-Instrumentation" +title: "Overview" description: "Trace calls made through model provider SDKs without wrapping each request." icon: "wave-pulse" --- Use SDK auto-instrumentation when your application code creates and calls a supported model provider's native client. PromptLayer registers the provider-specific instrumentor and trace exporter, so supported calls become OpenTelemetry spans and linked request logs without replacing the provider client or logging each request manually. +For the resulting hierarchy and dashboard view, see [Traces](/features/observability/traces). + ## Provider Guides | Provider | Languages | Guide | | --- | --- | --- | -| OpenAI | Python and JavaScript | [Auto-Instrument the OpenAI SDK](/features/auto-instrumentation/openai) | +| OpenAI | Python and JavaScript | [Auto-Instrument the OpenAI SDK](/features/observability/traces/auto-instrumentation/openai) | The provider guide is the source of truth for supported API surfaces, installation, initialization order, content capture, exporter settings, flushing, and verification. Coverage and configuration can differ between languages. -If a framework or agent SDK creates the provider client for you, use [Telemetry Integrations](/features/integrations). If no provider guide applies or you already own the telemetry pipeline, use [OpenTelemetry](/features/opentelemetry). See the [Observability overview](/features/observability#choose-a-tracing-page) for the complete routing guide. +If a framework or agent SDK creates the provider client for you, use [Telemetry Integrations](/features/observability/traces/integrations). If no provider guide applies or you already own the telemetry pipeline, use [OpenTelemetry](/features/observability/traces/opentelemetry). See [Configure Tracing](/features/observability/traces/configuration) for the complete routing guide. diff --git a/features/observability/traces/configuration.mdx b/features/observability/traces/configuration.mdx new file mode 100644 index 00000000..eb7aa86f --- /dev/null +++ b/features/observability/traces/configuration.mdx @@ -0,0 +1,18 @@ +--- +title: "Configuration" +description: "Choose how to send traces and linked request logs to PromptLayer." +icon: "diagram-project" +--- + +All tracing setup paths send spans to the same [PromptLayer trace view](/features/observability/traces). Choose the path that matches the code producing telemetry: + +| Setup | Use it when | +| --- | --- | +| [PromptLayer SDK spans](/features/observability/traces#trace-promptlayer-sdk-runs) | You use `PromptLayer.run()` or want to wrap custom application and tool functions. | +| [SDK Auto-Instrumentation](/features/observability/traces/auto-instrumentation/overview) | Your application calls a supported model provider SDK directly. | +| [Telemetry Integrations](/features/observability/traces/integrations) | A framework, agent SDK, or model router makes the calls. | +| [OpenTelemetry](/features/observability/traces/opentelemetry) | You already operate an OpenTelemetry pipeline or no dedicated integration supports your library. | + +Use the most specific supported setup. These are collection methods, not separate trace products. + +After setup, open [Traces](/features/observability/traces) to verify the span hierarchy and linked [request logs](/features/observability/request-logs). diff --git a/features/integrations.mdx b/features/observability/traces/integrations.mdx similarity index 96% rename from features/integrations.mdx rename to features/observability/traces/integrations.mdx index cad67ea3..3083e269 100644 --- a/features/integrations.mdx +++ b/features/observability/traces/integrations.mdx @@ -6,7 +6,7 @@ icon: 'handshake' Use this page when an LLM framework, agent SDK, or model router makes calls or produces telemetry for your application. Each section is the setup guide for that framework or tool. -If your tool is not listed, use [OpenTelemetry](/features/opentelemetry) or [email us](mailto:hello@promptlayer.com). If your code calls a model provider SDK directly, use [SDK Auto-Instrumentation](/features/auto-instrumentation/overview). The [Observability overview](/features/observability#choose-a-tracing-page) compares all tracing paths. +The resulting spans appear in [Traces](/features/observability/traces). If your tool is not listed, use [OpenTelemetry](/features/observability/traces/opentelemetry) or [email us](mailto:hello@promptlayer.com). If your code calls a model provider SDK directly, use [SDK Auto-Instrumentation](/features/observability/traces/auto-instrumentation/overview). [Configure Tracing](/features/observability/traces/configuration) compares all tracing paths. ## LiteLLM diff --git a/features/opentelemetry.mdx b/features/observability/traces/opentelemetry.mdx similarity index 95% rename from features/opentelemetry.mdx rename to features/observability/traces/opentelemetry.mdx index f6ff9a83..4bcda053 100644 --- a/features/opentelemetry.mdx +++ b/features/observability/traces/opentelemetry.mdx @@ -6,7 +6,7 @@ icon: "tower-broadcast" Use this path when your application already emits [OpenTelemetry (OTEL)](https://opentelemetry.io/) spans, you use an OpenTelemetry Collector, or no dedicated PromptLayer integration supports your library. You do not need a PromptLayer SDK. -For a supported model provider SDK or framework, use its dedicated [SDK Auto-Instrumentation](/features/auto-instrumentation/overview) or [Telemetry Integration](/features/integrations) instead. The [Observability overview](/features/observability#choose-a-tracing-page) compares all tracing paths. +Exported spans appear in [Traces](/features/observability/traces). For a supported model provider SDK or framework, use its dedicated [SDK Auto-Instrumentation](/features/observability/traces/auto-instrumentation/overview) or [Telemetry Integration](/features/observability/traces/integrations) instead. [Configure Tracing](/features/observability/traces/configuration) compares all tracing paths. ## OTLP Endpoint diff --git a/features/prompt-registry/dynamic-release-labels.mdx b/features/prompt-registry/dynamic-release-labels.mdx index 6c8001a7..3c8e4f27 100644 --- a/features/prompt-registry/dynamic-release-labels.mdx +++ b/features/prompt-registry/dynamic-release-labels.mdx @@ -33,7 +33,7 @@ This is powered by the A/B Releases feature. When you create an A/B Release, it - Example: Internal employees receive version 4 (dev) 50% of the time. 6. Save the A/B Release. It will now dynamically route traffic for the specified release label. -**Important**: When [logging requests](/features/prompt-history/tracking-templates), make sure to log the specific version returned, not just the release label. The label will always point to the original version in your logs. +**Important**: When [logging requests](/features/observability/request-logs/tracking-templates), make sure to log the specific version returned, not just the release label. The label will always point to the original version in your logs. To stop dynamically routing traffic, simply delete the A/B Release. The release label will revert to its base mapping. diff --git a/features/prompt-registry/release-labels.mdx b/features/prompt-registry/release-labels.mdx index a58b976b..70ffcc39 100644 --- a/features/prompt-registry/release-labels.mdx +++ b/features/prompt-registry/release-labels.mdx @@ -57,4 +57,4 @@ This approach allows you to test prompt changes on a subset of users, compare pe - Remove unused Release Labels to keep your prompt template organized - Use Dynamic Release Labels for more advanced traffic splitting and segmentation -With Release Labels, you can confidently manage prompt template versions and roll out updates without code changes. Combine them with PromptLayer's [versioning](/features/prompt-registry/overview), [analytics](/why-promptlayer/analytics), and [evaluations](/features/evaluations/overview) for a powerful prompt engineering workflow. \ No newline at end of file +With Release Labels, you can confidently manage prompt template versions and roll out updates without code changes. Combine them with PromptLayer's [versioning](/features/prompt-registry/overview), [analytics](/features/search-and-analytics/analytics), and [evaluations](/features/evaluations/overview) for a powerful prompt engineering workflow. diff --git a/features/search-and-analytics/analytics.mdx b/features/search-and-analytics/analytics.mdx new file mode 100644 index 00000000..eec751a3 --- /dev/null +++ b/features/search-and-analytics/analytics.mdx @@ -0,0 +1,60 @@ +--- +title: "Analytics" +description: "Turn a filtered set of request logs or traces into usage, performance, cost, and quality charts." +icon: "chart-pie-simple" +--- + +Analytics turns the current [Search](/features/search-and-analytics/search) query into charts. Use it to move from an individual [request log](/features/observability/request-logs) or [trace](/features/observability/traces) to trends in usage, performance, cost, errors, and application behavior. + +![Analytics Page Screenshot](/images/analytics-page.png) + +## Open Analytics + +1. Open [**Request Logs**](/features/observability/request-logs) or [**Traces**](/features/observability/traces) in the PromptLayer sidebar. +2. Set the date range and add any free-text or structured filters. +3. Switch from the Requests or Traces tab to **Analytics**. +4. Select **Run query** after changing the analytics query. + +The table and Analytics tab use the same query. You can narrow the data first and then chart only the matching request logs or traces. + +## Request Analytics + +Request analytics includes views for: + +- Latency, request volume, tokens, and cost over time +- Requests by model, prompt, and status +- Response format, cached tokens, and reasoning tokens +- Provider, prompt, tag, and metadata cost breakdowns +- Common errors, metadata keys, output keys, and tools + +Use metadata and tags to compare environments, customers, application versions, or other segments that matter to your application. + +## Trace Analytics + +Trace analytics includes views for: + +- End-to-end latency, including average, p50, p90, and p95 +- Trace volume and status over time +- Models and tools used across traces +- Prompt templates and workflows used across traces +- Trace- and span-level filters + +## Interact with Charts + +Many charts can update the current query. Depending on the chart, select a point, bar, or segment to add a filter, or drag across a time-series chart to narrow the date range. Use **Undo** to restore the previous analytics query. + +## Ask for an Analysis + +Open the PromptLayer assistant and ask a question such as, "Create a graph of the most used prompts over the past week." It can inspect request or trace analytics and create a chart from the relevant data. + +![PromptLayer assistant showing request-data insights](/images/wrangler_insights.png) + + +Analytics is unavailable when the selected search range includes data from before your workspace's OpenSearch cutover date. See [Search Older Data](/features/search-and-analytics/search#search-older-data). + + +## Use Analytics through the API + +- [Request Analytics](/reference/request-analytics) +- [Request Analytics - Custom Queries](/reference/request-analytics-custom-analytics) +- [Trace Analytics - Custom Queries](/reference/trace-analytics-custom-analytics) diff --git a/features/prompt-history/search-data-model.mdx b/features/search-and-analytics/search-data-model.mdx similarity index 95% rename from features/prompt-history/search-data-model.mdx rename to features/search-and-analytics/search-data-model.mdx index dc0c975b..d4ea2873 100644 --- a/features/prompt-history/search-data-model.mdx +++ b/features/search-and-analytics/search-data-model.mdx @@ -3,7 +3,7 @@ title: "Search Data Model" icon: "diagram-project" --- -When you log requests through PromptLayer, we process and index the data to make it searchable. Understanding how your data is indexed will help you write more effective filters when using the [Search Request Logs](/reference/search-request-logs) API or the dashboard's advanced search. +When you log requests through PromptLayer, we process and index the data to make it searchable. Understanding how your data is indexed will help you write more effective filters when using the [Search Request Logs](/reference/search-request-logs) API or [Search in the dashboard](/features/search-and-analytics/search). ## How Data Gets Indexed @@ -152,7 +152,7 @@ Nested metadata is also supported. If you attach `{"user": {"id": "abc", "role": Requests logged without an associated prompt template (no `prompt_id`) will have **no** input variables indexed at all. -If you need to filter by values that aren't part of the prompt template, attach them as [metadata](/features/prompt-history/metadata) instead — metadata is always fully indexed. +If you need to filter by values that aren't part of the prompt template, attach them as [metadata](/features/observability/request-logs/metadata) instead — metadata is always fully indexed. For example, if your prompt template uses `{question}` and `{context}`, but you also pass `user_id` as an input variable: @@ -229,5 +229,5 @@ Nested fields require `nested_key` to identify the flattened key to inspect. ## Related - [Search Request Logs API](/reference/search-request-logs) - API reference for filtering -- [Metadata](/features/prompt-history/metadata) - Attaching metadata to requests -- [Advanced Search](/why-promptlayer/advanced-search) - Using search in the dashboard +- [Metadata](/features/observability/request-logs/metadata) - Attaching metadata to requests +- [Search](/features/search-and-analytics/search) - Using search in the dashboard diff --git a/features/search-and-analytics/search.mdx b/features/search-and-analytics/search.mdx new file mode 100644 index 00000000..84f31fe7 --- /dev/null +++ b/features/search-and-analytics/search.mdx @@ -0,0 +1,63 @@ +--- +title: "Search" +description: "Find request logs and traces with structured filters, date ranges, and request-content search." +icon: "magnifying-glass" +--- + +Search narrows [request logs](/features/observability/request-logs) and [traces](/features/observability/traces) to the data you need. The same query also controls the corresponding Analytics view, so you can move from individual records to aggregate trends without rebuilding your filters. + +PromptLayer's current structured search is backed by OpenSearch. The dashboard automatically uses a compatibility mode when your date range includes data from before your workspace's OpenSearch cutover. + +## Search in the Dashboard + +1. Open [**Request Logs**](/features/observability/request-logs) or [**Traces**](/features/observability/traces) from the PromptLayer sidebar. +2. Set the date range and timezone. +3. Select the search bar, then choose a field, operator, and value. +4. Add more filters to narrow the results. +5. Stay on the Requests or Traces tab to inspect matching records, or switch to **Analytics** to chart the same query. + +Multiple structured filters are combined to narrow the result set. Suggested values are based on the selected field and data available in your workspace. + +## Search Request Logs + +Request-log search supports a free-text query across prompt input and model output, plus structured filters in these categories: + +| Category | Example fields | +| --- | --- | +| Prompt and user | Prompt, User ID | +| Model | Model, Provider | +| Labels and metadata | Tags, Metadata | +| Content | Input Text, Output Text, Input Variable, Output, User Intent, Agent Intent | +| Usage | Input Tokens, Output Tokens, Cost, Latency | +| Request | Request ID, Request Status, Error Type | +| Format | Response is JSON, Response is a Tool Call, Tool Name, Request is Part of Trace | + +Nested fields such as metadata, structured output, and input variables let you choose a key before matching its value. Use an exact operator for identifiers and short structured values; use a contains operator for longer text. See [Search Data Model](/features/search-and-analytics/search-data-model) for indexing and operator details. + +## Search Traces + +Trace search separates whole-trace fields from span-level fields: + +| Scope | Example fields | +| --- | --- | +| Trace | Name, status, ID, user, duration, cost, tokens, models, tools, prompts, workflows, span count | +| Span | Name, status, kind, duration, cost, tokens, model, prompt, workflow, tool, input, output, exception, attributes | + +A span-level filter returns traces containing a matching span. This makes it possible to find an end-to-end run by a tool call, model, exception, or custom span attribute anywhere in its hierarchy. + +## Search Older Data + +Each workspace has a search cutover date. When the selected date range starts before that date, the dashboard switches to older-data compatibility mode: + +- Filters that are unavailable for older data are marked as unavailable. +- Incompatible active filters are not applied to the older-data query. +- The Analytics tab is unavailable for that date range. + +Move the start of the date range to the displayed cutover date or later to use the complete OpenSearch filter set and Analytics. + +## Continue from Search Results + +- Select a request or trace to inspect its details. +- Switch to [Analytics](/features/search-and-analytics/analytics) to aggregate the current query. +- Select matching request logs or traces and import them into a [Table](/features/tables/overview#import-data). +- Use the [Search Request Logs API](/reference/search-request-logs) to run structured request-log searches outside the dashboard. diff --git a/onboarding-guides/agentic-workflows.mdx b/onboarding-guides/agentic-workflows.mdx index 4ad0a07d..a79a57c1 100644 --- a/onboarding-guides/agentic-workflows.mdx +++ b/onboarding-guides/agentic-workflows.mdx @@ -135,4 +135,4 @@ To see the traces for a workflow: **Additional Resources:** - [Learn more about workflows](/why-promptlayer/workflows). -- [Add tracing](/running-requests/traces) to your workflows. +- [Add tracing](/features/observability/traces) to your workflows. diff --git a/onboarding-guides/getting-started.mdx b/onboarding-guides/getting-started.mdx index c976dad3..d31bd5a1 100644 --- a/onboarding-guides/getting-started.mdx +++ b/onboarding-guides/getting-started.mdx @@ -97,7 +97,7 @@ Reviewing prompt logs helps you track old requests. PromptLayer stores historica Request Logs -You can search for a specific term or use filters to locate specific prompt log. ([Read more](/why-promptlayer/advanced-search)) +You can search for a specific term or use filters to locate a specific request log. ([Read more](/features/search-and-analytics/search)) **Additional Resources:** diff --git a/onboarding-guides/observability.mdx b/onboarding-guides/observability.mdx index 07f95c74..921482b7 100644 --- a/onboarding-guides/observability.mdx +++ b/onboarding-guides/observability.mdx @@ -103,7 +103,7 @@ Set up logging and tracing within your SDK to capture execution data. This enabl 5. Review the generated logs to analyze metrics like execution time, token usage, and cost, then use these insights to fine-tune your prompt. -To read more about logging, check out the [Metadata](/features/prompt-history/metadata) and [Tagging Requests](/features/prompt-history/tagging-requests) guides. +To read more about logging, check out the [Metadata](/features/observability/request-logs/metadata) and [Tagging Requests](/features/observability/request-logs/tags) guides. --- @@ -130,5 +130,5 @@ You can also open these logs in the Playground, share them with your team, and a **Additional Resources:** - For more on running prompts, see the [Python SDK](/sdks/python#using-the-run-method-recommended) and [JavaScript SDK](/sdks/javascript#using-the-run-method-recommended) guides. -- For more on Logging, check out our [Advanced Logging](/features/prompt-history/request-id) guide. -- To learn more about filtering logs, check out the [Advanced Search](/why-promptlayer/advanced-search#advanced-search) section of the Quickstart guide. +- For logging configuration and enrichment, see [Request Logs](/features/observability/request-logs). +- To learn more about filtering logs, see [Search](/features/search-and-analytics/search). diff --git a/overview.mdx b/overview.mdx index 43771ec4..45c1af21 100644 --- a/overview.mdx +++ b/overview.mdx @@ -23,7 +23,7 @@ mode: wide

- + Connect observability @@ -34,7 +34,7 @@ mode: wide
Explore - Observability + Observability Tables Prompt Registry API @@ -118,7 +118,7 @@ mode: wide
- + 01
@@ -214,7 +214,7 @@ mode: wide Webhooks React to PromptLayer events - + OpenTelemetry Connect tracing pipelines and providers diff --git a/reference/close-trace.mdx b/reference/close-trace.mdx index 153944f5..e3f7f665 100644 --- a/reference/close-trace.mdx +++ b/reference/close-trace.mdx @@ -16,4 +16,4 @@ Marks a trace as closed, preventing any further spans from being written to it. - [Get Trace](/reference/get-trace) - [Create Spans Bulk](/reference/spans-bulk) - [Ingest Traces (OTLP)](/reference/otlp-ingest-traces) -- [Traces](/running-requests/traces) +- [Traces](/features/observability/traces) diff --git a/reference/get-request.mdx b/reference/get-request.mdx index 62bc8c1e..16c43f5b 100644 --- a/reference/get-request.mdx +++ b/reference/get-request.mdx @@ -7,6 +7,6 @@ Retrieve the full payload of a logged request by ID. The response includes a pro ## Related -- [Request IDs](/features/prompt-history/request-id) +- [Request IDs](/features/observability/request-logs/request-ids) - [Prompt Blueprints](/running-requests/prompt-blueprints) - [Get Trace](/reference/get-trace) diff --git a/reference/get-trace.mdx b/reference/get-trace.mdx index b02e6b9e..620a7b9a 100644 --- a/reference/get-trace.mdx +++ b/reference/get-trace.mdx @@ -8,5 +8,5 @@ Retrieve all spans for a trace ID. Each span includes its metadata and, when the ## Related - [Get Request](/reference/get-request) -- [Traces](/running-requests/traces) -- [OpenTelemetry](/features/opentelemetry) +- [Traces](/features/observability/traces) +- [OpenTelemetry](/features/observability/traces/opentelemetry) diff --git a/reference/log-request.mdx b/reference/log-request.mdx index 4b0e524d..4cec51de 100644 --- a/reference/log-request.mdx +++ b/reference/log-request.mdx @@ -14,6 +14,6 @@ Log a request made outside of PromptLayer's managed run APIs. Use this endpoint ## Related -- [Custom Logging](/features/prompt-history/custom-logging) +- [Custom Logging](/features/observability/request-logs/custom-logging) - [Prompt Blueprints](/running-requests/prompt-blueprints) -- [Structured Output Logging](/features/prompt-history/structured-output-logging) +- [Structured Output Logging](/features/observability/request-logs/structured-outputs) diff --git a/reference/otlp-ingest-traces.mdx b/reference/otlp-ingest-traces.mdx index bf2f5d01..7049e6d7 100644 --- a/reference/otlp-ingest-traces.mdx +++ b/reference/otlp-ingest-traces.mdx @@ -13,10 +13,10 @@ Ingest OpenTelemetry traces through PromptLayer's OTLP/HTTP endpoint. - Gzip `Content-Encoding` is supported for both formats. - Spans can include `promptlayer.prompt.name`, optionally with `promptlayer.prompt.version`, to link the generated request log to an existing prompt template in your workspace. - Spans can include `user.id`/`enduser.id`, `gen_ai.conversation.id`/`session.id`, and `promptlayer.metadata.*` attributes to attach searchable user identity and metadata to the generated request log. -- For SDK setup, GenAI semantic conventions, prompt template linking, metadata, and collector configuration, see [OpenTelemetry](/features/opentelemetry). +- For SDK setup, GenAI semantic conventions, prompt template linking, metadata, and collector configuration, see [OpenTelemetry](/features/observability/traces/opentelemetry). ## Related -- [OpenTelemetry](/features/opentelemetry) -- [Traces](/running-requests/traces) +- [OpenTelemetry](/features/observability/traces/opentelemetry) +- [Traces](/features/observability/traces) - [Create Spans Bulk](/reference/spans-bulk) diff --git a/reference/request-analytics-custom-analytics.mdx b/reference/request-analytics-custom-analytics.mdx index 0a4930d0..cb6638c1 100644 --- a/reference/request-analytics-custom-analytics.mdx +++ b/reference/request-analytics-custom-analytics.mdx @@ -65,4 +65,4 @@ Each entry in the response `customCharts` array contains: - [Request Analytics](/reference/request-analytics) - [Search Request Logs](/reference/search-request-logs) -- [Analytics](/why-promptlayer/analytics) +- [Analytics](/features/search-and-analytics/analytics) diff --git a/reference/request-analytics.mdx b/reference/request-analytics.mdx index cce2517f..b5df0f21 100644 --- a/reference/request-analytics.mdx +++ b/reference/request-analytics.mdx @@ -15,4 +15,4 @@ Get aggregated analytics for request logs using the same filter syntax as [Searc - [Search Request Logs](/reference/search-request-logs) - [Search Request Suggestions](/reference/search-request-suggestions) -- [Analytics](/why-promptlayer/analytics) +- [Analytics](/features/search-and-analytics/analytics) diff --git a/reference/search-request-logs.mdx b/reference/search-request-logs.mdx index 1d281bbe..278c23e9 100644 --- a/reference/search-request-logs.mdx +++ b/reference/search-request-logs.mdx @@ -10,10 +10,10 @@ Search logged requests using structured filters, free-text search, sorting, and - This endpoint is rate limited to 10 requests per minute. - Results are capped at 25 items per page. - Use `filter_group` for structured filters and `q` for fuzzy prefix search across prompt input and LLM output text. -- Search indexing, nested fields, and operator behavior are explained in [Search Data Model](/features/prompt-history/search-data-model). +- Search indexing, nested fields, and operator behavior are explained in [Search Data Model](/features/search-and-analytics/search-data-model). ## Related - [Search Request Suggestions](/reference/search-request-suggestions) - [Request Analytics](/reference/request-analytics) -- [Search Data Model](/features/prompt-history/search-data-model) +- [Search Data Model](/features/search-and-analytics/search-data-model) diff --git a/reference/search-request-suggestions.mdx b/reference/search-request-suggestions.mdx index b4f3b308..c412e00c 100644 --- a/reference/search-request-suggestions.mdx +++ b/reference/search-request-suggestions.mdx @@ -14,5 +14,5 @@ Get autocomplete suggestions for request log fields. Use this endpoint to power ## Related - [Search Request Logs](/reference/search-request-logs) -- [Search Data Model](/features/prompt-history/search-data-model) -- [Advanced Search](/why-promptlayer/advanced-search) +- [Search Data Model](/features/search-and-analytics/search-data-model) +- [Search](/features/search-and-analytics/search) diff --git a/reference/spans-bulk.mdx b/reference/spans-bulk.mdx index 41b37ade..0620ca0a 100644 --- a/reference/spans-bulk.mdx +++ b/reference/spans-bulk.mdx @@ -15,5 +15,5 @@ Create multiple observability spans in one request, optionally creating a reques ## Related - [Ingest Traces (OTLP)](/reference/otlp-ingest-traces) -- [Traces](/running-requests/traces) -- [OpenTelemetry](/features/opentelemetry) +- [Traces](/features/observability/traces) +- [OpenTelemetry](/features/observability/traces/opentelemetry) diff --git a/reference/trace-analytics-custom-analytics.mdx b/reference/trace-analytics-custom-analytics.mdx index 38fdf334..f3c7d6fd 100644 --- a/reference/trace-analytics-custom-analytics.mdx +++ b/reference/trace-analytics-custom-analytics.mdx @@ -42,5 +42,5 @@ Identical to [Request Analytics — Custom Queries](/reference/request-analytics ## Related -- [Traces](/running-requests/traces) +- [Traces](/features/observability/traces) - [Request Analytics — Custom Queries](/reference/request-analytics-custom-analytics) diff --git a/reference/track-metadata.mdx b/reference/track-metadata.mdx index 0cb2a5eb..70f2524f 100644 --- a/reference/track-metadata.mdx +++ b/reference/track-metadata.mdx @@ -8,5 +8,5 @@ Associate metadata with an existing request log. Use this for values such as ses ## Related - [Track Prompt](/reference/track-prompt) -- [Metadata](/features/prompt-history/metadata) +- [Metadata](/features/observability/request-logs/metadata) - [Log Request](/reference/log-request) diff --git a/reference/track-prompt.mdx b/reference/track-prompt.mdx index d97f1cc7..d3c6e20a 100644 --- a/reference/track-prompt.mdx +++ b/reference/track-prompt.mdx @@ -8,5 +8,5 @@ Associate a prompt template with an existing request log. Use this after logging ## Related - [Track Metadata](/reference/track-metadata) -- [Tracking Templates](/features/prompt-history/tracking-templates) +- [Tracking Templates](/features/observability/request-logs/tracking-templates) - [Log Request](/reference/log-request) diff --git a/sdks/evals/agent-tracing.mdx b/sdks/evals/agent-tracing.mdx index 204009c7..40651a44 100644 --- a/sdks/evals/agent-tracing.mdx +++ b/sdks/evals/agent-tracing.mdx @@ -30,8 +30,8 @@ Tool spans matter for exactly one thing: the [Trajectory](/sdks/evals/scorers/ov You don't create those spans by hand. Emit them with either: -- **A framework helper** — OpenAI Agents, Claude, Vercel AI, and the rest in [Telemetry Integrations](/features/integrations) — which instruments your tools automatically. -- **`traceTool`** on a custom agent's tool handlers. It records the `Tool: ` span for you (on a tracing-enabled client); see [Tracing Tools](/running-requests/traces#tracing-tools) for the full contract. +- **A framework helper** — OpenAI Agents, Claude, Vercel AI, and the rest in [Telemetry Integrations](/features/observability/traces/integrations) — which instruments your tools automatically. +- **`traceTool`** on a custom agent's tool handlers. It records the `Tool: ` span for you (on a tracing-enabled client); see [Tracing Tools](/features/observability/traces#tracing-tools) for the full contract. Only the tools you assert on (the names in Trajectory `expected`) need a span — a helper tool you don't score can stay untraced. How the modes treat what they observe: @@ -44,7 +44,7 @@ Only the tools you assert on (the names in Trajectory `expected`) need a span | Issue | Fix | | --- | --- | -| Empty Trace / Trajectory fails | Emit `Tool:` spans for the tools you assert on — a framework helper or [`traceTool`](/running-requests/traces#tracing-tools) on a tracing-enabled client | +| Empty Trace / Trajectory fails | Emit `Tool:` spans for the tools you assert on — a framework helper or [`traceTool`](/features/observability/traces#tracing-tools) on a tracing-enabled client | | Traced tool missing in `strict` mode | Add it to the expected list, or switch to `non_strict` | | Streamed return | Collect the final answer before returning it | | Awaitable on the sync Python path | Use `aevaluate` | diff --git a/sdks/evals/building-an-eval.mdx b/sdks/evals/building-an-eval.mdx index 2c477fad..b80b7353 100644 --- a/sdks/evals/building-an-eval.mdx +++ b/sdks/evals/building-an-eval.mdx @@ -52,7 +52,7 @@ Cases feed the runner and scorers. [Datasets](/sdks/evals/datasets) covers inlin The `runner` is any callable that takes an `input` and returns a final value (not a stream) — no framework allowlist. -For Trajectory, tools must appear as `Tool: ` spans on the imported Trace; use a framework helper or [`traceTool`](/running-requests/traces#tracing-tools). The runner and tool-span rules live on [Runner](/sdks/evals/agent-tracing). +For Trajectory, tools must appear as `Tool: ` spans on the imported Trace; use a framework helper or [`traceTool`](/features/observability/traces#tracing-tools). The runner and tool-span rules live on [Runner](/sdks/evals/agent-tracing). ## Columns and scorers diff --git a/sdks/evals/quickstart.mdx b/sdks/evals/quickstart.mdx index d274ec95..584d9b93 100644 --- a/sdks/evals/quickstart.mdx +++ b/sdks/evals/quickstart.mdx @@ -171,7 +171,7 @@ npm install promptlayer @openai/agents zod ``` -Call `instrument_openai_agents` / `instrumentOpenAIAgents` once at the top of your file, then run your Agent from the eval `runner`. Tool spans are collected automatically — no `traceTool` needed. The same helper works outside evals via the [OpenAI Agents SDK](/features/integrations#openai-agents-sdk) integration. +Call `instrument_openai_agents` / `instrumentOpenAIAgents` once at the top of your file, then run your Agent from the eval `runner`. Tool spans are collected automatically — no `traceTool` needed. The same helper works outside evals via the [OpenAI Agents SDK](/features/observability/traces/integrations#openai-agents-sdk) integration. ```python Python @@ -272,7 +272,7 @@ npm install promptlayer @anthropic-ai/claude-agent-sdk ``` -Call `get_claude_config` / `getClaudeConfig` **inside** the runner so the Claude session nests under the eval span. Pass `plugin` and `env` into `ClaudeAgentOptions`. The same helper works outside evals via the [Claude Code](/features/integrations#claude-code) integration. +Call `get_claude_config` / `getClaudeConfig` **inside** the runner so the Claude session nests under the eval span. Pass `plugin` and `env` into `ClaudeAgentOptions`. The same helper works outside evals via the [Claude Code](/features/observability/traces/integrations#claude-code) integration. ```python Python @@ -389,7 +389,7 @@ Install: npm install promptlayer ai @ai-sdk/openai zod ``` -Enable `experimental_telemetry` on the AI SDK call. `evaluate(...)` already registers PromptLayer's OpenTelemetry exporter, so tool spans nest under the eval case — no separate `NodeSDK` setup for the eval path. For app-wide OTEL outside evals, follow the [Vercel AI SDK](/features/integrations#vercel-ai-sdk) integration. +Enable `experimental_telemetry` on the AI SDK call. `evaluate(...)` already registers PromptLayer's OpenTelemetry exporter, so tool spans nest under the eval case — no separate `NodeSDK` setup for the eval path. For app-wide OTEL outside evals, follow the [Vercel AI SDK](/features/observability/traces/integrations#vercel-ai-sdk) integration. ```typescript // evals/vercel_weather.eval.ts @@ -443,7 +443,7 @@ Install: pip install promptlayer litellm ``` -Enable PromptLayer tracing with `PromptLayer(enable_tracing=True)`, wrap tools with [`traceTool`](/running-requests/traces#tracing-tools) for Trajectory, and set LiteLLM's PromptLayer callback for request logging. LiteLLM does not emit separate `Tool:` spans on its own — `traceTool` is a no-op unless that `PromptLayer` instance has tracing enabled. +Enable PromptLayer tracing with `PromptLayer(enable_tracing=True)`, wrap tools with [`traceTool`](/features/observability/traces#tracing-tools) for Trajectory, and set LiteLLM's PromptLayer callback for request logging. LiteLLM does not emit separate `Tool:` spans on its own — `traceTool` is a no-op unless that `PromptLayer` instance has tracing enabled. ```python # evals/litellm_weather.eval.py @@ -522,11 +522,11 @@ evaluate( ) ``` -`evaluate(...)` enables tracing on its own client for nesting under each eval case, but your separate `PromptLayer(...)` instance still needs `enable_tracing=True` for `@pl.traceTool`. Callbacks: [LiteLLM](/features/integrations#litellm) and the [LiteLLM PromptLayer docs](https://docs.litellm.ai/docs/observability/promptlayer_integration). +`evaluate(...)` enables tracing on its own client for nesting under each eval case, but your separate `PromptLayer(...)` instance still needs `enable_tracing=True` for `@pl.traceTool`. Callbacks: [LiteLLM](/features/observability/traces/integrations#litellm) and the [LiteLLM PromptLayer docs](https://docs.litellm.ai/docs/observability/promptlayer_integration). -PromptLayer ingests LangChain spans through the [LangSmith OpenTelemetry bridge](/features/integrations#langchain-/-langsmith) to `https://api.promptlayer.com/v1/traces`. +PromptLayer ingests LangChain spans through the [LangSmith OpenTelemetry bridge](/features/observability/traces/integrations#langchain-/-langsmith) to `https://api.promptlayer.com/v1/traces`. Install (JavaScript): @@ -546,7 +546,7 @@ OTEL_EXPORTER_OTLP_ENDPOINT=https://api.promptlayer.com/v1/traces OTEL_EXPORTER_OTLP_HEADERS=X-API-KEY= ``` -Register the OTEL provider (same pattern as [Integrations](/features/integrations#langchain-/-langsmith)), then evaluate your agent: +Register the OTEL provider (same pattern as [Integrations](/features/observability/traces/integrations#langchain-/-langsmith)), then evaluate your agent: ```typescript // evals/langchain_weather.eval.ts @@ -601,7 +601,7 @@ await evaluate("langchain-weather-eval", { }); ``` -Reuse your real LangChain entrypoint as `runner` when you have one. Register OTEL once for the app using [LangChain / LangSmith](/features/integrations#langchain-/-langsmith). +Reuse your real LangChain entrypoint as `runner` when you have one. Register OTEL once for the app using [LangChain / LangSmith](/features/observability/traces/integrations#langchain-/-langsmith). @@ -662,7 +662,7 @@ evaluate( ) ``` -The same Logfire → `/v1/traces` path works outside evals via the [Pydantic AI](/features/integrations#pydantic-ai) integration. +The same Logfire → `/v1/traces` path works outside evals via the [Pydantic AI](/features/observability/traces/integrations#pydantic-ai) integration. @@ -695,7 +695,7 @@ evaluate( ) ``` -Plugin install and config steps are in the [OpenClaw](/features/integrations#openclaw) integration. +Plugin install and config steps are in the [OpenClaw](/features/observability/traces/integrations#openclaw) integration. diff --git a/sdks/evals/scorers/overview.mdx b/sdks/evals/scorers/overview.mdx index 142ddaf4..cf0c267e 100644 --- a/sdks/evals/scorers/overview.mdx +++ b/sdks/evals/scorers/overview.mdx @@ -593,7 +593,7 @@ await evaluate("llm-assert-demo", { ### Trajectory -Scores `Tool: ` spans from the imported `Trace` against accepted scenarios. Any `runner` works if those spans exist — emit them with a framework helper or [`traceTool`](/running-requests/traces#tracing-tools) (see [Runner](/sdks/evals/agent-tracing)), or copy a harness from the [Quickstart](/sdks/evals/quickstart). +Scores `Tool: ` spans from the imported `Trace` against accepted scenarios. Any `runner` works if those spans exist — emit them with a framework helper or [`traceTool`](/features/observability/traces#tracing-tools) (see [Runner](/sdks/evals/agent-tracing)), or copy a harness from the [Quickstart](/sdks/evals/quickstart). Pass **exactly one of** inline `expected` (list of tool-name lists), or `expected_column` / `expectedColumn` (usually `"expected_trace"` / `"expectedTrace"`). diff --git a/sdks/javascript.mdx b/sdks/javascript.mdx index 69c56c6b..532a0b2f 100644 --- a/sdks/javascript.mdx +++ b/sdks/javascript.mdx @@ -18,7 +18,7 @@ npm install promptlayer ## OpenAI SDK Auto-Instrumentation -PromptLayer can automatically trace supported direct OpenAI SDK calls. See [OpenAI SDK Auto-Instrumentation](/features/auto-instrumentation/openai) for the preload command, supported APIs, content capture, and flushing. +PromptLayer can automatically trace supported direct OpenAI SDK calls. See [OpenAI SDK Auto-Instrumentation](/features/observability/traces/auto-instrumentation/openai) for the preload command, supported APIs, content capture, and flushing. ## SDK evals @@ -382,7 +382,7 @@ await plClient.logRequest({ }); ``` -See the [Custom Logging documentation](/features/prompt-history/custom-logging) and [Log Request API Reference](/reference/log-request) for full details. +See the [Custom Logging documentation](/features/observability/request-logs/custom-logging) and [Log Request API Reference](/reference/log-request) for full details. ## Error Handling diff --git a/sdks/python.mdx b/sdks/python.mdx index c6e59fac..037240e9 100644 --- a/sdks/python.mdx +++ b/sdks/python.mdx @@ -18,7 +18,7 @@ pip install promptlayer ## OpenAI SDK Auto-Instrumentation -PromptLayer can automatically trace supported direct OpenAI SDK calls. See [OpenAI SDK Auto-Instrumentation](/features/auto-instrumentation/openai) for the tracing extra, initialization order, supported APIs, content capture, and flushing. +PromptLayer can automatically trace supported direct OpenAI SDK calls. See [OpenAI SDK Auto-Instrumentation](/features/observability/traces/auto-instrumentation/openai) for the tracing extra, initialization order, supported APIs, content capture, and flushing. ## SDK evals @@ -452,7 +452,7 @@ pl_client.log_request( ) ``` -See the [Custom Logging documentation](/features/prompt-history/custom-logging) and [Log Request API Reference](/reference/log-request) for full details. +See the [Custom Logging documentation](/features/observability/request-logs/custom-logging) and [Log Request API Reference](/reference/log-request) for full details. ## Error Handling @@ -688,7 +688,7 @@ async def main(): asyncio.run(main()) ``` -For more information on custom logging, please visit our [Custom Logging Documentation](/features/prompt-history/custom-logging). +For more information on custom logging, please visit our [Custom Logging Documentation](/features/observability/request-logs/custom-logging). #### Example 4: Asynchronous Prompt Execution with run Method diff --git a/why-promptlayer/advanced-search.mdx b/why-promptlayer/advanced-search.mdx deleted file mode 100644 index 3be3bce2..00000000 --- a/why-promptlayer/advanced-search.mdx +++ /dev/null @@ -1,123 +0,0 @@ ---- -title: "Advanced Search" -icon: "magnifying-glass" ---- - -PromptLayer advanced search capabilities allows you to find exactly what you want using tags, search queries, metadata, favorites, and score filtering. - -## Using the Search Bar - -To start your search, enter the keywords you want to find into the search bar and click on the "Search" button. You can use freeform search to find any text within the PromptLayer. - -![Advanced Search](/images/advanced-search.png) - -## Advanced Search Filters - -#### Metadata Search - -Use the metadata search filter to search for specific metadata within the PromptLayer. You can search for user IDs, session IDs, tokens, error messages, status codes, and other metadata by entering the metadata field name and value into the search bar. - -PromptLayer allows you to attach multiple key value pairs as metadata to a request. In the dashboard, you can look up requests and analyze analytics using metadata. The method for adding metadata to a request can be found in our documentation [here](/features/prompt-history/metadata). - - - -```python Python -promptlayer_client.track.metadata( - request_id=pl_request_id, - metadata={ - "user_id":"1abf2345f", - "session_id": "2cef2345f", - "error_message": "None" - } -) -``` - -```js JavaScript -promptLayerClient.track.metadata({ - request_id:pl_request_id, - metadata:{ - "user_id":"1abf2345f", - "session_id": "2cef2345f", - "error_message": "None" - } -}) -``` - - - -The metadata search filter works by clicking on "Key" in the advanced search filter, selecting the desired metadata key (in this case, user_id), selecting the relevant value under "Value", and clicking "Add filter". - - - -#### Score Filtering - -Use the score filtering feature to search for prompts based on their scores. You can filter prompts by selecting the score range in the "Score" dropdown. - -Score filtering is a powerful tool for analyzing the performance of your prompts. You can use it to identify high-performing prompts, or to find prompts that may need improvement. - - - -Below is an example of how you can score a request programmatically. It can also be done through the dashboard as shown [here](/features/prompt-history/scoring-requests). - - - - -```python Python -promptlayer_client.track.score( - request_id=pl_request_id, - score_name="summarization", # optional score name - score=100 -) -``` - -```js JavaScript -promptLayerClient.track.score({ - request_id: pl_request_id, - score: 100 -}) -``` - - - - -#### Tags Search - -Use the tags search filter to search for specific tags within the PromptLayer. - -Tags are used to group product features, prod/dev versions, and other categories. You can search for tags by selecting them in the "Tags" dropdown. - -Tagging a request is easy. Read more about it [here](/features/prompt-history/tagging-requests). - - - - -```python Python -from promptlayer import PromptLayer -pl_client = PromptLayer() - -response = pl_client.run( - prompt_name="my-prompt", - input_variables={"name": "world"}, - tags=["mytag1", "mytag2"] -) -``` - -```js JavaScript -import { PromptLayer } from "promptlayer"; -const plClient = new PromptLayer(); - -const response = await plClient.run({ - promptName: "my-prompt", - inputVariables: { name: "world" }, - tags: ["mytag1", "mytag2"] -}); -``` - - - - -#### Favorites - -By selecting the "favorite" tag, you can narrow by favorited requests. To favorite a request, click the star on the top right on the dashboard. - -![Favorites](/images/favorites.png) diff --git a/why-promptlayer/analytics.mdx b/why-promptlayer/analytics.mdx deleted file mode 100644 index 8afcdf0e..00000000 --- a/why-promptlayer/analytics.mdx +++ /dev/null @@ -1,34 +0,0 @@ ---- -title: "Analytics" -icon: "chart-pie-simple" ---- - -The Analytics page provides valuable insights into the performance and usage of your application. By leveraging various features and metrics, you can make data-driven decisions to optimize your application and enhance user experience. This documentation will guide you through the different analytics features available. - -![Analytics Page Screenshot](/images/analytics-page.png) - -## Metrics - -Here you can find key performance indicators to assess your application's performance and track its usage. Metrics include: average latency, total cost, and total requests. These metrics provide valuable information on response time, financial impact, and usage volume. - -## Analyzing Usage Patterns - -Understanding usage patterns is crucial for optimizing your application and improving user experience. Analyzing usage patterns involves exploring prompt registry states, model distributions, tokens and requests over time, latency and cost analytics, and prompt template overall costs. These features provide insights into how prompts, models, and resources are utilized, helping you make informed decisions to enhance your application's performance. - -## Ask Wrangler for request-data insights - -If you want a quick read on request usage, open the Wrangler assistant in the bottom-right of the screen and ask directly (for example: "Create a graph of most used prompts over the past week"). Wrangler can probe request analytics first, confirm which prompts have traffic, and then build a chart view from that data. - -![Wrangler insights in Prompt usage analysis](/images/wrangler_insights.png) - -## Filtering and Organization - -To streamline your analysis, the analytics page offers filtering options based on metadata and tags. - -### Filtering by Metadata -![Analytics Page Metadata Filter Screenshot](/images/analytics-metadata.png) -You can filter the analytics page using [metadata attributes](/features/prompt-history/metadata) such as user ID, location, version, and more. This allows you to narrow down the data and focus on specific segments for in-depth analysis. - -### Tag Filtering - -[Tag filtering](/features/prompt-history/tagging-requests) allows you to categorize and organize your requests based on specific tags you assign. It simplifies the process of analyzing specific groups of requests, making it easier to identify trends and patterns. \ No newline at end of file diff --git a/why-promptlayer/fine-tuning.mdx b/why-promptlayer/fine-tuning.mdx index e3c66f7e..fa8c8309 100644 --- a/why-promptlayer/fine-tuning.mdx +++ b/why-promptlayer/fine-tuning.mdx @@ -39,7 +39,7 @@ For example, to generate fine-tuning data you can run a prompt template from the Use the sidebar search area to filter for your training data. All the data that appears from that search query will be used to fine-tune. -[Learn more about search filters](/why-promptlayer/advanced-search) +[Learn more about search filters](/features/search-and-analytics/search)
![Select Training Data](/images/fine-tuning/select-training.gif) diff --git a/why-promptlayer/voice-agents.mdx b/why-promptlayer/voice-agents.mdx index 092d360d..ba213a01 100644 --- a/why-promptlayer/voice-agents.mdx +++ b/why-promptlayer/voice-agents.mdx @@ -11,7 +11,7 @@ Building a production-ready voice agent (like an after-hours appointment assista - **[Prompt Engineering & Version Control](/features/prompt-registry/overview)**: Iterate rapidly on conversation prompts without code deployments - **[Multi-Step Workflow Design](/why-promptlayer/workflows)**: Build complex voice agent logic with visual drag-and-drop interfaces -- **Comprehensive [Observability](/why-promptlayer/analytics)**: Track every interaction with full context of what was said and how the agent responded +- **Comprehensive [Observability](/features/observability/overview)**: Track every interaction with full context of what was said and how the agent responded - **[Rigorous Evaluation](/features/evaluations/overview)**: Test conversation flows, measure quality, and catch issues before they reach customers - **Cost Optimization**: Monitor token usage and latency across all voice interactions @@ -155,7 +155,7 @@ Run your agent against each test case and use PromptLayer's evaluation types inc ### 3. Human Feedback Integration -For production calls, capture customer satisfaction scores using the [Scoring API](/features/prompt-history/scoring-requests): +For production calls, capture customer satisfaction scores using the [Scoring API](/features/observability/request-logs/scores): @@ -196,7 +196,7 @@ PromptLayer evaluations automatically track and display latency for each request ## Observability for Voice Interactions -PromptLayer's [Observability](/features/observability) suite gives you full visibility into every voice interaction, even though the audio itself flows through external services. +PromptLayer's [Observability](/features/observability/overview) suite gives you full visibility into every voice interaction, even though the audio itself flows through external services. ### What You Can Track @@ -208,7 +208,7 @@ PromptLayer's [Observability](/features/observability) suite gives you full visi ### Traces for Multi-Step Workflows -When using PromptLayer Agents for voice workflows, [traces](/running-requests/traces) show each step: +When using PromptLayer Agents for voice workflows, [traces](/features/observability/traces) show each step: ``` Voice Call Trace #1234 From 7126689d7a0de6bbf8ae14955d7be4cba1fc347f Mon Sep 17 00:00:00 2001 From: C P Date: Wed, 29 Jul 2026 14:49:50 -0400 Subject: [PATCH 3/4] docs: expand provider SDK auto-instrumentation Add Anthropic and Google guides alongside expanded OpenAI coverage. Document provider selection, content capture, exporter options, and lifecycle. Clarify tracing navigation and integration discovery. --- docs.json | 10 +- features/observability/overview.mdx | 2 +- features/observability/traces.mdx | 133 ++-------------- .../traces/auto-instrumentation/anthropic.mdx | 150 ++++++++++++++++++ .../traces/auto-instrumentation/google.mdx | 150 ++++++++++++++++++ .../traces/auto-instrumentation/openai.mdx | 30 ++-- .../traces/auto-instrumentation/overview.mdx | 139 +++++++++++++++- .../observability/traces/configuration.mdx | 18 --- .../observability/traces/integrations.mdx | 72 ++++++++- .../observability/traces/manual-tracing.mdx | 123 ++++++++++++++ .../observability/traces/opentelemetry.mdx | 4 +- images/provider-logos/anthropic.svg | 9 ++ images/provider-logos/google.svg | 21 +++ images/provider-logos/langchain.svg | 11 ++ images/provider-logos/litellm.svg | 10 ++ images/provider-logos/llamaindex.svg | 18 +++ images/provider-logos/openai.svg | 8 + images/provider-logos/openclaw.svg | 29 ++++ images/provider-logos/openrouter.svg | 8 + images/provider-logos/opentelemetry.svg | 8 + images/provider-logos/pydantic.svg | 1 + images/provider-logos/vercel.svg | 8 + sdks/evals/agent-tracing.mdx | 4 +- sdks/evals/building-an-eval.mdx | 2 +- sdks/evals/quickstart.mdx | 2 +- sdks/evals/scorers/overview.mdx | 2 +- sdks/javascript.mdx | 4 +- sdks/python.mdx | 4 +- 28 files changed, 809 insertions(+), 171 deletions(-) create mode 100644 features/observability/traces/auto-instrumentation/anthropic.mdx create mode 100644 features/observability/traces/auto-instrumentation/google.mdx delete mode 100644 features/observability/traces/configuration.mdx create mode 100644 features/observability/traces/manual-tracing.mdx create mode 100644 images/provider-logos/anthropic.svg create mode 100644 images/provider-logos/google.svg create mode 100644 images/provider-logos/langchain.svg create mode 100644 images/provider-logos/litellm.svg create mode 100644 images/provider-logos/llamaindex.svg create mode 100644 images/provider-logos/openai.svg create mode 100644 images/provider-logos/openclaw.svg create mode 100644 images/provider-logos/openrouter.svg create mode 100644 images/provider-logos/opentelemetry.svg create mode 100644 images/provider-logos/pydantic.svg create mode 100644 images/provider-logos/vercel.svg diff --git a/docs.json b/docs.json index 07221f76..b19e716e 100644 --- a/docs.json +++ b/docs.json @@ -109,17 +109,19 @@ "icon": "diagram-project", "pages": [ "features/observability/traces", - "features/observability/traces/configuration", + "features/observability/traces/manual-tracing", { "group": "SDK Auto-Instrumentation", "icon": "code", "pages": [ "features/observability/traces/auto-instrumentation/overview", - "features/observability/traces/auto-instrumentation/openai" + "features/observability/traces/auto-instrumentation/openai", + "features/observability/traces/auto-instrumentation/anthropic", + "features/observability/traces/auto-instrumentation/google" ] }, - "features/observability/traces/integrations", - "features/observability/traces/opentelemetry" + "features/observability/traces/opentelemetry", + "features/observability/traces/integrations" ] } ] diff --git a/features/observability/overview.mdx b/features/observability/overview.mdx index 88a0b522..46a0cbe2 100644 --- a/features/observability/overview.mdx +++ b/features/observability/overview.mdx @@ -63,7 +63,7 @@ Once you know which artifact you need, follow the setup path for the code that p Choose PromptLayer SDK spans, provider auto-instrumentation, a telemetry integration, or OpenTelemetry. diff --git a/features/observability/traces.mdx b/features/observability/traces.mdx index 45b2620d..a556de11 100644 --- a/features/observability/traces.mdx +++ b/features/observability/traces.mdx @@ -1,6 +1,6 @@ --- title: "Overview" -description: "Understand PromptLayer traces and add custom spans with the PromptLayer SDK." +description: "Understand PromptLayer traces and choose how to instrument your application." icon: "diagram-project" --- @@ -8,9 +8,18 @@ A trace represents one end-to-end operation in your application. It contains a h Supported LLM spans link to PromptLayer request logs, so you can move from the full execution path to the input, output, model, tokens, cost, and metadata for an individual model call. - -This page explains the trace hierarchy and PromptLayer SDK span helpers. To choose between provider auto-instrumentation, a framework integration, and a custom OpenTelemetry pipeline, see [Configuration](/features/observability/traces/configuration). - +## Choose a Tracing Method + +All tracing methods send spans to the same PromptLayer trace view. Choose the method that matches the code producing telemetry: + +| Method | Use it when | +| --- | --- | +| [Manual Tracing](/features/observability/traces/manual-tracing) | You use `PromptLayer.run()` or want to wrap custom application and tool functions with PromptLayer SDK spans. | +| [SDK Auto-Instrumentation](/features/observability/traces/auto-instrumentation/overview) | Your application calls a supported model provider SDK directly. | +| [Telemetry Integrations](/features/observability/traces/integrations) | A framework, agent SDK, or model router makes the calls. | +| [OpenTelemetry](/features/observability/traces/opentelemetry) | You already operate an OpenTelemetry pipeline or no dedicated integration supports your library. | + +Use the most specific supported method. These are collection methods, not separate trace products. ## Inspect a Trace @@ -22,122 +31,6 @@ The trace view shows parent-child relationships, duration, status, attributes, i The trace list shows root spans. For a long-running operation, child spans can reach PromptLayer before the root span finishes, but the complete trace appears in the list after the root span ends. -## Trace PromptLayer SDK Runs - -Enable tracing when you create a PromptLayer client. Calls made through `run()` then participate in the active trace. - - -```python Python -from promptlayer import PromptLayer - -pl = PromptLayer(enable_tracing=True) - -result = pl.run( - prompt_name="simple-greeting", - input_variables={"name": "Alice"}, -) -``` - -```javascript JavaScript -import { PromptLayer } from "promptlayer"; - -const pl = new PromptLayer({ - apiKey: process.env.PROMPTLAYER_API_KEY, - enableTracing: true, -}); - -const result = await pl.run({ - promptName: "simple-greeting", - inputVariables: { name: "Alice" }, -}); -``` - - -For the full `run()` interface, see the [Python SDK](/sdks/python#using-the-run-method-recommended) or [JavaScript SDK](/sdks/javascript#using-the-run-method-recommended). - - -This setting enables tracing for the PromptLayer client. To trace calls made directly through a provider client such as `OpenAI()`, use [SDK Auto-Instrumentation](/features/observability/traces/auto-instrumentation/overview). - - -## Add Custom Spans - -Use `traceable` in Python or `wrapWithSpan` in JavaScript to record application functions that are not traced by an integration. Give important spans a descriptive name so they are easy to identify in the trace view. - - -```python Python -@pl.traceable( - name="calculate-total", - attributes={"component": "billing"}, -) -def calculate_total(items): - return sum(item["price"] for item in items) -``` - -```javascript JavaScript -const calculateTotal = pl.wrapWithSpan( - "calculate-total", - (items) => items.reduce((total, item) => total + item.price, 0) -); -``` - - -If you omit the Python `name`, PromptLayer uses the function name. JavaScript takes the span name as the first argument to `wrapWithSpan`. - -## Nest Spans - -Traced functions called inside another active span become children of that span. When an in-process provider or framework integration preserves the active OpenTelemetry context, its spans also appear as children. This allows one trace to combine application, tool, and LLM spans. - - -```python Python -@pl.traceable() -def retrieve_context(question): - return ["Relevant context"] - -@pl.traceable(name="answer-question") -def answer_question(question): - context = retrieve_context(question) - return {"question": question, "context": context} -``` - -```javascript JavaScript -const retrieveContext = pl.wrapWithSpan( - "retrieve-context", - async (question) => ["Relevant context"] -); - -const answerQuestion = pl.wrapWithSpan( - "answer-question", - async (question) => ({ - question, - context: await retrieveContext(question), - }) -); -``` - - -![Group nests spans](/images/traces/nested-spans.png) - -## Trace Tools - -Use `traceTool` for tool handlers in a custom agent that does not already have a dedicated [Telemetry Integration](/features/observability/traces/integrations). PromptLayer names the span `Tool: ` and marks it as a tool call. - - -```python Python -@pl.traceTool(name="get_weather") -def get_weather(city: str) -> str: - return f"{city} is 72F and sunny." -``` - -```javascript JavaScript -const getWeather = pl.traceTool( - "get_weather", - async (city) => `${city} is 72F and sunny.` -); -``` - - -`traceTool` records only when tracing is enabled on the client. The tool name is also used by tool-aware features such as the [Trajectory scorer](/sdks/evals/scorers/overview#trajectory). - ## Filter Traces The trace list can be filtered by metadata and resource attribute values. Filters search the entire span hierarchy: a trace appears if any root or child span has a matching attribute. diff --git a/features/observability/traces/auto-instrumentation/anthropic.mdx b/features/observability/traces/auto-instrumentation/anthropic.mdx new file mode 100644 index 00000000..d2956039 --- /dev/null +++ b/features/observability/traces/auto-instrumentation/anthropic.mdx @@ -0,0 +1,150 @@ +--- +title: "Anthropic SDK" +description: "Automatically trace supported direct Anthropic SDK calls with PromptLayer." +icon: "/images/provider-logos/anthropic.svg" +--- + +PromptLayer can auto-instrument the official Anthropic SDK and export supported calls as OpenTelemetry spans. Each supported direct SDK call appears in PromptLayer as both a trace span and an associated request log without changing how you create or use the Anthropic client. + + +This guide covers the Anthropic model SDK. If you use the Claude Agent SDK or Claude Code, follow the [Claude integration](/features/observability/traces/integrations#claude-code) instead. + + +## Supported APIs + +| Anthropic API | Python | JavaScript | +| --- | --- | --- | +| Messages (`messages.create`) | Sync and async | Supported | +| Messages streaming | `messages.create(..., stream=True)` and `messages.stream()`; sync and async | `messages.create({ stream: true })` | +| Structured message parsing (`messages.parse`) | Sync and async | Not supported | +| Beta Messages | Not included in the documented Python surface | Create and streaming | +| Anthropic Vertex | Supported | Supported | + +Anthropic Bedrock is not included in JavaScript provider auto-instrumentation. Other Anthropic SDK calls continue to work normally, but unsupported surfaces do not automatically create PromptLayer traces or request logs. + +## Prerequisites + +- A [PromptLayer API key](/quickstart#prerequisites) for the workspace that should receive the traces +- An Anthropic API key, or the standard Google Cloud credentials and project configuration for Anthropic Vertex +- Python 3.10 or later for Python, or Node.js 20 or later for JavaScript + +Export the direct Anthropic API keys before the application starts: + +```bash +export PROMPTLAYER_API_KEY="pl_..." +export ANTHROPIC_API_KEY="sk-ant-..." +``` + +PromptLayer strongly recommends setting the language-specific [content-capture value](/features/observability/traces/auto-instrumentation/overview#capture-prompts-and-responses) so request inspection, search, analytics, and debugging can use prompts, responses, and tool content. + +## Python + +### 1. Install the SDKs + +```bash +pip install "promptlayer[otel-genai-instrumentation]" anthropic +``` + +Install Anthropic's Vertex dependencies if your application uses `AnthropicVertex`. + +### 2. Configure Anthropic instrumentation + +```python +from anthropic import Anthropic +from promptlayer import configure_tracing + +tracer_provider = configure_tracing(providers=("anthropic",)) +client = Anthropic() + +try: + message = client.messages.create( + model="claude-sonnet-4-20250514", + max_tokens=256, + messages=[ + { + "role": "user", + "content": "Explain distributed tracing in one sentence.", + } + ], + ) + print(message.content[0].text) +finally: + client.close() + tracer_provider.force_flush() +``` + +The same `anthropic` selector instruments `AnthropicVertex`. Sync and async clients are supported. + + +If your application already creates a PromptLayer client, select Anthropic when enabling tracing: + +```python +from anthropic import Anthropic +from promptlayer import PromptLayer + +pl = PromptLayer( + enable_tracing=True, + tracing_providers=("anthropic",), +) +client = Anthropic() +``` + +Omit `tracing_providers` to instrument every supported provider SDK that is installed. + + +## JavaScript + +### 1. Install the SDKs + +```bash +npm install promptlayer @anthropic-ai/sdk +``` + +For Anthropic Vertex, also install `@anthropic-ai/vertex-sdk`. + +### 2. Preload PromptLayer instrumentation + +The preload must run before the application imports the Anthropic SDK: + +```bash +node --import promptlayer/register app.mjs +``` + +### 3. Use the Anthropic SDK normally + +```javascript +import Anthropic from "@anthropic-ai/sdk"; +import { shutdownTracing } from "promptlayer"; + +const client = new Anthropic(); + +try { + const message = await client.messages.create({ + model: "claude-sonnet-4-20250514", + max_tokens: 256, + messages: [ + { + role: "user", + content: "Explain distributed tracing in one sentence.", + }, + ], + }); + console.log(message.content); +} finally { + await shutdownTracing(); +} +``` + +The preload instruments every supported provider. To instrument only Anthropic, call `configureTracing({ providers: ["anthropic"] })` in a bootstrap module and dynamically import the application afterward. See [Select Providers](/features/observability/traces/auto-instrumentation/overview#select-providers). + +## Verify the Integration + +Run one supported Anthropic request, flush or shut down tracing, and open [Traces](/features/observability/traces) in PromptLayer. The Anthropic span should have an associated request log. + +If no span appears: + +- Confirm instrumentation is configured before the first Anthropic request. +- In JavaScript, confirm the preload or bootstrap runs before the Anthropic SDK module loads. +- Confirm the call uses an API surface listed in [Supported APIs](#supported-apis). +- Confirm `PROMPTLAYER_API_KEY` belongs to the workspace you are checking. +- Flush or shut down tracing before a short-lived process exits. diff --git a/features/observability/traces/auto-instrumentation/google.mdx b/features/observability/traces/auto-instrumentation/google.mdx new file mode 100644 index 00000000..d2f4f36d --- /dev/null +++ b/features/observability/traces/auto-instrumentation/google.mdx @@ -0,0 +1,150 @@ +--- +title: "Google SDK" +description: "Automatically trace supported direct Google GenAI SDK calls with PromptLayer." +icon: "/images/provider-logos/google.svg" +--- + +PromptLayer can auto-instrument the Google GenAI SDK in both Gemini Developer API and Vertex AI modes. Supported calls become OpenTelemetry spans and linked PromptLayer request logs without replacing the native provider client. + +## Supported APIs + +| Google GenAI API | Python | JavaScript | +| --- | --- | --- | +| Generate Content | Sync and async | Supported | +| Streaming Generate Content | Sync and async | Supported | +| Embeddings | Sync and async | Not supported | +| Interactions | Sync and async in supported Google GenAI SDK releases | Not supported | +| Vertex AI mode | Supported | Supported | + +In JavaScript, chat `sendMessage` and `sendMessageStream` calls are also traced. Other Google GenAI SDK calls continue to work normally, but unsupported surfaces do not automatically create PromptLayer traces or request logs. + +## Prerequisites + +- A [PromptLayer API key](/quickstart#prerequisites) for the workspace that should receive the traces +- A Gemini API key, or standard Google Cloud credentials, project, and location for Vertex AI +- Python 3.10 or later for Python, or Node.js 20 or later for JavaScript + +For Gemini Developer API calls, export: + +```bash +export PROMPTLAYER_API_KEY="pl_..." +export GOOGLE_API_KEY="..." +``` + +PromptLayer strongly recommends setting the language-specific [content-capture value](/features/observability/traces/auto-instrumentation/overview#capture-prompts-and-responses) so request inspection, search, analytics, and debugging can use prompts, responses, and tool content. + +## Python + +### 1. Install the SDKs + +```bash +pip install "promptlayer[otel-genai-instrumentation]" google-genai +``` + +### 2. Configure Google GenAI instrumentation + +```python +import os + +from google import genai +from promptlayer import configure_tracing + +tracer_provider = configure_tracing(providers=("google",)) +client = genai.Client(api_key=os.environ["GOOGLE_API_KEY"]) + +try: + response = client.models.generate_content( + model="gemini-2.5-flash", + contents="Explain distributed tracing in one sentence.", + ) + print(response.text) +finally: + client.close() + tracer_provider.force_flush() +``` + +The same `google` selector instruments a client created with `vertexai=True`: + +```python +vertex_client = genai.Client( + vertexai=True, + project=os.environ["GOOGLE_CLOUD_PROJECT"], + location=os.environ["GOOGLE_CLOUD_LOCATION"], +) +``` + + +If your application already creates a PromptLayer client, select Google when enabling tracing: + +```python +from promptlayer import PromptLayer + +pl = PromptLayer( + enable_tracing=True, + tracing_providers=("google",), +) +``` + +Omit `tracing_providers` to instrument every supported provider SDK that is installed. + + +## JavaScript + +### 1. Install the SDKs + +```bash +npm install promptlayer @google/genai +``` + +### 2. Preload PromptLayer instrumentation + +The preload must run before the application imports `@google/genai`: + +```bash +node --import promptlayer/register app.mjs +``` + +### 3. Use the Google GenAI SDK normally + +```javascript +import { GoogleGenAI } from "@google/genai"; +import { shutdownTracing } from "promptlayer"; + +const client = new GoogleGenAI({ + apiKey: process.env.GOOGLE_API_KEY, +}); + +try { + const response = await client.models.generateContent({ + model: "gemini-2.5-flash", + contents: "Explain distributed tracing in one sentence.", + }); + console.log(response.text); +} finally { + await shutdownTracing(); +} +``` + +The same instrumentation supports a client created in Vertex AI mode: + +```javascript +const vertexClient = new GoogleGenAI({ + vertexai: true, + project: process.env.GOOGLE_CLOUD_PROJECT, + location: process.env.GOOGLE_CLOUD_LOCATION, +}); +``` + +The preload instruments every supported provider. To instrument only Google, call `configureTracing({ providers: ["google"] })` in a bootstrap module and dynamically import the application afterward. See [Select Providers](/features/observability/traces/auto-instrumentation/overview#select-providers). + +## Verify the Integration + +Run one supported Google GenAI request, flush or shut down tracing, and open [Traces](/features/observability/traces) in PromptLayer. The provider span should have an associated request log and identify Gemini Developer API or Vertex AI mode. + +If no span appears: + +- Confirm instrumentation is configured before the first Google GenAI request. +- In JavaScript, confirm the preload or bootstrap runs before `@google/genai` loads. +- Confirm the call uses an API surface listed in [Supported APIs](#supported-apis). +- Confirm `PROMPTLAYER_API_KEY` belongs to the workspace you are checking. +- Flush or shut down tracing before a short-lived process exits. diff --git a/features/observability/traces/auto-instrumentation/openai.mdx b/features/observability/traces/auto-instrumentation/openai.mdx index 7ecbadca..6cf938f9 100644 --- a/features/observability/traces/auto-instrumentation/openai.mdx +++ b/features/observability/traces/auto-instrumentation/openai.mdx @@ -1,7 +1,7 @@ --- title: "OpenAI SDK" description: "Automatically trace supported direct OpenAI SDK calls with PromptLayer." -icon: "robot" +icon: "/images/provider-logos/openai.svg" --- PromptLayer can auto-instrument the official OpenAI SDK and export supported calls as OpenTelemetry spans. Each supported direct SDK call appears in PromptLayer as both a trace span and an associated request log without changing how you create or use the OpenAI client. @@ -14,9 +14,10 @@ This guide covers the OpenAI model SDK. If you use the OpenAI Agents SDK, follow | OpenAI API | Python | JavaScript | | --- | --- | --- | -| Chat Completions (`chat.completions.create`) | Supported | Supported | -| Responses (`responses.create`) | Not supported | Supported | -| Embeddings (`embeddings.create`) | Supported | Supported | +| Chat Completions (`chat.completions.create`) | Sync, async, and streaming | Supported, including streaming | +| Structured-output parsing | Sync and async | Not supported | +| Responses (`responses.create`) | Sync, async, and streaming | Supported, including streaming | +| Embeddings (`embeddings.create`) | Sync and async | Supported | Only the API surfaces in this table are auto-instrumented. Other OpenAI SDK calls continue to work normally, but this integration does not automatically create PromptLayer traces or request logs for them. @@ -26,7 +27,7 @@ Only the API surfaces in this table are auto-instrumented. Other OpenAI SDK call - An OpenAI API key - Python 3.10 or later for the Python integration, or Node.js 20 or later for the JavaScript integration -Export the variables for your language before the application starts. The API keys are required for the setup in this guide. The content-capture setting is required if request logs should include prompts, responses, and tool arguments. +Export the variables for your language before the application starts. The API keys are required for the setup in this guide. Content capture is strongly recommended so PromptLayer can fully support request inspection, search, analytics, and debugging with your prompts, responses, and tool arguments. ```bash Python @@ -42,7 +43,7 @@ export OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT="true" ``` -Omit `OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT` if you want metadata-only telemetry without message contents. +Omit `OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT` only if your data policies require metadata-only telemetry without message contents. This limits content-aware PromptLayer features. ## Python @@ -87,13 +88,16 @@ finally: `instrument_openai()` is idempotent when called again with the same tracer provider. If your application already owns an OpenTelemetry SDK tracer provider, pass it with `tracer_provider=`. -If your application already creates a PromptLayer client, `enable_tracing=True` configures the same OpenAI auto-instrumentation when the tracing extra is installed: +If your application already creates a PromptLayer client, select OpenAI when enabling tracing: ```python from openai import OpenAI from promptlayer import PromptLayer -promptlayer_client = PromptLayer(enable_tracing=True) +promptlayer_client = PromptLayer( + enable_tracing=True, + tracing_providers=("openai",), +) client = OpenAI() completion = client.chat.completions.create( @@ -104,7 +108,7 @@ completion = client.chat.completions.create( promptlayer_client.tracer_provider.force_flush() ``` -Use either this setup or `instrument_openai()` for the same tracer provider; you do not need both. +Omit `tracing_providers` to instrument every supported provider SDK that is installed. Use either this setup or `instrument_openai()` for the same tracer provider; you do not need both. ## JavaScript @@ -168,9 +172,11 @@ try { Call `shutdownTracing()` when a short-lived process finishes, not after every request in a long-running server. +The preload instruments every supported provider. To instrument only OpenAI, call `configureTracing({ providers: ["openai"] })` in a bootstrap module and dynamically import the application afterward. See [Select Providers](/features/observability/traces/auto-instrumentation/overview#select-providers). + ## Capture Prompts and Responses -Prompt and response content is disabled by default because it can contain sensitive data. Model names, timing, token usage when available, and other non-content telemetry are still recorded. +Prompt and response content is disabled by default because it can contain sensitive data. PromptLayer strongly recommends enabling it when your data policies allow it so search, analytics, request inspection, and debugging can use the complete LLM interaction. Model names, timing, token usage when available, and other non-content telemetry are still recorded when capture is disabled. The complete export blocks in [Prerequisites](#prerequisites) enable content capture. Use the language-specific value shown there, set it before `instrument_openai()` or the `promptlayer/register` preload runs, and restart an already-running process after changing it. @@ -182,7 +188,7 @@ Content capture can send user prompts, model responses, and tool arguments to Pr | Setting | Required | Description | | --- | --- | --- | -| `PROMPTLAYER_API_KEY` | Yes | Authenticates trace export and selects the PromptLayer workspace. Python can instead pass `api_key=` to `instrument_openai()`. | +| `PROMPTLAYER_API_KEY` | Yes | Authenticates trace export and selects the PromptLayer workspace. Python can instead pass `api_key=` to `instrument_openai()` or `configure_tracing()`. | | `OPENAI_API_KEY` | Yes | Authenticates OpenAI SDK requests. It is read by OpenAI and is not sent to PromptLayer. | | `PROMPTLAYER_BASE_URL` | No | Overrides the PromptLayer API root. The tracing endpoint defaults to `/v1/traces`. | | `PROMPTLAYER_OTLP_TRACES_ENDPOINT` | No | Overrides the complete OTLP/HTTP trace endpoint and takes precedence over `PROMPTLAYER_BASE_URL`. | @@ -200,6 +206,8 @@ tracer_provider = instrument_openai( Configure a tracer provider only once and reuse it. If the OpenAI SDK is already instrumented with a different provider, PromptLayer rejects the mismatch instead of silently exporting incomplete traces. +For multi-provider Python applications, use `configure_tracing(providers=("openai", ...))` or the `tracing_providers` PromptLayer client option described in the [auto-instrumentation overview](/features/observability/traces/auto-instrumentation/overview#select-providers). + ## Verify the Integration Run one supported OpenAI request, flush tracing, and open [Traces](/features/observability/traces) in PromptLayer. The OpenAI span should have an associated request log. If the call runs inside `PromptLayer.run()`, PromptLayer links the provider span to the existing run request log instead of creating a duplicate. diff --git a/features/observability/traces/auto-instrumentation/overview.mdx b/features/observability/traces/auto-instrumentation/overview.mdx index 6ac5d118..ebc8a182 100644 --- a/features/observability/traces/auto-instrumentation/overview.mdx +++ b/features/observability/traces/auto-instrumentation/overview.mdx @@ -10,10 +10,141 @@ For the resulting hierarchy and dashboard view, see [Traces](/features/observabi ## Provider Guides -| Provider | Languages | Guide | -| --- | --- | --- | -| OpenAI | Python and JavaScript | [Auto-Instrument the OpenAI SDK](/features/observability/traces/auto-instrumentation/openai) | + + + Python and JavaScript + + + Python and JavaScript + + + Python and JavaScript + + The provider guide is the source of truth for supported API surfaces, installation, initialization order, content capture, exporter settings, flushing, and verification. Coverage and configuration can differ between languages. -If a framework or agent SDK creates the provider client for you, use [Telemetry Integrations](/features/observability/traces/integrations). If no provider guide applies or you already own the telemetry pipeline, use [OpenTelemetry](/features/observability/traces/opentelemetry). See [Configure Tracing](/features/observability/traces/configuration) for the complete routing guide. +## Enable All Installed Providers + +Python requires the optional instrumentation extra. Install only the provider SDKs your application uses: + +```bash +pip install "promptlayer[otel-genai-instrumentation]" openai anthropic google-genai +``` + +Then enable tracing on the PromptLayer client. By default, the client instruments every supported provider SDK that is installed: + +```python +from promptlayer import PromptLayer + +pl = PromptLayer(enable_tracing=True) + +# Make direct OpenAI, Anthropic, or Google GenAI SDK calls. + +# Flush pending spans before a short-lived process exits. +pl.tracer_provider.force_flush() +``` + +The Python instrumentation extra requires Python 3.10 or later. The core PromptLayer package continues to support Python 3.9. + +For JavaScript, install PromptLayer and the provider SDKs your application uses: + +```bash +npm install promptlayer openai @anthropic-ai/sdk @google/genai +``` + +Provider SDK modules must load after tracing is configured. In an ESM application, preload PromptLayer before the application module: + +```bash +node --import promptlayer/register app.mjs +``` + +The preload reads `PROMPTLAYER_API_KEY` and instruments every supported provider. Call `shutdownTracing()` before a short-lived process exits: + +```javascript +import { shutdownTracing } from "promptlayer"; + +try { + await runApplication(); +} finally { + await shutdownTracing(); +} +``` + +Do not shut down tracing after every request in a long-running server. + +## Select Providers + +Omit the provider selection to instrument every supported provider. Pass an empty list or tuple to export PromptLayer spans without provider SDK auto-instrumentation. + + +```python Python +from promptlayer import PromptLayer + +pl = PromptLayer( + enable_tracing=True, + tracing_providers=("anthropic", "google"), +) +``` + +```javascript JavaScript +import { configureTracing } from "promptlayer"; + +const tracing = configureTracing({ + providers: ["anthropic", "google"], +}); + +try { + // Dynamic import ensures provider SDKs load after instrumentation. + await import("./app.mjs"); +} finally { + await tracing.shutdown(); +} +``` + + +Supported selectors are `openai`, `anthropic`, and `google`. Python also accepts `openai.azure` as an alias for the OpenAI instrumentor. Anthropic Vertex uses `anthropic`; Google clients created with `vertexai=True` use `google`. + +Python applications that do not create a PromptLayer client can call `configure_tracing(providers=(...))`. The function returns the configured tracer provider so short-lived processes can call `force_flush()`. + +## Capture Prompts and Responses + +Provider prompt and response content is disabled by default because it can contain sensitive data. PromptLayer strongly recommends enabling content capture when your data policies allow it. Content capture is crucial for getting the full value from PromptLayer search and analytics because it provides the prompt, response, and tool data used by request inspection, content search, analysis, and debugging. + +Without content capture, traces still include model names, timing, token usage when available, and other non-content telemetry. PromptLayer cannot search or analyze prompt, response, and tool content that was not exported. + +Set the language-specific value before tracing is configured: + + +```bash Python +export OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT="span_only" +``` + +```bash JavaScript +export OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT="true" +``` + + +JavaScript can instead pass `captureContent: true` to `configureTracing()`. + + +For the complete PromptLayer observability experience, enable content capture before your application makes its first provider request. + + + +Content capture can send user prompts, model responses, tool arguments, and other application data to PromptLayer. Review your privacy, retention, and compliance requirements before enabling it. + + +If a framework or agent SDK creates the provider client for you, use [Telemetry Integrations](/features/observability/traces/integrations). If no provider guide applies or you already own the telemetry pipeline, use [OpenTelemetry](/features/observability/traces/opentelemetry). See the [Tracing overview](/features/observability/traces) for the complete routing guide. diff --git a/features/observability/traces/configuration.mdx b/features/observability/traces/configuration.mdx deleted file mode 100644 index eb7aa86f..00000000 --- a/features/observability/traces/configuration.mdx +++ /dev/null @@ -1,18 +0,0 @@ ---- -title: "Configuration" -description: "Choose how to send traces and linked request logs to PromptLayer." -icon: "diagram-project" ---- - -All tracing setup paths send spans to the same [PromptLayer trace view](/features/observability/traces). Choose the path that matches the code producing telemetry: - -| Setup | Use it when | -| --- | --- | -| [PromptLayer SDK spans](/features/observability/traces#trace-promptlayer-sdk-runs) | You use `PromptLayer.run()` or want to wrap custom application and tool functions. | -| [SDK Auto-Instrumentation](/features/observability/traces/auto-instrumentation/overview) | Your application calls a supported model provider SDK directly. | -| [Telemetry Integrations](/features/observability/traces/integrations) | A framework, agent SDK, or model router makes the calls. | -| [OpenTelemetry](/features/observability/traces/opentelemetry) | You already operate an OpenTelemetry pipeline or no dedicated integration supports your library. | - -Use the most specific supported setup. These are collection methods, not separate trace products. - -After setup, open [Traces](/features/observability/traces) to verify the span hierarchy and linked [request logs](/features/observability/request-logs). diff --git a/features/observability/traces/integrations.mdx b/features/observability/traces/integrations.mdx index 3083e269..803f35b9 100644 --- a/features/observability/traces/integrations.mdx +++ b/features/observability/traces/integrations.mdx @@ -6,7 +6,75 @@ icon: 'handshake' Use this page when an LLM framework, agent SDK, or model router makes calls or produces telemetry for your application. Each section is the setup guide for that framework or tool. -The resulting spans appear in [Traces](/features/observability/traces). If your tool is not listed, use [OpenTelemetry](/features/observability/traces/opentelemetry) or [email us](mailto:hello@promptlayer.com). If your code calls a model provider SDK directly, use [SDK Auto-Instrumentation](/features/observability/traces/auto-instrumentation/overview). [Configure Tracing](/features/observability/traces/configuration) compares all tracing paths. +The resulting spans appear in [Traces](/features/observability/traces). If your tool is not listed, use [OpenTelemetry](/features/observability/traces/opentelemetry) or [email us](mailto:hello@promptlayer.com). If your code calls a model provider SDK directly, use [SDK Auto-Instrumentation](/features/observability/traces/auto-instrumentation/overview). The [Tracing overview](/features/observability/traces) compares all tracing paths. + + + + Model router + + + Data and agent framework + + + CLI and agent SDK + + + JavaScript and Python agents + + + Agent plugin + + + JavaScript AI applications + + + Python agents + + + Framework and observability + + + Model router + + + +*Third-party names and logos belong to their respective owners. Their inclusion identifies supported integrations and does not imply endorsement or partnership.* ## LiteLLM @@ -16,7 +84,7 @@ Please read the [LiteLLM documentation page](https://docs.litellm.ai/docs/observ ## LlamaIndex -[LlamaIndex](https://www.llamaindex.ai/) is a data framework for LLM-based applications. Read more about our integration on the [LlamaIndex documentation page](https://docs.llamaindex.ai/en/stable/module_guides/observability/observability.html#promptlayer) +[LlamaIndex](https://www.llamaindex.ai/) is a data framework for LLM-based applications. Read more about our integration on the [LlamaIndex documentation page](https://developers.llamaindex.ai/python/framework/module_guides/observability/#promptlayer) ## Claude Code diff --git a/features/observability/traces/manual-tracing.mdx b/features/observability/traces/manual-tracing.mdx new file mode 100644 index 00000000..cea0d221 --- /dev/null +++ b/features/observability/traces/manual-tracing.mdx @@ -0,0 +1,123 @@ +--- +title: "Manual Tracing" +description: "Add PromptLayer SDK spans around prompt runs, application functions, and tools." +icon: "code" +--- + +Use manual tracing when you call prompts with `PromptLayer.run()` or want to add spans around application functions and tools. If a supported provider SDK or framework already produces the calls, choose its setup method from the [Tracing overview](/features/observability/traces). + +## Trace PromptLayer SDK Runs + +Enable tracing when you create a PromptLayer client. Calls made through `run()` then participate in the active trace. + + +```python Python +from promptlayer import PromptLayer + +pl = PromptLayer(enable_tracing=True) + +result = pl.run( + prompt_name="simple-greeting", + input_variables={"name": "Alice"}, +) +``` + +```javascript JavaScript +import { PromptLayer } from "promptlayer"; + +const pl = new PromptLayer({ + apiKey: process.env.PROMPTLAYER_API_KEY, + enableTracing: true, +}); + +const result = await pl.run({ + promptName: "simple-greeting", + inputVariables: { name: "Alice" }, +}); +``` + + +For the full `run()` interface, see the [Python SDK](/sdks/python#using-the-run-method-recommended) or [JavaScript SDK](/sdks/javascript#using-the-run-method-recommended). + + +This setting also enables installed provider auto-instrumentation. To trace direct OpenAI, Anthropic, or Google GenAI SDK calls, follow [SDK Auto-Instrumentation](/features/observability/traces/auto-instrumentation/overview) for the required packages, supported APIs, and JavaScript initialization order. + + +## Add Custom Spans + +Use `traceable` in Python or `wrapWithSpan` in JavaScript to record application functions that are not traced by an integration. Give important spans a descriptive name so they are easy to identify in the trace view. + + +```python Python +@pl.traceable( + name="calculate-total", + attributes={"component": "billing"}, +) +def calculate_total(items): + return sum(item["price"] for item in items) +``` + +```javascript JavaScript +const calculateTotal = pl.wrapWithSpan( + "calculate-total", + (items) => items.reduce((total, item) => total + item.price, 0) +); +``` + + +If you omit the Python `name`, PromptLayer uses the function name. JavaScript takes the span name as the first argument to `wrapWithSpan`. + +## Nest Spans + +Traced functions called inside another active span become children of that span. When an in-process provider or framework integration preserves the active OpenTelemetry context, its spans also appear as children. This allows one trace to combine application, tool, and LLM spans. + + +```python Python +@pl.traceable() +def retrieve_context(question): + return ["Relevant context"] + +@pl.traceable(name="answer-question") +def answer_question(question): + context = retrieve_context(question) + return {"question": question, "context": context} +``` + +```javascript JavaScript +const retrieveContext = pl.wrapWithSpan( + "retrieve-context", + async (question) => ["Relevant context"] +); + +const answerQuestion = pl.wrapWithSpan( + "answer-question", + async (question) => ({ + question, + context: await retrieveContext(question), + }) +); +``` + + +![Group nests spans](/images/traces/nested-spans.png) + +## Trace Tools + +Use `traceTool` for tool handlers in a custom agent that does not already have a dedicated [Telemetry Integration](/features/observability/traces/integrations). PromptLayer names the span `Tool: ` and marks it as a tool call. + + +```python Python +@pl.traceTool(name="get_weather") +def get_weather(city: str) -> str: + return f"{city} is 72F and sunny." +``` + +```javascript JavaScript +const getWeather = pl.traceTool( + "get_weather", + async (city) => `${city} is 72F and sunny.` +); +``` + + +`traceTool` records only when tracing is enabled on the client. The tool name is also used by tool-aware features such as the [Trajectory scorer](/sdks/evals/scorers/overview#trajectory). diff --git a/features/observability/traces/opentelemetry.mdx b/features/observability/traces/opentelemetry.mdx index 4bcda053..b5619e95 100644 --- a/features/observability/traces/opentelemetry.mdx +++ b/features/observability/traces/opentelemetry.mdx @@ -1,12 +1,12 @@ --- title: "OpenTelemetry" description: "Send an existing or custom OpenTelemetry pipeline to PromptLayer." -icon: "tower-broadcast" +icon: "/images/provider-logos/opentelemetry.svg" --- Use this path when your application already emits [OpenTelemetry (OTEL)](https://opentelemetry.io/) spans, you use an OpenTelemetry Collector, or no dedicated PromptLayer integration supports your library. You do not need a PromptLayer SDK. -Exported spans appear in [Traces](/features/observability/traces). For a supported model provider SDK or framework, use its dedicated [SDK Auto-Instrumentation](/features/observability/traces/auto-instrumentation/overview) or [Telemetry Integration](/features/observability/traces/integrations) instead. [Configure Tracing](/features/observability/traces/configuration) compares all tracing paths. +Exported spans appear in [Traces](/features/observability/traces). For a supported model provider SDK or framework, use its dedicated [SDK Auto-Instrumentation](/features/observability/traces/auto-instrumentation/overview) or [Telemetry Integration](/features/observability/traces/integrations) instead. The [Tracing overview](/features/observability/traces) compares all tracing paths. ## OTLP Endpoint diff --git a/images/provider-logos/anthropic.svg b/images/provider-logos/anthropic.svg new file mode 100644 index 00000000..fae4018b --- /dev/null +++ b/images/provider-logos/anthropic.svg @@ -0,0 +1,9 @@ + + Anthropic + + + + diff --git a/images/provider-logos/google.svg b/images/provider-logos/google.svg new file mode 100644 index 00000000..93f9a56e --- /dev/null +++ b/images/provider-logos/google.svg @@ -0,0 +1,21 @@ + + + + + google_buttn + Created with Sketch. + + + + + + + + + + + + + + + diff --git a/images/provider-logos/langchain.svg b/images/provider-logos/langchain.svg new file mode 100644 index 00000000..bf374225 --- /dev/null +++ b/images/provider-logos/langchain.svg @@ -0,0 +1,11 @@ + + LangChain + + + + + + diff --git a/images/provider-logos/litellm.svg b/images/provider-logos/litellm.svg new file mode 100644 index 00000000..86363f2e --- /dev/null +++ b/images/provider-logos/litellm.svg @@ -0,0 +1,10 @@ + + LiteLLM + 🚅 + diff --git a/images/provider-logos/llamaindex.svg b/images/provider-logos/llamaindex.svg new file mode 100644 index 00000000..fdcc834c --- /dev/null +++ b/images/provider-logos/llamaindex.svg @@ -0,0 +1,18 @@ + + + + + + + + + + + + + + + + + + diff --git a/images/provider-logos/openai.svg b/images/provider-logos/openai.svg new file mode 100644 index 00000000..7b344589 --- /dev/null +++ b/images/provider-logos/openai.svg @@ -0,0 +1,8 @@ + + OpenAI + + + diff --git a/images/provider-logos/openclaw.svg b/images/provider-logos/openclaw.svg new file mode 100644 index 00000000..76c10463 --- /dev/null +++ b/images/provider-logos/openclaw.svg @@ -0,0 +1,29 @@ + + OpenClaw + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/images/provider-logos/openrouter.svg b/images/provider-logos/openrouter.svg new file mode 100644 index 00000000..d702d8e3 --- /dev/null +++ b/images/provider-logos/openrouter.svg @@ -0,0 +1,8 @@ + + OpenRouter + + + diff --git a/images/provider-logos/opentelemetry.svg b/images/provider-logos/opentelemetry.svg new file mode 100644 index 00000000..44715c68 --- /dev/null +++ b/images/provider-logos/opentelemetry.svg @@ -0,0 +1,8 @@ + + OpenTelemetry + + + diff --git a/images/provider-logos/pydantic.svg b/images/provider-logos/pydantic.svg new file mode 100644 index 00000000..c5736926 --- /dev/null +++ b/images/provider-logos/pydantic.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/images/provider-logos/vercel.svg b/images/provider-logos/vercel.svg new file mode 100644 index 00000000..c15fa175 --- /dev/null +++ b/images/provider-logos/vercel.svg @@ -0,0 +1,8 @@ + + Vercel + + + diff --git a/sdks/evals/agent-tracing.mdx b/sdks/evals/agent-tracing.mdx index 40651a44..83613348 100644 --- a/sdks/evals/agent-tracing.mdx +++ b/sdks/evals/agent-tracing.mdx @@ -31,7 +31,7 @@ Tool spans matter for exactly one thing: the [Trajectory](/sdks/evals/scorers/ov You don't create those spans by hand. Emit them with either: - **A framework helper** — OpenAI Agents, Claude, Vercel AI, and the rest in [Telemetry Integrations](/features/observability/traces/integrations) — which instruments your tools automatically. -- **`traceTool`** on a custom agent's tool handlers. It records the `Tool: ` span for you (on a tracing-enabled client); see [Tracing Tools](/features/observability/traces#tracing-tools) for the full contract. +- **`traceTool`** on a custom agent's tool handlers. It records the `Tool: ` span for you (on a tracing-enabled client); see [Tracing Tools](/features/observability/traces/manual-tracing#trace-tools) for the full contract. Only the tools you assert on (the names in Trajectory `expected`) need a span — a helper tool you don't score can stay untraced. How the modes treat what they observe: @@ -44,7 +44,7 @@ Only the tools you assert on (the names in Trajectory `expected`) need a span | Issue | Fix | | --- | --- | -| Empty Trace / Trajectory fails | Emit `Tool:` spans for the tools you assert on — a framework helper or [`traceTool`](/features/observability/traces#tracing-tools) on a tracing-enabled client | +| Empty Trace / Trajectory fails | Emit `Tool:` spans for the tools you assert on — a framework helper or [`traceTool`](/features/observability/traces/manual-tracing#trace-tools) on a tracing-enabled client | | Traced tool missing in `strict` mode | Add it to the expected list, or switch to `non_strict` | | Streamed return | Collect the final answer before returning it | | Awaitable on the sync Python path | Use `aevaluate` | diff --git a/sdks/evals/building-an-eval.mdx b/sdks/evals/building-an-eval.mdx index b80b7353..94d77a25 100644 --- a/sdks/evals/building-an-eval.mdx +++ b/sdks/evals/building-an-eval.mdx @@ -52,7 +52,7 @@ Cases feed the runner and scorers. [Datasets](/sdks/evals/datasets) covers inlin The `runner` is any callable that takes an `input` and returns a final value (not a stream) — no framework allowlist. -For Trajectory, tools must appear as `Tool: ` spans on the imported Trace; use a framework helper or [`traceTool`](/features/observability/traces#tracing-tools). The runner and tool-span rules live on [Runner](/sdks/evals/agent-tracing). +For Trajectory, tools must appear as `Tool: ` spans on the imported Trace; use a framework helper or [`traceTool`](/features/observability/traces/manual-tracing#trace-tools). The runner and tool-span rules live on [Runner](/sdks/evals/agent-tracing). ## Columns and scorers diff --git a/sdks/evals/quickstart.mdx b/sdks/evals/quickstart.mdx index 584d9b93..ea921b3f 100644 --- a/sdks/evals/quickstart.mdx +++ b/sdks/evals/quickstart.mdx @@ -443,7 +443,7 @@ Install: pip install promptlayer litellm ``` -Enable PromptLayer tracing with `PromptLayer(enable_tracing=True)`, wrap tools with [`traceTool`](/features/observability/traces#tracing-tools) for Trajectory, and set LiteLLM's PromptLayer callback for request logging. LiteLLM does not emit separate `Tool:` spans on its own — `traceTool` is a no-op unless that `PromptLayer` instance has tracing enabled. +Enable PromptLayer tracing with `PromptLayer(enable_tracing=True)`, wrap tools with [`traceTool`](/features/observability/traces/manual-tracing#trace-tools) for Trajectory, and set LiteLLM's PromptLayer callback for request logging. LiteLLM does not emit separate `Tool:` spans on its own — `traceTool` is a no-op unless that `PromptLayer` instance has tracing enabled. ```python # evals/litellm_weather.eval.py diff --git a/sdks/evals/scorers/overview.mdx b/sdks/evals/scorers/overview.mdx index cf0c267e..233a5007 100644 --- a/sdks/evals/scorers/overview.mdx +++ b/sdks/evals/scorers/overview.mdx @@ -593,7 +593,7 @@ await evaluate("llm-assert-demo", { ### Trajectory -Scores `Tool: ` spans from the imported `Trace` against accepted scenarios. Any `runner` works if those spans exist — emit them with a framework helper or [`traceTool`](/features/observability/traces#tracing-tools) (see [Runner](/sdks/evals/agent-tracing)), or copy a harness from the [Quickstart](/sdks/evals/quickstart). +Scores `Tool: ` spans from the imported `Trace` against accepted scenarios. Any `runner` works if those spans exist — emit them with a framework helper or [`traceTool`](/features/observability/traces/manual-tracing#trace-tools) (see [Runner](/sdks/evals/agent-tracing)), or copy a harness from the [Quickstart](/sdks/evals/quickstart). Pass **exactly one of** inline `expected` (list of tool-name lists), or `expected_column` / `expectedColumn` (usually `"expected_trace"` / `"expectedTrace"`). diff --git a/sdks/javascript.mdx b/sdks/javascript.mdx index 532a0b2f..ffc1b275 100644 --- a/sdks/javascript.mdx +++ b/sdks/javascript.mdx @@ -16,9 +16,9 @@ title: "JavaScript" npm install promptlayer ``` -## OpenAI SDK Auto-Instrumentation +## Provider SDK Auto-Instrumentation -PromptLayer can automatically trace supported direct OpenAI SDK calls. See [OpenAI SDK Auto-Instrumentation](/features/observability/traces/auto-instrumentation/openai) for the preload command, supported APIs, content capture, and flushing. +PromptLayer can automatically trace supported direct OpenAI, Anthropic, and Google GenAI SDK calls. See [SDK Auto-Instrumentation](/features/observability/traces/auto-instrumentation/overview) for provider guides, the ESM preload command, content capture, and flushing. ## SDK evals diff --git a/sdks/python.mdx b/sdks/python.mdx index 037240e9..0cd64d9f 100644 --- a/sdks/python.mdx +++ b/sdks/python.mdx @@ -16,9 +16,9 @@ title: 'Python' pip install promptlayer ``` -## OpenAI SDK Auto-Instrumentation +## Provider SDK Auto-Instrumentation -PromptLayer can automatically trace supported direct OpenAI SDK calls. See [OpenAI SDK Auto-Instrumentation](/features/observability/traces/auto-instrumentation/openai) for the tracing extra, initialization order, supported APIs, content capture, and flushing. +PromptLayer can automatically trace supported direct OpenAI, Anthropic, and Google GenAI SDK calls. See [SDK Auto-Instrumentation](/features/observability/traces/auto-instrumentation/overview) for the tracing extra, provider guides, content capture, and flushing. ## SDK evals From 8960af64bd7020aa12ac906e600cd69977710d91 Mon Sep 17 00:00:00 2001 From: C P Date: Tue, 4 Aug 2026 12:23:21 -0400 Subject: [PATCH 4/4] docs: add AWS Bedrock auto-instrumentation guide Document supported Boto3 Bedrock Runtime operations and setup. Clarify content capture, streaming, and Botocore-wide tracing behavior. Link the guide from observability navigation and Python SDK entry points. --- docs.json | 3 +- .../traces/auto-instrumentation/bedrock.mdx | 136 ++++++++++++++++++ .../traces/auto-instrumentation/overview.mdx | 13 +- .../observability/traces/manual-tracing.mdx | 2 +- sdks/python.mdx | 2 +- 5 files changed, 150 insertions(+), 6 deletions(-) create mode 100644 features/observability/traces/auto-instrumentation/bedrock.mdx diff --git a/docs.json b/docs.json index b19e716e..5ece07c2 100644 --- a/docs.json +++ b/docs.json @@ -117,7 +117,8 @@ "features/observability/traces/auto-instrumentation/overview", "features/observability/traces/auto-instrumentation/openai", "features/observability/traces/auto-instrumentation/anthropic", - "features/observability/traces/auto-instrumentation/google" + "features/observability/traces/auto-instrumentation/google", + "features/observability/traces/auto-instrumentation/bedrock" ] }, "features/observability/traces/opentelemetry", diff --git a/features/observability/traces/auto-instrumentation/bedrock.mdx b/features/observability/traces/auto-instrumentation/bedrock.mdx new file mode 100644 index 00000000..bb9f03e9 --- /dev/null +++ b/features/observability/traces/auto-instrumentation/bedrock.mdx @@ -0,0 +1,136 @@ +--- +title: "AWS Bedrock" +description: "Automatically trace supported direct Amazon Bedrock Runtime calls made with Boto3." +icon: "aws" +--- + +PromptLayer can auto-instrument Boto3 calls to Amazon Bedrock Runtime and export supported operations as OpenTelemetry spans. Each supported direct Bedrock call appears in PromptLayer as both a trace span and an associated request log without replacing the Boto3 client. + + +This Python integration covers direct Boto3 `bedrock-runtime` calls. It does not instrument `aioboto3`, Anthropic's Bedrock client, or the AWS SDK for JavaScript. + + +## Supported APIs + +| Boto3 Bedrock Runtime method | Python | +| --- | --- | +| Converse (`converse`) | Supported | +| Converse streaming (`converse_stream`) | Supported | +| InvokeModel (`invoke_model`) | Supported | +| InvokeModel streaming (`invoke_model_with_response_stream`) | Supported | + +Only the Bedrock Runtime operations in this table receive Bedrock-specific PromptLayer request log enrichment. For streaming operations, consume or close the response stream before flushing so the span can finish. + + +Bedrock auto-instrumentation uses the OpenTelemetry Botocore instrumentor. Enabling it also traces other Botocore service calls made by the same process, although Bedrock-specific request log enrichment applies only to the supported `bedrock-runtime` operations. + + +## Prerequisites + +- A [PromptLayer API key](/quickstart#prerequisites) for the workspace that should receive the traces +- AWS credentials, a region, and access to the Bedrock model or inference profile your application calls +- Python 3.10 or later + +Export the PromptLayer API key before the application starts. The example below also uses `AWS_REGION` and `AWS_BEDROCK_MODEL` as application configuration: + +```bash +export PROMPTLAYER_API_KEY="pl_..." +export AWS_REGION="us-east-1" +export AWS_BEDROCK_MODEL="your-model-or-inference-profile-id" +export OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT="span_only" +``` + +PromptLayer strongly recommends enabling content capture when your data policies allow it so request inspection, search, analytics, and debugging can use Bedrock messages and tool content. + +## Python + +### 1. Install the SDKs + +```bash +pip install "promptlayer[otel-genai-instrumentation]" boto3 +``` + +### 2. Configure Bedrock instrumentation + +Configure tracing before creating the Bedrock Runtime client or making the first request: + +```python +import os + +import boto3 +from promptlayer import configure_tracing + +tracer_provider = configure_tracing(providers=("bedrock",)) +client = boto3.client( + "bedrock-runtime", + region_name=os.environ["AWS_REGION"], +) + +try: + response = client.converse( + modelId=os.environ["AWS_BEDROCK_MODEL"], + messages=[ + { + "role": "user", + "content": [ + {"text": "Explain distributed tracing in one sentence."} + ], + } + ], + inferenceConfig={"maxTokens": 128}, + ) + print(response["output"]["message"]["content"][0]["text"]) +finally: + client.close() + tracer_provider.force_flush() +``` + +The canonical selector is `bedrock`. The `amazon.bedrock` and `aws.bedrock` aliases select the same Botocore instrumentor. + + +If your application already creates a PromptLayer client, select Bedrock when enabling tracing: + +```python +import boto3 +from promptlayer import PromptLayer + +pl = PromptLayer( + enable_tracing=True, + tracing_providers=("bedrock",), +) +client = boto3.client("bedrock-runtime", region_name="us-east-1") +``` + +Omit `tracing_providers` to instrument every supported provider SDK that is installed. + + +## Capture Prompts and Responses + +Bedrock request and response content is disabled by default because it can contain sensitive data. Set `OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT=span_only` before configuring tracing to add supported message content to PromptLayer spans. + +Content coverage differs by Bedrock Runtime operation: + +| Operation | Content added to the PromptLayer span | +| --- | --- | +| `converse` | Request messages, system instructions, and the response message | +| `converse_stream` | Request messages and system instructions; streamed response content is not added | +| `invoke_model` and `invoke_model_with_response_stream` | Request and response bodies are not added | + +All supported operations still include non-content telemetry supplied by the instrumentor, such as the model, timing, token usage when available, and errors. + + +Content capture can send user messages, model responses, system instructions, tool arguments and results, and other application data to PromptLayer. Review your privacy, retention, and compliance requirements before enabling it. + + +## Verify the Integration + +Run one supported Bedrock Runtime request, consume any response stream, flush tracing, and open [Traces](/features/observability/traces) in PromptLayer. The Bedrock span should have an associated request log and identify the Converse or InvokeModel API. + +If no span appears: + +- Confirm instrumentation is configured before the first Bedrock Runtime request. +- Confirm the Boto3 client uses `bedrock-runtime` and a method listed in [Supported APIs](#supported-apis). +- Confirm `PROMPTLAYER_API_KEY` belongs to the workspace you are checking. +- For streaming calls, fully consume or close the stream before flushing. +- Flush tracing before a short-lived process exits. +- If only message content is missing, check the content-capture value and the operation-specific coverage above, then restart the process. diff --git a/features/observability/traces/auto-instrumentation/overview.mdx b/features/observability/traces/auto-instrumentation/overview.mdx index ebc8a182..ca2e32d6 100644 --- a/features/observability/traces/auto-instrumentation/overview.mdx +++ b/features/observability/traces/auto-instrumentation/overview.mdx @@ -32,6 +32,13 @@ For the resulting hierarchy and dashboard view, see [Traces](/features/observabi > Python and JavaScript + + Python + The provider guide is the source of truth for supported API surfaces, installation, initialization order, content capture, exporter settings, flushing, and verification. Coverage and configuration can differ between languages. @@ -41,7 +48,7 @@ The provider guide is the source of truth for supported API surfaces, installati Python requires the optional instrumentation extra. Install only the provider SDKs your application uses: ```bash -pip install "promptlayer[otel-genai-instrumentation]" openai anthropic google-genai +pip install "promptlayer[otel-genai-instrumentation]" openai anthropic google-genai boto3 ``` Then enable tracing on the PromptLayer client. By default, the client instruments every supported provider SDK that is installed: @@ -51,7 +58,7 @@ from promptlayer import PromptLayer pl = PromptLayer(enable_tracing=True) -# Make direct OpenAI, Anthropic, or Google GenAI SDK calls. +# Make direct OpenAI, Anthropic, Google GenAI, or AWS Bedrock calls. # Flush pending spans before a short-lived process exits. pl.tracer_provider.force_flush() @@ -115,7 +122,7 @@ try { ``` -Supported selectors are `openai`, `anthropic`, and `google`. Python also accepts `openai.azure` as an alias for the OpenAI instrumentor. Anthropic Vertex uses `anthropic`; Google clients created with `vertexai=True` use `google`. +Supported selectors are `openai`, `anthropic`, and `google`. Python also supports `bedrock`, with `amazon.bedrock` and `aws.bedrock` as aliases for the Botocore instrumentor. Python accepts `openai.azure` as an alias for the OpenAI instrumentor. Anthropic Vertex uses `anthropic`; Google clients created with `vertexai=True` use `google`. Python applications that do not create a PromptLayer client can call `configure_tracing(providers=(...))`. The function returns the configured tracer provider so short-lived processes can call `force_flush()`. diff --git a/features/observability/traces/manual-tracing.mdx b/features/observability/traces/manual-tracing.mdx index cea0d221..827f96f0 100644 --- a/features/observability/traces/manual-tracing.mdx +++ b/features/observability/traces/manual-tracing.mdx @@ -40,7 +40,7 @@ const result = await pl.run({ For the full `run()` interface, see the [Python SDK](/sdks/python#using-the-run-method-recommended) or [JavaScript SDK](/sdks/javascript#using-the-run-method-recommended). -This setting also enables installed provider auto-instrumentation. To trace direct OpenAI, Anthropic, or Google GenAI SDK calls, follow [SDK Auto-Instrumentation](/features/observability/traces/auto-instrumentation/overview) for the required packages, supported APIs, and JavaScript initialization order. +This setting also enables installed provider auto-instrumentation. To trace direct OpenAI, Anthropic, or Google GenAI SDK calls, or Boto3 Bedrock Runtime calls in Python, follow [SDK Auto-Instrumentation](/features/observability/traces/auto-instrumentation/overview) for the required packages, supported APIs, and JavaScript initialization order. ## Add Custom Spans diff --git a/sdks/python.mdx b/sdks/python.mdx index 0cd64d9f..6dc94505 100644 --- a/sdks/python.mdx +++ b/sdks/python.mdx @@ -18,7 +18,7 @@ pip install promptlayer ## Provider SDK Auto-Instrumentation -PromptLayer can automatically trace supported direct OpenAI, Anthropic, and Google GenAI SDK calls. See [SDK Auto-Instrumentation](/features/observability/traces/auto-instrumentation/overview) for the tracing extra, provider guides, content capture, and flushing. +PromptLayer can automatically trace supported direct OpenAI, Anthropic, and Google GenAI SDK calls, plus AWS Bedrock Runtime calls through Boto3. See [SDK Auto-Instrumentation](/features/observability/traces/auto-instrumentation/overview) for the tracing extra, provider guides, content capture, and flushing. ## SDK evals