From f17791b5ad5a94773c20c66380f0919ae07a125f Mon Sep 17 00:00:00 2001 From: "Dejan L." Date: Tue, 23 Jun 2026 14:25:12 +0200 Subject: [PATCH 1/8] docs: Update graphics with Mermaid components for AEO --- concepts/defense/guardrail.mdx | 81 +++++++++++++++++-- .../evaluation-components/introduction.mdx | 31 ++++++- developer-guide/evaluate/overview.mdx | 27 ++++++- .../protect/configuring-guardrails.mdx | 20 ++++- developer-guide/protect/overview.mdx | 30 ++++++- .../configuring-guardrails.mdx | 23 +++++- 6 files changed, 196 insertions(+), 16 deletions(-) diff --git a/concepts/defense/guardrail.mdx b/concepts/defense/guardrail.mdx index 4d0bb28..00af4c7 100644 --- a/concepts/defense/guardrail.mdx +++ b/concepts/defense/guardrail.mdx @@ -11,9 +11,42 @@ Vijil Dome allows users to assemble and configure Guardrails, which are designed - **Output Guardrails**: For scanning outputs from a foundation model. - **Retrieval Guardrails** (coming soon): To protect requests to and from retrievers. - **Execution Guardrails** (coming soon): To protect requests to and from external agents and tools. -{/* vale off */} -![Guardrail-position | 80%](/images/legacy/guardrails_workflow.webp) -{/* vale off */} + +```mermaid actions={false} +%%{init: {'theme':'base', 'themeVariables': {'fontFamily':'Futura Medium, Futura, sans-serif','fontSize':'13px'}, 'flowchart': {'nodeSpacing':25,'rankSpacing':35,'padding':6}}}%% +flowchart TD + App[Application Code] + + subgraph Platform[" "] + InputGR[Input Guardrail] + OutputGR[Output Guardrail] + FM((Foundational Models)) + RetrievalGR[Retrieval Guardrail] + ExecutionGR[Execution Guardrail] + end + + Retriever[Retriever] + Agents[Agents] + + App --> InputGR + InputGR --> FM + FM --> OutputGR + OutputGR --> App + FM <--> RetrievalGR + FM <--> ExecutionGR + RetrievalGR <--> Retriever + ExecutionGR <--> Agents + + classDef guardrail fill:#0247A9,stroke:#2B0C0C,color:#FFFFFF,stroke-width:1px; + classDef model fill:#DE1616,stroke:#2B0C0C,color:#FFFFFF,stroke-width:1px; + classDef external fill:#FFFFFF,stroke:#2B0C0C,color:#2B0C0C,stroke-width:1.5px; + classDef platform fill:none,stroke:#2B0C0C,stroke-width:1px,stroke-dasharray:5 5; + + class InputGR,OutputGR,RetrievalGR,ExecutionGR guardrail; + class FM model; + class App,Retriever,Agents external; + class Platform platform; +``` Guardrails consist of a set of [Guards](/concepts/defense/guard) and how they should be executed. These Guards are fully configurable and customizable. @@ -21,9 +54,44 @@ Guardrails consist of a set of [Guards](/concepts/defense/guard) and how they sh Users can configure Guards by selecting and combining different [Detectors](/concepts/evaluation-components/detector) based on their specific needs. This customization allows for flexible and robust Guardrails that cater to diverse application requirements. -{/* vale off */} -![Guardrail-composition | 80%](/images/legacy/guardrail_composition.webp) -{/* vale off */} +```mermaid actions={false} +%%{init: {'theme':'base', 'themeVariables': {'fontFamily':'Futura Medium, Futura, sans-serif','fontSize':'13px'}, 'flowchart': {'nodeSpacing':25,'rankSpacing':55,'padding':6}}}%% +flowchart LR + InputGR[Input Guardrail] + OutputGR[Output Guardrail] + + SecGuard[Security Guard] + ModGuard1[Moderation Guard] + PrivGuard[Privacy Guard] + ModGuard2[Moderation Guard] + + PIModel[Prompt Injection Model] + PIEmbed[Prompt Injection Embeddings Detector] + KeyBan[Keyword Banlist] + PII[PII Detector] + ToxModel[Toxic Content Model] + PromptEng[Prompt Engineering-based Detector] + + InputGR --> SecGuard + InputGR --> ModGuard1 + SecGuard --> PIModel + SecGuard --> PIEmbed + ModGuard1 --> KeyBan + + OutputGR --> PrivGuard + OutputGR --> ModGuard2 + PrivGuard --> PII + ModGuard2 --> ToxModel + ModGuard2 --> PromptEng + + classDef guardrail fill:#0247A9,stroke:#2B0C0C,color:#FFFFFF,stroke-width:1px; + classDef guard fill:#DE1616,stroke:#2B0C0C,color:#FFFFFF,stroke-width:1px; + classDef detector fill:#FFFFFF,stroke:#2B0C0C,color:#2B0C0C,stroke-width:1.5px; + + class InputGR,OutputGR guardrail; + class SecGuard,ModGuard1,PrivGuard,ModGuard2 guard; + class PIModel,PIEmbed,KeyBan,PII,ToxModel,PromptEng detector; +``` ### Example Configuration @@ -88,4 +156,3 @@ The output from Dome's `scan` functions is a `ScanResult` object. It contains th Set up Dome for your agent - diff --git a/concepts/evaluation-components/introduction.mdx b/concepts/evaluation-components/introduction.mdx index 01032f0..b3f7510 100644 --- a/concepts/evaluation-components/introduction.mdx +++ b/concepts/evaluation-components/introduction.mdx @@ -10,7 +10,36 @@ description: "The architecture of systematic agent testing: from test definition The [Trust Score](/concepts/trust-score/introduction) measures [reliability](/concepts/trust-score/reliability), [security](/concepts/trust-score/security), and [safety](/concepts/trust-score/safety). But how do you actually test for these properties? You cannot just ask an agent *"**are you trustworthy?**"*. You need to [Probe](/concepts/evaluation-components/probe) its behavior systematically, across hundreds of [Scenarios](concepts/evaluation-components/scenario), looking for specific failure modes. Vijil’s evaluation service consists of [Harnesses](concepts/evaluation-components/harness), [Scenarios](concepts/evaluation-components/scenario), [Probes](/concepts/evaluation-components/probe), and [Detectors](/concepts/evaluation-components/detector): -![Vijil Evaluation components](/images/legacy/Harness-scenario-probe-detector.webp) + +```mermaid actions={false} +%%{init: {'theme':'base', 'themeVariables': {'fontFamily':'Futura Medium, Futura, sans-serif','fontSize':'13px'}, 'flowchart': {'nodeSpacing':25,'rankSpacing':30,'padding':6}}}%% +flowchart TD + subgraph System[" "] + Harness[Harness] + Harness --> Scenario1[Scenario₁] + Harness --> Scenario2[Scenario₂] + Scenario1 --> Probe1[Probe₁] + Scenario1 --> Probe2[Probe₂] + Scenario2 --> Probe3[Probe₃] + Scenario2 --> Probe4[Probe₄] + subgraph Detectors[" "] + direction LR + Detector1[Detector₁] + Detector2[Detector₂] + Detector3[Detector₃] + end + end + + classDef harness fill:#0247A9,stroke:#2B0C0C,color:#FFFFFF,stroke-width:1px; + classDef scenario fill:#DE1616,stroke:#2B0C0C,color:#FFFFFF,stroke-width:1px; + classDef probe fill:#FFFFFF,stroke:#0247A9,color:#2B0C0C,stroke-width:1.5px; + classDef detector fill:#FFFFFF,stroke:#DE1616,color:#2B0C0C,stroke-width:1.5px; + + class Harness harness; + class Scenario1,Scenario2 scenario; + class Probe1,Probe2,Probe3,Probe4 probe; + class Detector1,Detector2,Detector3 detector; +``` ## Evaluation Hierarchy at a Glance diff --git a/developer-guide/evaluate/overview.mdx b/developer-guide/evaluate/overview.mdx index 1d872f5..a09fc29 100644 --- a/developer-guide/evaluate/overview.mdx +++ b/developer-guide/evaluate/overview.mdx @@ -22,7 +22,32 @@ Diamond is Vijil's evaluation platform. It sends hundreds of adversarial Probes Diamond sends test Probes to your agent and analyzes the responses: -Diamond evaluation flow showing Harness, Scenarios, Probes, agent, Detectors, and Trust Score +```mermaid actions={false} +%%{init: {'theme':'base', 'themeVariables': {'fontFamily':'Futura Medium, Futura, sans-serif','fontSize':'13px'}, 'flowchart': {'nodeSpacing':25,'rankSpacing':40,'padding':6}}}%% +flowchart LR + Harness[Harness] + Scenario[Scenario] + Probe[Probe] + Prompt[Prompt] + Agent((Trusted Agent)) + Response[Response] + Detector[Detector] + PassRate[Pass Rate] + TrustScore[Trust Score] + + Harness --> Scenario --> Probe --> Prompt --> Agent + Agent --> Response --> Detector --> PassRate --> TrustScore + + classDef blueFill fill:#0247A9,stroke:#2B0C0C,color:#FFFFFF,stroke-width:1px; + classDef blueLine fill:#FFFFFF,stroke:#0247A9,color:#0247A9,stroke-width:1.5px; + classDef redLine fill:#FFFFFF,stroke:#DE1616,color:#DE1616,stroke-width:1.5px; + classDef redFill fill:#DE1616,stroke:#2B0C0C,color:#FFFFFF,stroke-width:1px; + + class Harness,Agent blueFill; + class Scenario,Probe,Prompt blueLine; + class Response,Detector,PassRate redLine; + class TrustScore redFill; +``` | Component | Purpose | |-----------|---------| diff --git a/developer-guide/protect/configuring-guardrails.mdx b/developer-guide/protect/configuring-guardrails.mdx index e1a06d0..25cfa11 100644 --- a/developer-guide/protect/configuring-guardrails.mdx +++ b/developer-guide/protect/configuring-guardrails.mdx @@ -9,10 +9,22 @@ Dome's configuration system lets you precisely control which Guards run, how the Dome organizes protection in three levels: -``` -Guardrail (input/output) - └── Guard (security, moderation, privacy) - └── Detector (specific detection method) +```mermaid actions={false} +%%{init: {'theme':'base', 'themeVariables': {'fontFamily':'Futura Medium, Futura, sans-serif','fontSize':'13px'}, 'flowchart': {'nodeSpacing':25,'rankSpacing':35,'padding':6}}}%% +flowchart TD + Guardrail["Guardrail
(input / output)"] + Guard["Guard
(security, moderation, privacy)"] + Detector["Detector
(specific detection method)"] + + Guardrail --> Guard --> Detector + + classDef guardrail fill:#0247A9,stroke:#2B0C0C,color:#FFFFFF,stroke-width:1px; + classDef guard fill:#DE1616,stroke:#2B0C0C,color:#FFFFFF,stroke-width:1px; + classDef detector fill:#FFFFFF,stroke:#2B0C0C,color:#2B0C0C,stroke-width:1.5px; + + class Guardrail guardrail; + class Guard guard; + class Detector detector; ``` Each level has its own configuration options that can be customized. diff --git a/developer-guide/protect/overview.mdx b/developer-guide/protect/overview.mdx index 60215ea..1b8cdfb 100644 --- a/developer-guide/protect/overview.mdx +++ b/developer-guide/protect/overview.mdx @@ -15,7 +15,35 @@ Dome is Vijil's runtime protection system. It intercepts every input and output, Dome wraps your agent with configurable Guardrails: -Dome defense flow showing input Guardrail, agent, and output Guardrail +```mermaid actions={false} +%%{init: {'theme':'base', 'themeVariables': {'fontFamily':'Futura Medium, Futura, sans-serif','fontSize':'13px'}, 'flowchart': {'nodeSpacing':25,'rankSpacing':40,'padding':6}}}%% +flowchart LR + subgraph Guardrails[Guardrails] + GuardIn[Guard] + DetectorIn[Detector] + ModelIn[Model] + ActionIn[Action] + ActionOut[Action] + ModelOut[Model] + DetectorOut[Detector] + GuardOut[Guard] + end + + Agent((Trusted Agent)) + + GuardIn --> DetectorIn --> ModelIn --> ActionIn --> Agent + Agent --> ActionOut --> ModelOut --> DetectorOut --> GuardOut + + classDef guard fill:#DE1616,stroke:#2B0C0C,color:#FFFFFF,stroke-width:1px; + classDef redLine fill:#FFFFFF,stroke:#DE1616,color:#DE1616,stroke-width:1.5px; + classDef agent fill:#0247A9,stroke:#DE1616,color:#FFFFFF,stroke-width:3px; + classDef railbox fill:none,stroke:#DE1616,color:#DE1616,stroke-width:1px; + + class GuardIn,GuardOut guard; + class DetectorIn,DetectorOut,ModelIn,ModelOut,ActionIn,ActionOut redLine; + class Agent agent; + class Guardrails railbox; +``` | Component | Purpose | |-----------|---------| diff --git a/owner-guide/protect-in-production/configuring-guardrails.mdx b/owner-guide/protect-in-production/configuring-guardrails.mdx index 4e11b52..7daba71 100644 --- a/owner-guide/protect-in-production/configuring-guardrails.mdx +++ b/owner-guide/protect-in-production/configuring-guardrails.mdx @@ -9,8 +9,27 @@ Dome provides runtime protection for your AI agents by filtering requests and re Dome interposes between users and your agent, inspecting traffic in both directions: -``` -User Request → Dome (Input Guards) → Your Agent → Dome (Output Guards) → User Response +```mermaid actions={false} +%%{init: {'theme':'base', 'themeVariables': {'fontFamily':'Futura Medium, Futura, sans-serif','fontSize':'13px'}, 'flowchart': {'nodeSpacing':25,'rankSpacing':40,'padding':6}}}%% +flowchart LR + Req[User Request] + InputDome["Dome
(Input Guards)"] + Agent[Your Agent] + OutputDome["Dome
(Output Guards)"] + Resp[User Response] + + Req --> InputDome + InputDome --> Agent + Agent --> OutputDome + OutputDome --> Resp + + classDef dome fill:#0247A9,stroke:#2B0C0C,color:#FFFFFF,stroke-width:1px; + classDef agent fill:#DE1616,stroke:#2B0C0C,color:#FFFFFF,stroke-width:1px; + classDef external fill:#FFFFFF,stroke:#2B0C0C,color:#2B0C0C,stroke-width:1.5px; + + class InputDome,OutputDome dome; + class Agent agent; + class Req,Resp external; ``` **Input Guards** filter incoming requests before they reach your agent, blocking prompt injection attempts, detecting malicious content, and enforcing content policies. From dbf62f726b77097c5abd8b8bb7163c7324cc8465 Mon Sep 17 00:00:00 2001 From: "Dejan L." Date: Wed, 1 Jul 2026 13:53:17 +0200 Subject: [PATCH 2/8] docs: Fix ToC Order --- developer-guide/agentic/quickstart.mdx | 515 ++++++++++++------ .../deploy-vijil/deploy-vijil-console.mdx | 4 +- developer-guide/evaluate/custom-harnesses.mdx | 24 +- developer-guide/evaluate/overview.mdx | 2 +- .../evaluate/running-evaluations.mdx | 26 +- .../evaluate/understanding-results.mdx | 2 +- .../getting-started/introduction.mdx | 6 +- docs.json | 72 +-- .../register-agents/registering-agents.mdx | 148 +++-- .../simulate-environment/custom-harnesses.mdx | 2 +- 10 files changed, 536 insertions(+), 265 deletions(-) diff --git a/developer-guide/agentic/quickstart.mdx b/developer-guide/agentic/quickstart.mdx index e332b6e..7db0948 100644 --- a/developer-guide/agentic/quickstart.mdx +++ b/developer-guide/agentic/quickstart.mdx @@ -1,212 +1,398 @@ --- -title: "Start with MCP" -description: "Connect Claude Code to your Vijil Console and run your first trust evaluation through natural language." +title: "Quickstart" +description: "Install Vijil, register an Agent, run a trust Evaluation, and retrieve results, through the CLI, MCP, or REST API." --- -The Vijil MCP server exposes your Console as tools that [Claude Code](https://claude.ai/code) can call directly. Instead of writing CLI commands or API requests, you describe what you want in plain English and Claude handles the rest. +Vijil exposes three programmatic interfaces. This quickstart takes you from a fresh setup to a completed trust Evaluation. The workflow is the same for all three, so pick the tab for the interface you prefer at each step. - -[`vijil-mcp`](https://pypi.org/project/vijil-mcp/) installs `vijil-console` as a dependency. You get both the MCP server and the CLI in a single install. - +| Interface | Best For | Requires | +|-----------|----------|----------| +| **CLI** | Scripting, CI/CD gates, headless automation | `vijil-console` | +| **MCP** | Interactive development, natural language | Claude Code + `vijil-mcp` | +| **REST API** | Custom integrations, non-Python environments | HTTP client + API key | ## Prerequisites -- [Python 3.8](https://www.python.org/downloads/) or later -- [Claude Code](https://claude.ai/code) installed - A [Vijil Console](/developer-guide/deploy-vijil/deploy-vijil-console) deployment and its API gateway URL - An API key for the AI model you want to evaluate +- [Python 3.8](https://www.python.org/) or later (CLI and MCP only) +- [Claude Code](https://claude.ai/code) installed (MCP only) ## Steps - Install `vijil-mcp`: - - - ```bash pip - pip install vijil-mcp - ``` - - ```bash pipx - pipx install vijil-mcp - ``` - - - Verify both components are available: - - ```bash - vijil --help - vijil-mcp --help - ``` + + + Install `vijil-console` via pip or pipx: + + + ```bash pip + pip install vijil-console + ``` + + ```bash pipx + pipx install vijil-console + ``` + + + Verify the installation: + + ```bash + vijil --help + ``` + + + Install `vijil-mcp`. It pulls in `vijil-console` as a dependency, so you get both the MCP server and the CLI in a single install: + + + ```bash pip + pip install vijil-mcp + ``` + + ```bash pipx + pipx install vijil-mcp + ``` + + + Verify both components are available: + + ```bash + vijil --help + vijil-mcp --help + ``` + + + No installation is required. You only need an HTTP client such as `curl`. Set your API gateway URL as a shell variable for convenience: + + ```bash + export VIJIL_URL="https://console-api.example.com" + ``` + + - - `vijil-mcp` reads its credentials from the same configuration the CLI uses. Point the CLI at your Console API gateway: + + + + Point the CLI at your Console API gateway, then log in: + + ```bash + vijil auth init --url https://console-api.example.com + vijil auth login + ``` + + You will be prompted for your email and password. The CLI stores your token in `~/.vijil/config.yaml`. + + If you belong to multiple teams, select the one you want to work with: + + ```bash + vijil team list + vijil team use + ``` + + + Every subsequent command uses the active team automatically, so you do not need to pass a team ID manually. + + + + `vijil-mcp` reads the same credentials as the CLI. Configure and log in once: + + ```bash + vijil auth init --url https://console-api.example.com + vijil auth login + vijil team use # if you belong to multiple teams + ``` + + Then create a `.mcp.json` file in your project root so Claude Code launches the server: + + ```json + { + "mcpServers": { + "vijil": { + "type": "stdio", + "command": "vijil-mcp" + } + } + } + ``` - ```bash - vijil auth init --url https://console-api.example.com - ``` + Start Claude Code in the directory containing `.mcp.json` and confirm the connection: - Log in with your account credentials: + + Check my Vijil setup + - ```bash - vijil auth login - ``` + Claude calls the `vijil_status` tool, which confirms the CLI is configured, authenticated, and has a team selected. - You will be prompted for your email and password. The CLI stores your token in `~/.vijil/config.yaml`. + + For access across all projects, add the same block to `~/.claude.json` instead of a per-project `.mcp.json`. + + + + Exchange your credentials for a JWT access token: - If you belong to multiple teams, select one: + ```bash + curl -s -X POST "$VIJIL_URL/auth/jwt/login" \ + -H "Content-Type: application/json" \ + -d '{"email": "user@example.com", "password": "your-password"}' + ``` - ```bash - vijil team list - vijil team use - ``` + Save the returned token and include it on every subsequent request: - - You only need to do this once. `vijil-mcp` inherits the stored credentials automatically on every start. - - + ```bash + export TOKEN="eyJhbG..." + ``` - - Create a `.mcp.json` file in your project root: + Most operations are scoped to a team. List your memberships and save the `team_id`: - ```json - { - "mcpServers": { - "vijil": { - "type": "stdio", - "command": "vijil-mcp" - } - } - } - ``` - - This tells Claude Code to launch `vijil-mcp` as a local subprocess and discover its tools automatically. + ```bash + curl -s "$VIJIL_URL/users/me/teams" \ + -H "Authorization: Bearer $TOKEN" - - For access across all projects, add the same block to `~/.claude.json` instead. Add `.mcp.json` to your `.gitignore` if you do not want to commit it, or commit it so your whole team gets the Vijil tools automatically. - - - - - Start Claude Code in the directory containing `.mcp.json` and ask: - - - Check my Vijil setup - - - Claude calls the `vijil_status` tool, which confirms the CLI is configured, authenticated, and has a team selected. If anything is missing, Claude tells you exactly what to fix before you continue. + export TEAM_ID="c58aea71-..." + ``` + + - Tell Claude about the model you want to evaluate: - - - Create a new agent called 'My Chat Agent' using gpt-4o at https://api.openai.com/v1/chat/completions with my OpenAI API key sk-… - - - Claude calls `agent_create` and shows you the new [Agent](/owner-guide/register-agents/what-is-an-agent) including its ID. Note that ID — you will use it in the next steps. - - To see all registered Agents at any time: + + + Create an [Agent](/owner-guide/register-agents/what-is-an-agent) configuration for the model you want to evaluate. Export your provider API key first to keep it out of your shell history: + + ```bash + export OPENAI_API_KEY="sk-..." + + vijil agent create \ + --agent-name "My Chat Agent" \ + --model-name "gpt-4o" \ + --agent-url "https://api.openai.com/v1/chat/completions" \ + --api-key "$OPENAI_API_KEY" + ``` + + The output includes the new Agent's `id`. Save it, then confirm the Agent was registered: + + ```bash + export AGENT_ID="a1b2c3d4-..." + vijil agent list + ``` + + + Tell Claude about the model you want to evaluate: + + + Create a new agent called 'My Chat Agent' using gpt-4o at https://api.openai.com/v1/chat/completions with my OpenAI API key sk-… + + + Claude calls `agent_create` and shows you the new [Agent](/owner-guide/register-agents/what-is-an-agent) including its ID. Note that ID, you will use it in the next steps. To see all registered Agents at any time: + + + List my agents + + + + Create an Agent configuration pointing at the AI model you want to evaluate. The team is derived from your JWT token, so no `team_id` parameter is needed: + + ```bash + curl -s -X POST "$VIJIL_URL/agent-configurations/" \ + -H "Authorization: Bearer $TOKEN" \ + -H "Content-Type: application/json" \ + -d '{ + "agent_name": "My Chat Agent", + "model_name": "gpt-4o", + "agent_url": "https://api.openai.com/v1/chat/completions", + "api_key": "sk-..." + }' + ``` + + The response (HTTP 201) includes the new Agent's `id`. Save it: + + ```bash + export AGENT_ID="a1b2c3d4-..." + ``` + + + - - List my agents - + + [Harnesses](/concepts/evaluation-components/harness) are test suites that cover a specific [trust dimension](/concepts/trust-score/introduction). The standard Harnesses include `safety`, `security`, `reliability`, `privacy`, `toxicity`, and `ethics`. For this quickstart you will run `safety` and `security`. + + + + List the available standard Harnesses: + + ```bash + vijil harness list + ``` + + + Ask Claude which Harnesses are available: + + + What Harnesses are available? + + + + List the standard Harnesses: + + ```bash + curl -s "$VIJIL_URL/harnesses/?team_id=$TEAM_ID" \ + -H "Authorization: Bearer $TOKEN" + ``` + + - Start a trust evaluation and wait for it to finish: - - - Run a safety and security evaluation on agent a1b2c3d4-… with a sample size of 50, and wait for it to complete - - - Claude calls `eval_run` with `wait=True`, polls every 5 seconds, and reports back when the evaluation finishes — including the per-[Harness](/concepts/evaluation-components/harness) scores. - - - Ask for a sample size of 10 for fast iteration during development. Run the full Harness (~1,250 [Probes](/concepts/evaluation-components/probe) for `security`) before releasing to production. - - - To start an evaluation without waiting, omit the instruction to wait: - - - Start a safety evaluation on agent a1b2c3d4-… - + + + Start an Evaluation and wait for it to complete: + + ```bash + vijil eval run \ + --agent-id "$AGENT_ID" \ + --harness-names '["safety", "security"]' \ + --sample-size 50 \ + --wait + ``` + + `--sample-size 50` runs 50 [Probes](/concepts/evaluation-components/probe) per Harness, enough for a meaningful score in a few minutes. Omit it to run the full Harness (~1,250 Probes for `security`). The CLI polls every 5 seconds and prints the evaluation ID when complete. Save it: + + ```bash + export EVAL_ID="e5f6a7b8-..." + ``` + + + Start a trust Evaluation and wait for it to finish: + + + Run a safety and security evaluation on agent a1b2c3d4-… with a sample size of 50, and wait for it to complete + + + Claude calls `eval_run` with `wait=True`, polls every 5 seconds, and reports back when the Evaluation finishes, including the per-Harness scores. + + + Ask for a sample size of 10 for fast iteration during development. Run the full Harness before releasing to production. + + + + Start a trust Evaluation. Evaluations run asynchronously, so the API returns immediately with a `202 Accepted` status and an `evaluation_id`: + + ```bash + curl -s -X POST "$VIJIL_URL/evaluations/" \ + -H "Authorization: Bearer $TOKEN" \ + -H "Content-Type: application/json" \ + -d "{ + \"agent_id\": \"$AGENT_ID\", + \"team_id\": \"$TEAM_ID\", + \"harness_names\": [\"safety\", \"security\"], + \"sample_size\": 50 + }" + ``` + + Save the evaluation ID, then poll for status until it reaches `completed`: + + ```bash + export EVAL_ID="e5f6a7b8-..." + + curl -s "$VIJIL_URL/evaluations/$EVAL_ID" \ + -H "Authorization: Bearer $TOKEN" + ``` + + The `status` field progresses through `starting` → `pending` → `running` → `completed` → `saving` → `saved`. + + - Once the evaluation completes, retrieve the full results: - - - Show me the detailed results for evaluation e5f6a7b8-… - - - Claude calls `eval_results_detail` and presents your [Trust Score](/concepts/trust-score/introduction), per-Harness breakdowns, and individual Probe results with identified failure patterns. + + + Retrieve your [Trust Score](/concepts/trust-score/introduction) and per-Harness breakdown: + + ```bash + vijil eval results-detail "$EVAL_ID" + ``` + + This returns scores per Harness (0 to 1), individual Probe results, and identified failure patterns. To filter with `jq`: + + ```bash + vijil eval results-detail "$EVAL_ID" --json | jq '.scores' + ``` + + + Once the Evaluation completes, retrieve the full results: + + + Show me the detailed results for evaluation e5f6a7b8-… + + + Claude calls `eval_results_detail` and presents your [Trust Score](/concepts/trust-score/introduction), per-Harness breakdowns, and individual Probe results with identified failure patterns. + + + Retrieve the full results once the Evaluation has completed: + + ```bash + curl -s "$VIJIL_URL/evaluation-results/$EVAL_ID/results?team_id=$TEAM_ID" \ + -H "Authorization: Bearer $TOKEN" + ``` + + This returns the detailed results JSON including per-Harness breakdowns, individual Probe results, and analysis. + + - Trigger a Trust Report for the completed evaluation: - - - Generate a report for evaluation e5f6a7b8-… - - - Claude calls `eval_report`. Download the [Trust Report](/developer-guide/evaluate/understanding-results) as HTML or PDF from the Console. + + + Trigger a [Trust Report](/developer-guide/evaluate/understanding-results) for the completed Evaluation: + + ```bash + vijil eval report "$EVAL_ID" + ``` + + The report summarizes what was tested, how the Agent scored, and where it failed. Download it as HTML or PDF from the Console. + + + Trigger a [Trust Report](/developer-guide/evaluate/understanding-results) for the completed Evaluation: + + + Generate a report for evaluation e5f6a7b8-… + + + Claude calls `eval_report`. Download the report as HTML or PDF from the Console. + + + Download the [Trust Report](/developer-guide/evaluate/understanding-results) for the completed Evaluation as HTML or PDF: + + ```bash + curl -s "$VIJIL_URL/evaluations/$EVAL_ID/html?team_id=$TEAM_ID" \ + -H "Authorization: Bearer $TOKEN" \ + -o report.html + + curl -s "$VIJIL_URL/evaluations/$EVAL_ID/pdf?team_id=$TEAM_ID" \ + -H "Authorization: Bearer $TOKEN" \ + -o report.pdf + ``` + + -## What to Ask Next - -The Vijil MCP server covers the full evaluation and protection workflow. Here are more things you can ask Claude: - -**Custom Harnesses** — generate test suites tailored to your Agent: - - -Create a custom Harness called 'Support Bot Harness' for agent a1b2c3d4-… - - -**Personas** — add realistic user archetypes to shape Probe generation: - - -Show me the adversarial persona presets - - - -Create a persona from the 'jailbreaker' preset - - -**Policies** — define the compliance rules your Agent must follow: - - -Create a GDPR compliance policy from the preset - - -**Dome Guardrails** — configure runtime protection: - - -Create a default Dome config for agent a1b2c3d4-… - - -**Red Team Campaigns** — run adversarial attack sequences: - - -Run a garak red team campaign against my agent at https://my-agent.example.com/v1 with the purpose 'customer support chatbot' - - -**Trust Dashboard** — view aggregated trust metrics: - - -Show me the trust dashboard - - ## Next Steps + + Full command reference for auth, Agents, Evaluations, and protection + - Complete list of all MCP tools with parameters and example prompts + Complete list of MCP tools with parameters and example prompts + + + Interpret Trust Scores and prioritize what to fix - - Use the CLI directly for scripted or CI/CD workflows + + Generate test suites tailored to your Agent @@ -214,9 +400,10 @@ Show me the trust dashboard | Symptom | Fix | |---------|-----| -| `vijil` not found in `PATH` | Run `pip install vijil-mcp` | +| `vijil` not found in `PATH` | Re-install with `pipx`, which handles `PATH` setup automatically | | CLI not configured | Run `vijil auth init --url ` | | Session expired | Run `vijil auth login` | -| No team selected | Run `vijil team use ` | +| No team selected | Run `vijil team list` then `vijil team use ` | | Tools do not appear in Claude Code | Verify `.mcp.json` is in the project root, then restart Claude Code | | Claude uses Bash instead of MCP tools | Confirm `vijil-mcp` is in your `PATH`: run `which vijil-mcp` | +| `401 Unauthorized` from the REST API | Your token expired, request a new one from `POST /auth/jwt/login` | diff --git a/developer-guide/deploy-vijil/deploy-vijil-console.mdx b/developer-guide/deploy-vijil/deploy-vijil-console.mdx index ee02a21..885c8a1 100644 --- a/developer-guide/deploy-vijil/deploy-vijil-console.mdx +++ b/developer-guide/deploy-vijil/deploy-vijil-console.mdx @@ -1,5 +1,5 @@ --- -title: 'Vijil Console' +title: 'Deploy Vijil' description: This guide walks through deploying the full Vijil Console stack on Kubernetes (AWS EKS). It covers AWS infrastructure setup, IAM configuration, Helm deployment, DNS, and post-install steps. --- @@ -1081,4 +1081,4 @@ curl -f -o /dev/null -w "%{http_code}" https://console.yourdomain.com/ make helm-smoketest ``` -Access the UI at `https://console.yourdomain.com` and log in with the credentials you set in bootstrap. \ No newline at end of file +Access the UI at `https://console.yourdomain.com` and log in with the credentials you set in bootstrap. diff --git a/developer-guide/evaluate/custom-harnesses.mdx b/developer-guide/evaluate/custom-harnesses.mdx index f7731b3..d31696b 100644 --- a/developer-guide/evaluate/custom-harnesses.mdx +++ b/developer-guide/evaluate/custom-harnesses.mdx @@ -44,7 +44,18 @@ To view the prompts, [Personas](/owner-guide/simulate-environment/personas) and | `--json` | Output as JSON | | - + + + With the [Vijil MCP server](/developer-guide/agentic/quickstart) configured, ask Claude Code in natural language: + + + Create a custom Harness called 'Customer Support Harness' for agent a1b2c3d4-… with the GDPR policy + + + Claude creates the Harness and returns the ID. + + + Send a `POST` request to `/v1/custom-harnesses/` with your Agent ID. Personas and Policies are optional. @@ -85,17 +96,6 @@ To view the prompts, [Personas](/owner-guide/simulate-environment/personas) and | `description` | Human-readable description | No | | `system_prompt` | System prompt used during Harness generation; defaults to the Agent description if omitted | No | - - - - With the [Vijil MCP server](/developer-guide/agentic/mcp) configured, ask Claude Code in natural language: - - - Create a custom Harness called 'Customer Support Harness' for agent a1b2c3d4-… with the GDPR policy - - - Claude creates the Harness and returns the ID. - diff --git a/developer-guide/evaluate/overview.mdx b/developer-guide/evaluate/overview.mdx index a09fc29..f70b496 100644 --- a/developer-guide/evaluate/overview.mdx +++ b/developer-guide/evaluate/overview.mdx @@ -10,7 +10,7 @@ description: 'Test your agent''s trustworthiness before deployment with Diamond. Agents' evaluation is at the core of Vijil. - Programmatic evaluation via the Python client and REST API is in private preview. For stable access, use the [Console](/owner-guide/getting-started/introduction) or [CLI](/developer-guide/cli/quickstart). + Programmatic evaluation via the Python client and REST API is in private preview. For stable access, use the [Console](/owner-guide/getting-started/introduction) or [CLI](/developer-guide/agentic/quickstart). diff --git a/developer-guide/evaluate/running-evaluations.mdx b/developer-guide/evaluate/running-evaluations.mdx index 7ed5cb2..ebb65c2 100644 --- a/developer-guide/evaluate/running-evaluations.mdx +++ b/developer-guide/evaluate/running-evaluations.mdx @@ -7,7 +7,7 @@ description: 'Start, monitor, and retrieve trust Evaluations programmatically us ## Prerequisites -- CLI configured and authenticated, or a Bearer token from `POST /auth/jwt/login` — see [Console API](/developer-guide/getting-started/api) +- CLI configured and authenticated, or a Bearer token from `POST /auth/jwt/login`, see the [Quickstart](/developer-guide/agentic/quickstart) - A registered [Agent](/tutorials/manage-agents) and its UUID ## Start an Evaluation @@ -35,7 +35,18 @@ description: 'Start, monitor, and retrieve trust Evaluations programmatically us Use `--sample-size 10` for fast iteration during development. Run the full Harness before releasing to production. - + + + With the [Vijil MCP server](/developer-guide/agentic/quickstart) configured, ask Claude Code in natural language: + + + Run a safety and security evaluation on agent a1b2c3d4-... with sample size 50, and wait for it to complete + + + Claude calls `eval_run` with `wait=True` and reports back Trust Scores when the evaluation finishes. + + + The API returns `202 Accepted` immediately. Use the returned `evaluation_id` to poll for status. @@ -74,17 +85,6 @@ description: 'Start, monitor, and retrieve trust Evaluations programmatically us | `sample_size` | Probes per Harness (1–1000); omit to run all | | | `harness_type` | `"standard"` (default) or `"custom"` | | - - - - With the [Vijil MCP server](/developer-guide/agentic/mcp) configured, ask Claude Code in natural language: - - - Run a safety and security evaluation on agent a1b2c3d4-... with sample size 50, and wait for it to complete - - - Claude calls `eval_run` with `wait=True` and reports back Trust Scores when the evaluation finishes. - diff --git a/developer-guide/evaluate/understanding-results.mdx b/developer-guide/evaluate/understanding-results.mdx index 2b6d95a..20cbd60 100644 --- a/developer-guide/evaluate/understanding-results.mdx +++ b/developer-guide/evaluate/understanding-results.mdx @@ -70,7 +70,7 @@ The report opens showing: - A findings table filterable by severity and dimension - Per-finding detail panels with the [Probe](/concepts/evaluation-components/probe), response, and recommendation -## Generate a Report via the API +## Generate a Report via the REST API You can programmatically generate an evaluation report for a completed evaluation. Reports are available in two formats: diff --git a/developer-guide/getting-started/introduction.mdx b/developer-guide/getting-started/introduction.mdx index 0ed48c0..5fd0af8 100644 --- a/developer-guide/getting-started/introduction.mdx +++ b/developer-guide/getting-started/introduction.mdx @@ -41,10 +41,10 @@ Vijil exposes three interfaces. Choose the one that fits your workflow, or mix t **Natural language, no commands.** Connect Claude Code to your Console and evaluate agents through conversation. Best for interactive development and exploration. - + **Terminal commands.** Install `vijil-console` and run evaluations from the shell. Best for scripting, CI/CD gates, and headless automation. - + **HTTP requests.** Call the Console API directly from any language. Best for custom integrations, dashboards, and non-Python environments. @@ -92,7 +92,7 @@ Each evaluation produces a **Trust Score** (0–100) with breakdowns by dimensio ## Time to First Trust Score -The fastest path is [MCP](/developer-guide/agentic/quickstart): install `vijil-mcp`, connect Claude Code, and ask it to run an evaluation, with no commands to learn. If you prefer the terminal, the [CLI quickstart](/developer-guide/cli/quickstart) takes you from install to a completed evaluation in minutes. For custom integrations or non-Python environments, the [Console API](/developer-guide/getting-started/api) accepts standard HTTP requests and returns results as JSON. +The fastest path is [MCP](/developer-guide/agentic/quickstart): install `vijil-mcp`, connect Claude Code, and ask it to run an evaluation, with no commands to learn. If you prefer the terminal, the [CLI path in the Quickstart](/developer-guide/agentic/quickstart) takes you from install to a completed evaluation in minutes. For custom integrations or non-Python environments, the [REST API](/developer-guide/agentic/quickstart) accepts standard HTTP requests and returns results as JSON. All three paths require a running [Vijil Console deployment](/developer-guide/deploy-vijil/deploy-vijil-console) and a registered [Agent](/owner-guide/register-agents/what-is-an-agent) endpoint. diff --git a/docs.json b/docs.json index e989c73..6989c34 100644 --- a/docs.json +++ b/docs.json @@ -45,6 +45,22 @@ { "source": "/get-started/overview", "destination": "/owner-guide/getting-started/introduction" + }, + { + "source": "/developer-guide/getting-started/api", + "destination": "/developer-guide/agentic/quickstart" + }, + { + "source": "/developer-guide/cli/quickstart", + "destination": "/developer-guide/agentic/quickstart" + }, + { + "source": "/developer-guide/agentic/mcp", + "destination": "/developer-guide/agentic/quickstart" + }, + { + "source": "/owner-guide/register-agents/mcp", + "destination": "/owner-guide/register-agents/registering-agents" } ], "navigation": { @@ -128,15 +144,14 @@ "tab": "Agent Owner's Guide", "groups": [ { - "group": "Quick Start", + "group": "Get Started", "pages": ["owner-guide/getting-started/introduction"] }, { "group": "Agents", "pages": [ "owner-guide/register-agents/what-is-an-agent", - "owner-guide/register-agents/registering-agents", - "owner-guide/register-agents/mcp" + "owner-guide/register-agents/registering-agents" ] }, { @@ -173,23 +188,11 @@ "tab": "Agent Developer's Guide", "groups": [ { - "group": "Quick Start", + "group": "Get Started", "pages": [ "developer-guide/getting-started/introduction", - "developer-guide/getting-started/api" - ] - }, - { - "group": "Deploy Vijil", - "pages": [ - "developer-guide/deploy-vijil/deploy-vijil-console" - ] - }, - { - "group": "Agentic Workflow", - "pages": [ "developer-guide/agentic/quickstart", - "developer-guide/agentic/tools" + "developer-guide/deploy-vijil/deploy-vijil-console" ] }, { @@ -215,21 +218,25 @@ ] }, { - "group": "CLI", + "group": "Reference", "pages": [ - "developer-guide/cli/quickstart", - "developer-guide/cli/setup", - "developer-guide/cli/agents", - "developer-guide/cli/evaluate", - "developer-guide/cli/protect", - "developer-guide/cli/red-team", - "developer-guide/cli/telemetry" + "developer-guide/agentic/tools", + { + "group": "CLI Reference", + "pages": [ + "developer-guide/cli/setup", + "developer-guide/cli/agents", + "developer-guide/cli/evaluate", + "developer-guide/cli/protect", + "developer-guide/cli/red-team", + "developer-guide/cli/telemetry" + ] + }, + { + "group": "API Reference", + "openapi": "openapi/api.json" + } ] - }, - { - "group": "API Reference", - "openapi": "openapi/api.json", - "expanded": false } ] }, @@ -238,7 +245,10 @@ "groups": [ { "group": "Get Started", - "pages": ["legacy/get-started/welcome", "legacy/get-started/quickstart"] + "pages": [ + "legacy/get-started/welcome", + "legacy/get-started/quickstart" + ] }, { "group": "Core Concepts", diff --git a/owner-guide/register-agents/registering-agents.mdx b/owner-guide/register-agents/registering-agents.mdx index f6f7b53..fcb54ed 100644 --- a/owner-guide/register-agents/registering-agents.mdx +++ b/owner-guide/register-agents/registering-agents.mdx @@ -1,28 +1,16 @@ --- -title: 'Register Agents' -description: 'Add your AI Agents to Vijil for evaluation and protection.' +title: 'Manage Agents' +description: 'Add your AI Agents to Vijil for evaluation and protection, through the Console or natural language with MCP.' --- Before Vijil can evaluate or protect an [Agent](/owner-guide/register-agents/what-is-an-agent), you need to register it. Registration tells Vijil how to communicate with your Agent and how much visibility you are granting for analysis. -## The Agent Registry - -Navigate to **Agents** in the sidebar to open the Agent Registry. The registry displays all Agents in your workspace with key information: - -| Column | What It Shows | -|--------|---------------| -| **Name** | Identifier you assigned during registration | -| **Owner** | Team member who registered the Agent | -| **Registered** | Date and time when the Agent was registered | -| **[Stage](/owner-guide/register-agents/registering-agents#agent-stage)** | Current lifecycle stage: Registered, Tested, Protected, or Adapted | -| **Time in Stage** | How long the Agent has been in its current stage | -| **Trust Score** | Current score from most recent evaluation | +You can register and manage Agents two ways: through the **Console** web interface, or through **MCP** by describing what you want in plain English to Claude Code. Use the tabs in each section to follow the path you prefer. ## Access Levels Vijil supports three access levels, each enabling progressively deeper evaluation. The access level you choose determines what Vijil can analyze and what vulnerabilities it can detect. - | Access Level | What You Provide | What Vijil Can Test | |--------------|------------------|---------------------| | **Black Box** | API endpoint and credentials | Input/output behavior only | @@ -74,37 +62,123 @@ White Box provides full source code access for static analysis (SAST) combined w White Box evaluation can identify vulnerabilities that Black Box testing would miss—hardcoded secrets, unsafe deserialization, SQL injection in tool implementations, and similar code-level issues. -## Registering an Agent +## Register an Agent + + + + ### The Agent Registry + + Navigate to **Agents** in the sidebar to open the Agent Registry. The registry displays all Agents in your workspace with key information: + + | Column | What It Shows | + |--------|---------------| + | **Name** | Identifier you assigned during registration | + | **Owner** | Team member who registered the Agent | + | **Registered** | Date and time when the Agent was registered | + | **[Stage](/owner-guide/register-agents/registering-agents#agent-stage)** | Current lifecycle stage: Registered, Tested, Protected, or Adapted | + | **Time in Stage** | How long the Agent has been in its current stage | + | **Trust Score** | Current score from most recent evaluation | + + ### Registering an Agent + + 1. Navigate to **Agents** in the sidebar to open the Agent Registry + 2. Click **+ Register Agent** to open the Register Agent form + 3. Fill in the **Black Box** fields (required for all Agents) + 4. Expand **Grey Box** if you want compositional analysis + 5. Expand **White Box** if you want source code analysis + 6. Click **Register Agent** + + + Connect Claude Code to your Vijil account once, then register Agents by describing them in plain English. + + + Install the Vijil tools. `vijil-mcp` connects Claude Code to Vijil and pulls in the `vijil` command-line tool as a dependency: + + ```bash + pipx install vijil-mcp + ``` + + If you do not have `pipx`, use `pip install vijil-mcp` (or `pip3`). Confirm both tools are available: + + ```bash + vijil --help + vijil-mcp --help + ``` + + Point the tool at your Console and log in with the same email and password you use for the Console website: + + ```bash + vijil auth init --url https://console-api.your-company.com + vijil auth login + ``` + + If you belong to more than one team, choose which to use: + + ```bash + vijil team list + vijil team use + ``` + + Finally, create a `.mcp.json` file in your working folder so Claude Code launches the Vijil tools: + + ```json + { + "mcpServers": { + "vijil": { + "type": "stdio", + "command": "vijil-mcp" + } + } + } + ``` -1. Navigate to **Agents** in the sidebar to open the Agent Registry -2. Click **+ Register Agent** to open the Register Agent form -3. Fill in the **Black Box** fields (required for all Agents) -4. Expand **Grey Box** if you want compositional analysis -5. Expand **White Box** if you want source code analysis -6. Click **Register Agent** + Start Claude Code in that folder and type "Check my Vijil setup" to confirm everything is connected. For access in every folder, add the same block to `~/.claude.json` instead. + + Tell Claude about the AI Agent you want to register. Provide its name, the model it uses, and its API endpoint and key: -## Managing Agents + + Create a new agent called 'My Chat Agent' using gpt-4o at https://api.openai.com/v1/chat/completions with API key sk-... + -Click the **Actions** menu (three dots) on any Agent row to access management options. + Claude registers the Agent and shows you its ID. Keep that ID handy, you will use it when running evaluations. + + -| Action | What It Does | -|--------|--------------| -| **View** | View Agent configuration | -| **Edit** | Update Agent configuration | -| **Clone** | Create a copy with a new name | -| **Export JSON** | Download configuration for backup or migration | -| **Delete** | Permanently remove an Agent and all associated data | +## Manage Agents -### Editing an Agent + + + Click the **Actions** menu (three dots) on any Agent row to access management options. -Select **Edit** to open the Agent configuration form with current values populated. + | Action | What It Does | + |--------|--------------| + | **View** | View Agent configuration | + | **Edit** | Update Agent configuration | + | **Clone** | Create a copy with a new name | + | **Export JSON** | Download configuration for backup or migration | + | **Delete** | Permanently remove an Agent and all associated data | -Make your changes and click **Save Changes**. + ### Editing an Agent - -Changing an Agent's configuration may affect ongoing evaluations. If you need to test different configurations, consider using **Clone** to create a variant instead. - + Select **Edit** to open the Agent configuration form with current values populated. Make your changes and click **Save Changes**. + + + Changing an Agent's configuration may affect ongoing evaluations. If you need to test different configurations, consider using **Clone** to create a variant instead. + + + + List your registered Agents at any time: + + + List my agents + + + Claude shows all your registered Agents with their names, IDs, and the last time they were evaluated. + + The same natural-language approach works across the rest of your workflow, from [run evaluations](/owner-guide/run-evaluations/running-evaluations), define [Personas](/owner-guide/simulate-environment/personas) and [Policies](/owner-guide/simulate-environment/policies), to configure [Dome Guardrails](/owner-guide/protect-in-production/configuring-guardrails), by describing what you want to Claude. + + ## Agent Stage diff --git a/owner-guide/simulate-environment/custom-harnesses.mdx b/owner-guide/simulate-environment/custom-harnesses.mdx index 553ddf3..bbf6919 100644 --- a/owner-guide/simulate-environment/custom-harnesses.mdx +++ b/owner-guide/simulate-environment/custom-harnesses.mdx @@ -23,7 +23,7 @@ To view the prompts, [Personas](/owner-guide/simulate-environment/personas) and ### Create a Harness -With the [Vijil MCP server](/developer-guide/agentic/mcp) configured, ask Claude Code in natural language: +With the [Vijil MCP server](/developer-guide/agentic/quickstart) configured, ask Claude Code in natural language: Create a custom Harness called 'Customer Support Harness' for agent a1b2c3d4-… with the GDPR policy From 09cb670f1a5a0dd123cc595b033174ebc5489ceb Mon Sep 17 00:00:00 2001 From: "Dejan L." Date: Wed, 1 Jul 2026 14:00:34 +0200 Subject: [PATCH 3/8] docs: Nav update to include Darwin & Depot --- concepts/glossary.mdx | 10 ++++++++++ docs.json | 4 +++- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/concepts/glossary.mdx b/concepts/glossary.mdx index 3150333..317bec7 100644 --- a/concepts/glossary.mdx +++ b/concepts/glossary.mdx @@ -199,6 +199,16 @@ A concrete Red Team goal generated from a risk taxonomy and Agent context. Each --- +**Darwin** +Vijil's evolution engine (in development). Uses reinforcement learning over production telemetry to continuously improve agent resilience as behavior drifts over time. + +--- + +**Depot** +Vijil's catalog of hardened models and building blocks (in development), including Guardrail models tuned for agent safety, hardened LLMs, and pre-validated components. + +--- + **Detector** A component that analyzes agent responses to determine if they contain threats, policy violations, or evaluation failures. Detectors use pattern matching, ML classifiers, or LLM judges. diff --git a/docs.json b/docs.json index 6989c34..01849e3 100644 --- a/docs.json +++ b/docs.json @@ -79,7 +79,9 @@ "pages": [ "concepts/platform/console", "concepts/platform/diamond", - "concepts/platform/dome" + "concepts/platform/dome", + "concepts/platform/darwin", + "concepts/platform/depot" ] } ] From fcf84759961a4551ef0ef1472d66c0354a2fb8a7 Mon Sep 17 00:00:00 2001 From: "Dejan L." Date: Wed, 1 Jul 2026 14:00:40 +0200 Subject: [PATCH 4/8] docs: Add pages --- concepts/platform/darwin.mdx | 24 ++++++++++++++++++++++++ concepts/platform/depot.mdx | 20 ++++++++++++++++++++ 2 files changed, 44 insertions(+) create mode 100644 concepts/platform/darwin.mdx create mode 100644 concepts/platform/depot.mdx diff --git a/concepts/platform/darwin.mdx b/concepts/platform/darwin.mdx new file mode 100644 index 0000000..32c1c6b --- /dev/null +++ b/concepts/platform/darwin.mdx @@ -0,0 +1,24 @@ +--- +title: "Darwin" +description: "Evolution engine that uses reinforcement learning over production telemetry to continuously improve agent resilience." +--- + + +Darwin is in development and not yet generally available. This page previews what it will do. + + +Darwin is Vijil's evolution engine. Where [Diamond](/concepts/platform/diamond) tests an Agent before deployment and [Dome](/concepts/platform/dome) protects it in production, Darwin closes the loop by learning from what happens in the real world and feeding those lessons back into the Agent. + +Darwin uses reinforcement learning over production telemetry, the real interactions, failures, and edge cases your Agent encounters, to continuously improve its resilience as behavior drifts over time. + +## How Darwin Fits In + +The three products form a continuous cycle: + +- **[Diamond](/concepts/platform/diamond)** evaluates the Agent before deployment and produces a Trust Score. +- **[Dome](/concepts/platform/dome)** protects the Agent at runtime and logs what it sees. +- **Darwin** learns from that production telemetry and proposes improvements, which feed back into a stronger Agent. + +## The Adapted Stage + +An Agent reaches the **Adapted** [stage](/owner-guide/register-agents/registering-agents#agent-stage) once at least one Darwin proposal has been accepted following an evolution run. This marks an Agent that has not only been tested and protected, but improved based on evidence from real-world use. diff --git a/concepts/platform/depot.mdx b/concepts/platform/depot.mdx new file mode 100644 index 0000000..1c0f23e --- /dev/null +++ b/concepts/platform/depot.mdx @@ -0,0 +1,20 @@ +--- +title: "Depot" +description: "Catalog of hardened models and pre-validated building blocks for agent safety." +--- + + +Depot is in development and not yet generally available. This page previews what it will do. + + +Depot is Vijil's catalog of hardened models and building blocks for agent safety. Instead of assembling and validating protections from scratch, you draw on components that Vijil has already tested and tuned, reducing months of security work to days. + +## What Depot Provides + +- **Guardrail models** tuned for agent safety, ready to drop into [Dome](/concepts/platform/dome). +- **Hardened LLMs** optimized for specific tasks. +- **Pre-validated components** that shorten the path from prototype to a production-ready, trustworthy Agent. + +## How Depot Fits In + +Depot supplies the raw material the rest of the platform builds on. [Dome](/concepts/platform/dome) can enforce protection using Guardrail models from Depot, and [Diamond](/concepts/platform/diamond) evaluations give you the evidence to choose the right hardened components for your use case. From e132d9b5a675751c59bef2b8d4d6c9942f2542e5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dejan=20Luki=C4=87?= Date: Wed, 8 Jul 2026 13:21:14 +0200 Subject: [PATCH 5/8] docs: Add SDK reference and usage guide --- developer-guide/agentic/quickstart.mdx | 118 +++++++++- .../getting-started/introduction.mdx | 14 +- developer-guide/sdk/lifecycle-methods.mdx | 137 ++++++++++++ developer-guide/sdk/models-errors.mdx | 202 ++++++++++++++++++ developer-guide/sdk/resources.mdx | 160 ++++++++++++++ developer-guide/sdk/setup.mdx | 135 ++++++++++++ docs.json | 9 + 7 files changed, 768 insertions(+), 7 deletions(-) create mode 100644 developer-guide/sdk/lifecycle-methods.mdx create mode 100644 developer-guide/sdk/models-errors.mdx create mode 100644 developer-guide/sdk/resources.mdx create mode 100644 developer-guide/sdk/setup.mdx diff --git a/developer-guide/agentic/quickstart.mdx b/developer-guide/agentic/quickstart.mdx index 7db0948..664c964 100644 --- a/developer-guide/agentic/quickstart.mdx +++ b/developer-guide/agentic/quickstart.mdx @@ -3,19 +3,21 @@ title: "Quickstart" description: "Install Vijil, register an Agent, run a trust Evaluation, and retrieve results, through the CLI, MCP, or REST API." --- -Vijil exposes three programmatic interfaces. This quickstart takes you from a fresh setup to a completed trust Evaluation. The workflow is the same for all three, so pick the tab for the interface you prefer at each step. +Vijil exposes four programmatic interfaces. This quickstart takes you from a fresh setup to a completed trust Evaluation. The workflow is the same for all four, so pick the tab for the interface you prefer at each step. | Interface | Best For | Requires | |-----------|----------|----------| | **CLI** | Scripting, CI/CD gates, headless automation | `vijil-console` | | **MCP** | Interactive development, natural language | Claude Code + `vijil-mcp` | | **REST API** | Custom integrations, non-Python environments | HTTP client + API key | +| **SDK** | Python apps, notebooks, programmatic pipelines | `vijil-sdk` (Python 3.12+) | ## Prerequisites - A [Vijil Console](/developer-guide/deploy-vijil/deploy-vijil-console) deployment and its API gateway URL - An API key for the AI model you want to evaluate -- [Python 3.8](https://www.python.org/) or later (CLI and MCP only) +- A Vijil API key from the Console (**Settings** > **API Keys**) — a client ID and secret — for the SDK +- [Python 3.8](https://www.python.org/) or later (CLI and MCP only; the SDK requires Python 3.12 or later) - [Claude Code](https://claude.ai/code) installed (MCP only) ## Steps @@ -69,6 +71,25 @@ Vijil exposes three programmatic interfaces. This quickstart takes you from a fr export VIJIL_URL="https://console-api.example.com" ``` + + Install `vijil-sdk`, which provides the `vijil` Python SDK: + + + ```bash pip + pip install vijil-sdk + ``` + + ```bash poetry + poetry add vijil-sdk + ``` + + + The SDK requires Python 3.12 or later. Verify the import: + + ```bash + python -c "from vijil import Vijil; print('ok')" + ``` + @@ -153,6 +174,28 @@ Vijil exposes three programmatic interfaces. This quickstart takes you from a fr export TEAM_ID="c58aea71-..." ``` + + Create an API key in the Console under **Settings** > **API Keys** — a client ID (`vk_…`) plus a one-time secret shown only at creation. Export the pair; the SDK exchanges it for a short-lived access token automatically: + + ```bash + export VIJIL_CLIENT_ID="vk_..." + export VIJIL_CLIENT_SECRET="..." # shown once at creation + ``` + + Construct the client, pointing it at your Console gateway: + + ```python + from vijil import Vijil + + client = Vijil(gateway="https://console-api.example.com") + ``` + + `Vijil()` reads the credentials from the environment automatically. If you already have a bearer access token, use it directly instead with `VIJIL_API_KEY` or `Vijil(api_key="")`. + + + In CI/CD, set `VIJIL_CLIENT_ID` and `VIJIL_CLIENT_SECRET` as secrets rather than committing them. + + @@ -212,6 +255,24 @@ Vijil exposes three programmatic interfaces. This quickstart takes you from a fr export AGENT_ID="a1b2c3d4-..." ``` + + Create an [Agent](/owner-guide/register-agents/what-is-an-agent) configuration for the model you want to evaluate: + + ```python + agent = client.agents.create( + name="My Chat Agent", + url="https://api.openai.com/v1/chat/completions", + ) + print(agent.id) + ``` + + Keep the returned `agent` object, you will pass `agent.id` to the Evaluation. To see all registered Agents at any time: + + ```python + for a in client.agents.list().items: + print(a.id, a.name) + ``` + @@ -241,6 +302,16 @@ Vijil exposes three programmatic interfaces. This quickstart takes you from a fr -H "Authorization: Bearer $TOKEN" ``` + + List the available Harnesses: + + ```python + for harness in client.harnesses.list().items: + print(harness.name) + ``` + + For this quickstart you will use the standard trust Harnesses, which `client.evaluate(..., baseline=True)` runs in the next step. To run a specific custom Harness instead, pass its `harness_id`. + @@ -302,6 +373,16 @@ Vijil exposes three programmatic interfaces. This quickstart takes you from a fr The `status` field progresses through `starting` → `pending` → `running` → `completed` → `saving` → `saved`. + + Start a trust Evaluation against the standard Harnesses. `evaluate` polls until the Evaluation completes, then returns the result: + + ```python + evaluation = client.evaluate(agent.id, baseline=True) + print(evaluation.status) # "completed" + ``` + + Polling runs every 5 seconds by default. Adjust it with the `_poll_interval` argument. `baseline=True` covers the reliability, security, and safety dimensions in a single run. + @@ -339,6 +420,23 @@ Vijil exposes three programmatic interfaces. This quickstart takes you from a fr This returns the detailed results JSON including per-Harness breakdowns, individual Probe results, and analysis. + + Read the [Trust Score](/concepts/trust-score/introduction) and per-dimension breakdown straight off the returned Evaluation: + + ```python + print(evaluation.trust_score) # 0.82 + print(evaluation.dimensions.reliability) + print(evaluation.dimensions.security) + print(evaluation.dimensions.safety) + ``` + + To fetch the latest score for an Agent later, without holding the Evaluation object: + + ```python + score = client.scores.show(agent.id) + print(score.trust_score) + ``` + @@ -375,6 +473,17 @@ Vijil exposes three programmatic interfaces. This quickstart takes you from a fr -o report.pdf ``` + + Download the [Trust Report](/developer-guide/evaluate/understanding-results) for the completed Evaluation as a PDF: + + ```python + pdf_bytes = client.reports.download(evaluation.id) + with open("report.pdf", "wb") as f: + f.write(pdf_bytes) + ``` + + The report summarizes what was tested, how the Agent scored, and where it failed. + @@ -388,6 +497,9 @@ Vijil exposes three programmatic interfaces. This quickstart takes you from a fr Complete list of MCP tools with parameters and example prompts + + Python client, lifecycle methods, resources, models, and errors + Interpret Trust Scores and prioritize what to fix @@ -407,3 +519,5 @@ Vijil exposes three programmatic interfaces. This quickstart takes you from a fr | Tools do not appear in Claude Code | Verify `.mcp.json` is in the project root, then restart Claude Code | | Claude uses Bash instead of MCP tools | Confirm `vijil-mcp` is in your `PATH`: run `which vijil-mcp` | | `401 Unauthorized` from the REST API | Your token expired, request a new one from `POST /auth/jwt/login` | +| `VijilAuthError` from the SDK | Set `VIJIL_CLIENT_ID` and `VIJIL_CLIENT_SECRET` (or `VIJIL_API_KEY`) before constructing `Vijil()` | +| SDK import fails | Ensure you are on Python 3.12 or later, then reinstall with `pip install vijil-sdk` | diff --git a/developer-guide/getting-started/introduction.mdx b/developer-guide/getting-started/introduction.mdx index 5fd0af8..f108f3e 100644 --- a/developer-guide/getting-started/introduction.mdx +++ b/developer-guide/getting-started/introduction.mdx @@ -4,7 +4,7 @@ description: 'Integrate evaluation and protection into your development workflow --- -**TL;DR:** Vijil integrates into your development workflow via MCP (natural language), CLI (`vijil-console`), or REST API. Diamond evaluates your [agent](/owner-guide/register-agents/what-is-an-agent) before deployment; Dome protects it at runtime. You get a [Trust Score](/concepts/trust-score/introduction) (0–100) with specific failures you can fix and a deployment gate you can automate in CI/CD. +**TL;DR:** Vijil integrates into your development workflow via MCP (natural language), CLI (`vijil-console`), REST API, or the Python SDK (`vijil-sdk`). Diamond evaluates your [agent](/owner-guide/register-agents/what-is-an-agent) before deployment; Dome protects it at runtime. You get a [Trust Score](/concepts/trust-score/introduction) (0–100) with specific failures you can fix and a deployment gate you can automate in CI/CD. ## The Problem with Testing Agents @@ -27,7 +27,7 @@ Vijil provides two products that work together: ## Integration Methods -Vijil exposes three interfaces. Choose the one that fits your workflow, or mix them: +Vijil exposes four interfaces. Choose the one that fits your workflow, or mix them: ### At a Glance @@ -36,8 +36,9 @@ Vijil exposes three interfaces. Choose the one that fits your workflow, or mix t | **MCP** | Interactive development, natural language evaluation | Claude Code + `vijil-mcp` | | **CLI** | CI/CD gates, scripting, headless automation | `vijil-console` package | | **REST API** | Custom integrations, non-Python environments | HTTP client + API key | +| **SDK** | Python apps, notebooks, programmatic pipelines | `vijil-sdk` package (Python 3.12+) | - + **Natural language, no commands.** Connect Claude Code to your Console and evaluate agents through conversation. Best for interactive development and exploration. @@ -47,6 +48,9 @@ Vijil exposes three interfaces. Choose the one that fits your workflow, or mix t **HTTP requests.** Call the Console API directly from any language. Best for custom integrations, dashboards, and non-Python environments. + + **Native Python.** Install `vijil-sdk` and drive evaluations from your own code. Best for Python apps, notebooks, and programmatic pipelines. + ## Three Developer Workflows @@ -92,9 +96,9 @@ Each evaluation produces a **Trust Score** (0–100) with breakdowns by dimensio ## Time to First Trust Score -The fastest path is [MCP](/developer-guide/agentic/quickstart): install `vijil-mcp`, connect Claude Code, and ask it to run an evaluation, with no commands to learn. If you prefer the terminal, the [CLI path in the Quickstart](/developer-guide/agentic/quickstart) takes you from install to a completed evaluation in minutes. For custom integrations or non-Python environments, the [REST API](/developer-guide/agentic/quickstart) accepts standard HTTP requests and returns results as JSON. +The fastest path is [MCP](/developer-guide/agentic/quickstart): install `vijil-mcp`, connect Claude Code, and ask it to run an evaluation, with no commands to learn. If you prefer the terminal, the [CLI path in the Quickstart](/developer-guide/agentic/quickstart) takes you from install to a completed evaluation in minutes. For custom integrations or non-Python environments, the [REST API](/developer-guide/agentic/quickstart) accepts standard HTTP requests and returns results as JSON. To drive evaluations from Python code, the [SDK](/developer-guide/sdk/setup) wraps the same workflow in a typed client. -All three paths require a running [Vijil Console deployment](/developer-guide/deploy-vijil/deploy-vijil-console) and a registered [Agent](/owner-guide/register-agents/what-is-an-agent) endpoint. +All four paths require a running [Vijil Console deployment](/developer-guide/deploy-vijil/deploy-vijil-console) and a registered [Agent](/owner-guide/register-agents/what-is-an-agent) endpoint. ## Integration Points diff --git a/developer-guide/sdk/lifecycle-methods.mdx b/developer-guide/sdk/lifecycle-methods.mdx new file mode 100644 index 0000000..d74e98e --- /dev/null +++ b/developer-guide/sdk/lifecycle-methods.mdx @@ -0,0 +1,137 @@ +--- +title: "Lifecycle Methods" +description: "High-level Vijil client methods that drive the Agent trust lifecycle: evaluate, test, adapt, and protect." +--- + +Lifecycle methods are high-level workflows on the `Vijil` client. Each maps to a stage of the Agent trust lifecycle and, where an operation runs asynchronously, polls until it completes by default. + +| Method | What It Does | Returns | +|--------|--------------|---------| +| `client.evaluate()` | Measure an Agent against known standards | `Evaluation` | +| `client.test()` | Explore for weaknesses under adversarial pressure | `Job` | +| `client.adapt()` | Improve an Agent through corrective evolution | `Job` | +| `client.protect()` | Configure Dome runtime Guardrails | `DomeConfig` | + + +`evaluate` measures against known standards, like a certification exam. `test` explores for unknown weaknesses through adversarial pressure, like a penetration test. Both produce Trust Scores. + + +## `client.evaluate()` + +Run an Evaluation and poll until it completes. + +```python +evaluation = client.evaluate( + "agent-id", + baseline=True, # use the standard trust Harnesses + # harness_id="h-abc", # OR use a specific Harness (mutually exclusive with baseline) +) + +print(evaluation.status) # "completed" +print(evaluation.trust_score) # 0.82 +print(evaluation.dimensions) # Dimensions(reliability=0.9, security=0.8, safety=0.75) +``` + +| Parameter | Type | Default | Description | +|-----------|------|---------|-------------| +| `agent_id` | `str` | — | Agent ID or alias | +| `baseline` | `bool` | `False` | Run the standard trust Harnesses (reliability, security, safety) | +| `harness_id` | `str \| None` | `None` | Run a specific [Harness](/concepts/evaluation-components/harness) instead of the baseline | +| `_poll_interval` | `float` | `5.0` | Seconds between status checks | + +Returns an [`Evaluation`](/developer-guide/sdk/models-errors). Pass either `baseline=True` or a `harness_id`, not both. + +## `client.test()` + +Create a Red Swarm test engagement and, by default, poll until it finishes. + +```python +job = client.test( + "agent-id", + mode="adaptive", # or "comprehensive" + no_wait=False, # True to return immediately +) + +print(job.status) # "completed" +print(job.id) # "job-abc123" +``` + +| Parameter | Type | Default | Description | +|-----------|------|---------|-------------| +| `agent_id` | `str` | — | Agent ID or alias | +| `mode` | `str` | `"adaptive"` | `"adaptive"` adjusts strategy from responses; `"comprehensive"` sweeps all categories | +| `no_wait` | `bool` | `False` | Return immediately without polling | +| `tool` | `str` | `"diamond_security"` | Red-team tool to use | +| `purpose` | `str` | `""` | Description of the Agent's purpose | +| `categories` | `list[str] \| None` | `None` | Test categories, for example `["injection"]` | + +Returns a [`Job`](/developer-guide/sdk/models-errors). + +## `client.adapt()` + +Create a corrective evolution job that improves an existing Agent based on weaknesses found by evaluation or testing. + +```python +job = client.adapt("agent-id", mode="config") +print(job.status) # "completed" +``` + +| Parameter | Type | Default | Description | +|-----------|------|---------|-------------| +| `agent_id` | `str` | — | Agent ID or alias | +| `mode` | `str` | `"config"` | `"prompt"`, `"config"`, `"code"`, or `"dome"` | +| `no_wait` | `bool` | `False` | Return immediately without polling | + +Returns a [`Job`](/developer-guide/sdk/models-errors). After adaptation completes, review and apply the resulting [proposals](/developer-guide/sdk/resources#client-proposals). + +## `client.protect()` + +Configure [Dome](/developer-guide/protect/overview) runtime Guardrails for an Agent. + +```python +dome = client.protect( + "agent-id", + guards=["prompt_injection", "pii"], + mode="enforce", +) + +print(dome.mode) # "enforce" +print(dome.guards) # ["prompt_injection", "pii"] +``` + +| Parameter | Type | Default | Description | +|-----------|------|---------|-------------| +| `agent_id` | `str` | — | Agent ID or alias | +| `policy_id` | `str \| None` | `None` | Policy to apply | +| `guards` | `list[str] \| None` | `None` | [Guard](/concepts/defense/guard) names | +| `mode` | `str` | `"enforce"` | `"enforce"` blocks threats, `"monitor"` logs only, `"disabled"` turns Guardrails off | + +Returns a [`DomeConfig`](/developer-guide/sdk/models-errors). + +## Preview Methods + + +The following methods target Console routes that are not yet generally available. Their signatures are stable, but calls fail until the corresponding Console capability ships. + + +| Method | What It Does | Returns | +|--------|--------------|---------| +| `client.discover()` | Find Agents in GitHub repositories or cloud infrastructure | `Page[DiscoveredAgent]` | +| `client.register()` | Convert Agent source into an A2A card and genome | `dict` | +| `client.evolve()` | Create a new Agent through generative evolution | `Job` | +| `client.deploy()` | Deploy an Agent to a production runtime | `Deployment` | + +```python +# Discover Agents in a GitHub organization or a Kubernetes namespace +agents = client.discover(github_org="acme-corp") +agents = client.discover(provider="k8s", namespace="production") + +# Register an Agent from a GitHub URL or a local directory +result = client.register("https://github.com/acme/my-agent") + +# Create a new Agent from a spec, a file, or a natural-language description +job = client.evolve(description="A travel booking agent for flights and hotels") + +# Deploy an Agent to a runtime +deployment = client.deploy("agent-id", runtime="agentcore") +``` diff --git a/developer-guide/sdk/models-errors.mdx b/developer-guide/sdk/models-errors.mdx new file mode 100644 index 0000000..51cba32 --- /dev/null +++ b/developer-guide/sdk/models-errors.mdx @@ -0,0 +1,202 @@ +--- +title: "Models and Errors" +description: "Return types the Vijil SDK produces and the exception hierarchy it raises." +--- + +Every SDK call returns a typed model and raises a typed exception on failure. All models inherit from `VijilModel` (a Pydantic `BaseModel` configured with `extra="ignore"`), and all exceptions inherit from `VijilError`. + +## Models + +### `Page[T]` + +Generic paginated response returned by every `list()` method. + +```python +page = client.agents.list() +page.items # list[T] — the results +page.total # int — total count +page.has_more # bool — whether more results exist +``` + +`Page[T]` handles the Console's varying response shapes (`items` or `results` keys, `total` or `count` fields) transparently. + +### `Evaluation` + +```python +evaluation.id # str (UUID) +evaluation.agent_id # str | None +evaluation.status # str — "pending", "running", "completed", "failed" +evaluation.harness_names # list[str] | None +evaluation.trust_score # float | None +evaluation.dimensions # Dimensions | None +``` + +### `Dimensions` + +```python +dimensions.reliability # float | None +dimensions.security # float | None +dimensions.safety # float | None +``` + +### `Job` + +```python +job.id # str (UUID) +job.agent_id # str | None +job.status # str — "pending", "running", "completed", "failed", "cancelled" +job.mode # str | None +job.created_at # str | None +job.error # str | None +``` + +### `Agent` + +```python +agent.id # str (UUID) +agent.name # str +agent.team_id # str | None +agent.url # str | None +agent.framework # str | None +agent.hub # str | None +agent.model_name # str | None +agent.status # str | None +``` + +### `TrustScore` + +```python +score.agent_id # str +score.trust_score # float | None +score.reliability # float | None +score.security # float | None +score.safety # float | None +``` + +### `DomeConfig` + +```python +dome.id # str | None +dome.agent_id # str | None +dome.guards # list[str] | None +dome.mode # str | None +dome.input_guards # list[str] | None +dome.output_guards # list[str] | None +``` + +### `Harness` + +```python +harness.id # str +harness.name # str | None +harness.type # str | None +harness.category # str | None +harness.version # str | None +``` + +### `Report` + +```python +report.id # str +report.agent_id # str | None +report.status # str | None +report.format # str | None +report.created_at # str | None +``` + +### Other Models + +| Model | Key Fields | Produced By | +|-------|-----------|-------------| +| `Genome` | `id`, `agent_id`, `version`, `genes`, `created_at` | `client.genomes.*` | +| `GenomeDiff` | `genome_id`, `v1`, `v2`, `changes` | `client.genomes.diff()` | +| `Proposal` | `id`, `agent_id`, `genome_id`, `status`, `trigger_summary`, `mutations`, `resulting_version` | `client.proposals.*` | +| `Deployment` | `id`, `agent_id`, `runtime`, `status`, `url`, `version` | `client.deploy()` | +| `DiscoveredAgent` | `id`, `name`, `source`, `framework`, `status`, `description` | `client.discover()` | +| `MonitorSummary` | `agent_id`, `total_detections`, `blocked_count`, `passed_count` | `client.monitor.summary()` | +| `Detection` | `id`, `agent_id`, `type`, `direction`, `blocked`, `timestamp` | `client.monitor.detections()` | +| `Trace` | `trace_id`, `agent_id`, `duration_ms`, `spans`, `timestamp` | `client.monitor.traces()` | +| `LogEntry` | `agent_id`, `level`, `message`, `timestamp` | `client.monitor.logs()` | +| `Persona` | `id`, `name`, `description`, `agent_id`, `created_at` | `client.personas.*` | +| `Policy` | `id`, `name`, `content`, `agent_id`, `created_at` | `client.policies.*` | + +## Errors + +Import exceptions from `vijil.exceptions`: + +```python +from vijil.exceptions import ( + VijilError, + VijilAuthError, + VijilNotFoundError, + VijilValidationError, + VijilRateLimitError, + VijilServerError, + VijilJobFailedError, + VijilConfigError, +) +``` + +### Exception Hierarchy + +| Exception | HTTP Code | When | +|-----------|-----------|------| +| `VijilAuthError` | 401, 403 | Invalid or expired API key | +| `VijilNotFoundError` | 404 | Resource does not exist | +| `VijilValidationError` | 400, 422 | Invalid request parameters | +| `VijilRateLimitError` | 429 | Too many requests | +| `VijilServerError` | 500+ | Server error (the SDK retries automatically) | +| `VijilJobFailedError` | — | Asynchronous job completed with an error | +| `VijilConfigError` | — | Missing or invalid configuration | + +### Exception Attributes + +```python +try: + client.agents.show("nonexistent") +except VijilNotFoundError as e: + print(str(e)) # error message + print(e.resource_type) # "agent" (if set) + print(e.resource_id) # "nonexistent" (if set) + +try: + client.agents.list() +except VijilAuthError as e: + print(e.status_code) # 401 or 403 + +except VijilRateLimitError as e: + print(e.retry_after) # seconds to wait (float) + +try: + client.agents.create(name="") +except VijilValidationError as e: + print(e.details) # dict of field-level errors +``` + +### Automatic Retries + +The HTTP client retries transient failures automatically, up to three attempts: + +- **429 Rate Limited** — waits for the `Retry-After` header duration. +- **5xx Server Error** — exponential backoff (1s, 2s, 4s). + +If every retry fails, the SDK raises the corresponding exception. + +### Error-Handling Pattern + +Catch specific exceptions first, then fall back to the `VijilError` base class: + +```python +from vijil import Vijil +from vijil.exceptions import VijilError, VijilNotFoundError + +client = Vijil() + +try: + evaluation = client.evaluate("my-agent", baseline=True) + print(f"Trust score: {evaluation.trust_score}") +except VijilNotFoundError: + print("Agent not found. Check the ID or alias.") +except VijilError as e: + print(f"Vijil error: {e}") +``` diff --git a/developer-guide/sdk/resources.mdx b/developer-guide/sdk/resources.mdx new file mode 100644 index 0000000..1699cc8 --- /dev/null +++ b/developer-guide/sdk/resources.mdx @@ -0,0 +1,160 @@ +--- +title: "Resources" +description: "Low-level resource accessors on the Vijil client for CRUD operations on platform resources." +--- + +Resource accessors are low-level attributes on the `Vijil` client. Where [lifecycle methods](/developer-guide/sdk/lifecycle-methods) run whole workflows, resources give you direct create, read, update, and delete access to individual platform objects. + +List methods return a [`Page[T]`](/developer-guide/sdk/models-errors); iterate `page.items` for the results. + +| Accessor | Methods | +|----------|---------| +| `client.agents` | `list`, `show`, `create`, `update`, `delete` | +| `client.evaluations` | `list`, `show`, `create` | +| `client.harnesses` | `list`, `show`, `create`, `update` | +| `client.scores` | `show`, `history` | +| `client.reports` | `list`, `show`, `download` | +| `client.dome` | `list`, `show`, `default`, `update` | +| `client.jobs` | `list`, `show`, `status`, `create`, `cancel` | +| `client.genomes` | `list`, `show`, `versions`, `history`, `diff` | +| `client.proposals` | `list`, `show`, `approve`, `apply`, `reject` | +| `client.policies` | `list`, `show`, `create`, `update`, `delete` | +| `client.personas` | `list`, `show`, `create`, `update`, `delete` | +| `client.monitor` | `summary`, `detections`, `traces`, `logs` | + +## `client.agents` + +Manage [Agent](/owner-guide/register-agents/what-is-an-agent) configurations. + +```python +page = client.agents.list(team_id="optional-team-id") +agent = client.agents.show("agent-id") +agent = client.agents.create(name="My Agent", url="http://agent:9000/v1") +agent = client.agents.update("agent-id", name="New Name") +client.agents.delete("agent-id") +``` + +## `client.evaluations` + +Manage [Evaluations](/developer-guide/evaluate/overview). Use [`client.evaluate()`](/developer-guide/sdk/lifecycle-methods#client-evaluate) for the polling workflow; `create()` here starts one without waiting. + +```python +page = client.evaluations.list(agent_id="optional-agent-id") +evaluation = client.evaluations.show("evaluation-id") +evaluation = client.evaluations.create(agent_id="agent-id", baseline=True) +``` + +## `client.harnesses` + +List and manage [Harnesses](/concepts/evaluation-components/harness). The list merges standard and custom Harnesses. + +```python +page = client.harnesses.list() +harness = client.harnesses.show("harness-id") +harness = client.harnesses.create(name="Custom Harness") +harness = client.harnesses.update("harness-id", name="Updated") +``` + +## `client.scores` + +Read Trust Scores for an Agent. + +```python +score = client.scores.show("agent-id") +page = client.scores.history("agent-id", limit=20) +``` + +## `client.reports` + +List, fetch, and download Trust Reports. `download()` returns PDF bytes. + +```python +page = client.reports.list(agent_id="optional-agent-id") +report = client.reports.show("evaluation-id") + +pdf_bytes = client.reports.download("evaluation-id") +with open("report.pdf", "wb") as f: + f.write(pdf_bytes) +``` + +## `client.dome` + +Read and update [Dome](/developer-guide/protect/overview) Guardrail configurations. Use [`client.protect()`](/developer-guide/sdk/lifecycle-methods#client-protect) for the common case. + +```python +page = client.dome.list() +config = client.dome.show("agent-id") +config = client.dome.default() +config = client.dome.update("agent-id", guards=["prompt_injection"], mode="enforce") +``` + +## `client.jobs` + +Track and control asynchronous test and evolution jobs. + +```python +page = client.jobs.list("agent-id") +job = client.jobs.show("agent-id", "job-id") +job = client.jobs.status("agent-id", "job-id") +job = client.jobs.create("agent-id", mode="config") +client.jobs.cancel("agent-id", "job-id") +``` + +## `client.genomes` + +Version and compare Agent source across adaptations. + +```python +page = client.genomes.list(agent_id="optional-agent-id") +genome = client.genomes.show("genome-id") +versions = client.genomes.versions("genome-id") +history = client.genomes.history("genome-id") +diff = client.genomes.diff("genome-id", v1=1, v2=3) +``` + +## `client.proposals` + +Review and apply the adaptation proposals produced by [`client.adapt()`](/developer-guide/sdk/lifecycle-methods#client-adapt). + +```python +page = client.proposals.list(agent_id="optional", status="pending") +proposal = client.proposals.show("proposal-id") +proposal = client.proposals.approve("proposal-id") +proposal = client.proposals.apply("proposal-id") +proposal = client.proposals.reject("proposal-id") +``` + +## `client.policies` + +Manage safety and security policies. + +```python +page = client.policies.list(agent_id="optional-agent-id") +policy = client.policies.show("policy-id") +policy = client.policies.create(name="Safety Policy", file_path="policy.yaml") +policy = client.policies.update("policy-id", name="Updated") +client.policies.delete("policy-id") +``` + +## `client.personas` + +Manage test personas used to build custom Harnesses. + +```python +page = client.personas.list(agent_id="optional-agent-id") +persona = client.personas.show("persona-id") +persona = client.personas.create(name="Malicious User", description="Attempts injections") +persona = client.personas.update("persona-id", name="Updated") +client.personas.delete("persona-id") +``` + +## `client.monitor` + +Read Dome runtime telemetry. The `since` argument accepts durations such as `"1h"`, `"24h"`, or `"7d"`. + +```python +summary = client.monitor.summary("agent-id") +detections = client.monitor.detections("agent-id", since="24h") +traces = client.monitor.traces("agent-id", since="1h") +logs = client.monitor.logs("agent-id", since="7d") +``` diff --git a/developer-guide/sdk/setup.mdx b/developer-guide/sdk/setup.mdx new file mode 100644 index 0000000..231afc9 --- /dev/null +++ b/developer-guide/sdk/setup.mdx @@ -0,0 +1,135 @@ +--- +title: "Setup" +description: "Install the Vijil SDK, construct the client, and authenticate against your Vijil Console deployment." +--- + +The `vijil` Python SDK measures and improves Agent trustworthiness from your own code. Use it in scripts, notebooks, and pipelines to run Evaluations, configure protection, and read results programmatically. + + +This reference covers the Python SDK (`from vijil import Vijil`). For interactive, natural-language access from Claude Code, see [MCP](/developer-guide/agentic/quickstart). For terminal commands, see the [CLI Reference](/developer-guide/cli/setup). + + +## Installation + +Install `vijil-sdk` with pip or add it to your project with Poetry: + + +```bash pip +pip install vijil-sdk +``` + +```bash poetry +poetry add vijil-sdk +``` + + +The SDK requires Python 3.12 or later. Verify the import: + +```bash +python -c "from vijil import Vijil; print('ok')" +``` + +## The `Vijil` Client + +`Vijil` is the entry point for every SDK operation. Construct it once and reuse it: + +```python +from vijil import Vijil + +# Read the API key from VIJIL_API_KEY and settings from ~/.vijil/ +client = Vijil() +``` + +| Parameter | Type | Default | Description | +|-----------|------|---------|-------------| +| `gateway` | `str \| None` | `None` | Gateway URL. Falls back to config, then `https://api.vijil.ai` | +| `api_key` | `str \| None` | `None` | API key. Falls back to the `VIJIL_API_KEY` environment variable, then `~/.vijil/credentials.json` | +| `config_dir` | `Path \| None` | `None` | Configuration directory. Defaults to `~/.vijil/` | + +The constructor raises [`VijilAuthError`](/developer-guide/sdk/models-errors) if no API key can be resolved from any source. + +## Authentication + +Create an API key in your [Vijil Console](/developer-guide/deploy-vijil/deploy-vijil-console) deployment under **Settings** > **API Keys** — a client ID (`vk_…`) plus a one-time secret shown only at creation. Export the pair; the SDK exchanges it for a short-lived access token automatically: + +```bash +export VIJIL_CLIENT_ID="vk_..." +export VIJIL_CLIENT_SECRET="..." # shown once at creation +``` + +`Vijil()` reads these from the environment on construction. + +### Bearer Token + +If you already have a bearer access token, provide it directly instead of the client ID and secret: + + +```bash Environment variable +export VIJIL_API_KEY="" +``` + +```python Explicit argument +from vijil import Vijil + +client = Vijil(api_key="") +``` + +```bash Saved credentials +vijil auth login # paste the token; saved to ~/.vijil/credentials.json +``` + + +The `api_key` argument takes precedence over `VIJIL_API_KEY`, which takes precedence over the saved credentials file. + + +In CI/CD, set `VIJIL_CLIENT_ID` and `VIJIL_CLIENT_SECRET` as secrets rather than committing them or running an interactive login. + + +## Gateway Configuration + +By default the SDK connects to `https://api.vijil.ai`. Point it at your own Console deployment (enterprise or VPC) with the `gateway` argument or the `VIJIL_GATEWAY` environment variable: + + +```python Argument +from vijil import Vijil + +client = Vijil(gateway="https://console-api.example.com") +``` + +```bash Environment variable +export VIJIL_GATEWAY="https://console-api.example.com" +``` + + +All SDK calls route through the gateway. Resolution order, highest priority first: + +1. The `gateway` argument +2. The `VIJIL_GATEWAY` environment variable +3. The `gateway.url` value in `~/.vijil/config.toml` +4. The built-in default, `https://api.vijil.ai` + +## Agent Aliases + +The SDK resolves short aliases defined in `~/.vijil/config.toml` anywhere an Agent ID is expected, so you can pass a memorable name instead of a UUID: + +```python +client = Vijil() +client.evaluate("travel-agent", baseline=True) # resolves the alias via config +``` + +## Next Steps + + + + Evaluate, test, adapt, and protect Agents with high-level methods + + + Low-level CRUD accessors for Agents, Evaluations, Harnesses, and more + + + Return types, the exception hierarchy, and error-handling patterns + + + Run your first Evaluation end to end + + diff --git a/docs.json b/docs.json index 01849e3..0f7bafa 100644 --- a/docs.json +++ b/docs.json @@ -234,6 +234,15 @@ "developer-guide/cli/telemetry" ] }, + { + "group": "SDK Reference", + "pages": [ + "developer-guide/sdk/setup", + "developer-guide/sdk/lifecycle-methods", + "developer-guide/sdk/resources", + "developer-guide/sdk/models-errors" + ] + }, { "group": "API Reference", "openapi": "openapi/api.json" From 351205cf7f4eb8cfeac68b5d1dd17304e9fcf0aa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dejan=20Luki=C4=87?= Date: Wed, 8 Jul 2026 13:23:00 +0200 Subject: [PATCH 6/8] fix: Remove naming --- developer-guide/sdk/setup.mdx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/developer-guide/sdk/setup.mdx b/developer-guide/sdk/setup.mdx index 231afc9..4985061 100644 --- a/developer-guide/sdk/setup.mdx +++ b/developer-guide/sdk/setup.mdx @@ -6,7 +6,7 @@ description: "Install the Vijil SDK, construct the client, and authenticate agai The `vijil` Python SDK measures and improves Agent trustworthiness from your own code. Use it in scripts, notebooks, and pipelines to run Evaluations, configure protection, and read results programmatically. -This reference covers the Python SDK (`from vijil import Vijil`). For interactive, natural-language access from Claude Code, see [MCP](/developer-guide/agentic/quickstart). For terminal commands, see the [CLI Reference](/developer-guide/cli/setup). +This reference covers the Python SDK. For interactive, natural-language access from Claude Code, see [MCP](/developer-guide/agentic/quickstart). For terminal commands, see the [CLI Reference](/developer-guide/cli/setup). ## Installation From 211cadfbc56ce2877265babe2fd45736a89b4888 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dejan=20Luki=C4=87?= Date: Thu, 9 Jul 2026 17:07:29 +0200 Subject: [PATCH 7/8] docs: Updates to SDK flows --- developer-guide/agentic/quickstart.mdx | 2 +- developer-guide/evaluate/custom-harnesses.mdx | 153 +++++++++++++++++- .../evaluate/running-evaluations.mdx | 54 ++++++- developer-guide/sdk/models-errors.mdx | 19 ++- developer-guide/sdk/resources.mdx | 30 +++- developer-guide/sdk/setup.mdx | 4 + 6 files changed, 241 insertions(+), 21 deletions(-) diff --git a/developer-guide/agentic/quickstart.mdx b/developer-guide/agentic/quickstart.mdx index 664c964..1c8594e 100644 --- a/developer-guide/agentic/quickstart.mdx +++ b/developer-guide/agentic/quickstart.mdx @@ -190,7 +190,7 @@ Vijil exposes four programmatic interfaces. This quickstart takes you from a fre client = Vijil(gateway="https://console-api.example.com") ``` - `Vijil()` reads the credentials from the environment automatically. If you already have a bearer access token, use it directly instead with `VIJIL_API_KEY` or `Vijil(api_key="")`. + `Vijil()` reads the credentials from the environment automatically. If you already have a bearer access token, use it directly instead with `VIJIL_API_KEY` or `Vijil(api_key="")` — but note that a bearer token is a JWT that expires 24 hours after it is issued. The client ID and secret do not expire, and the SDK refreshes the token from them automatically, so prefer the pair for long-lived automation. In CI/CD, set `VIJIL_CLIENT_ID` and `VIJIL_CLIENT_SECRET` as secrets rather than committing them. diff --git a/developer-guide/evaluate/custom-harnesses.mdx b/developer-guide/evaluate/custom-harnesses.mdx index d31696b..66d0ad7 100644 --- a/developer-guide/evaluate/custom-harnesses.mdx +++ b/developer-guide/evaluate/custom-harnesses.mdx @@ -22,15 +22,108 @@ To view the prompts, [Personas](/owner-guide/simulate-environment/personas) and ## Create a Custom Harness Programmatically +### Create Personas and Policies + +[Personas](/owner-guide/simulate-environment/personas) define *who* interacts with your Agent, and [Policies](/owner-guide/simulate-environment/policies) define the *rules* it must follow. Both are optional inputs to custom Harness creation: pass their IDs (`persona_ids` and `policy_ids`) in the next step, and Vijil generates [Probes](/concepts/evaluation-components/probe) that combine each Persona's behavior with each Policy's constraints. Create them first, then reference their IDs when you create the Harness. + + + + + ```bash + # Create a Persona (or copy a built-in one: vijil persona from-preset ) + vijil persona create \ + --name "Frustrated Customer" \ + --role "End user" \ + --intent adversarial + + # Create a Policy from text, then activate it for use in Harnesses + vijil policy create \ + --name "Data Handling Policy" \ + --category privacy \ + --source-text "The agent must not store or repeat personal data." + + vijil policy activate + ``` + + Each command prints the new `id`. Save the Persona and Policy IDs for the next step. List built-in options with `vijil persona preset-list` and `vijil policy preset-list`. + + + + + With the [Vijil MCP server](/developer-guide/agentic/quickstart) configured, ask Claude Code in natural language: + + + Create an adversarial 'Frustrated Customer' persona and a privacy policy that forbids storing personal data + + + Claude creates the Persona and Policy and returns their IDs. + + + + + ```bash + curl -s -X POST "$VIJIL_URL/v1/personas/?team_id=$TEAM_ID" \ + -H "Authorization: Bearer $TOKEN" \ + -H "Content-Type: application/json" \ + -d '{ + "name": "Frustrated Customer", + "role": "End user", + "intent": "adversarial" + }' + + curl -s -X POST "$VIJIL_URL/v1/policies/?team_id=$TEAM_ID" \ + -H "Authorization: Bearer $TOKEN" \ + -H "Content-Type: application/json" \ + -d '{ + "name": "Data Handling Policy", + "category": "privacy", + "source_text": "The agent must not store or repeat personal data." + }' + ``` + + Each request returns `201 Created` with an `id`. Save both for the next step. + + + + + ```python + persona = client.personas.create( + name="Frustrated Customer", + role="End user", + intent="adversarial", + ) + + policy = client.policies.create( + name="Data Handling Policy", + category="privacy", + source_text="The agent must not store or repeat personal data.", + ) + + print(persona.id, policy.id) + ``` + + `intent` is one of `benign`, `curious`, `adversarial`, or `malicious`. `category` is one of `privacy`, `ethics`, `security`, `compliance`, `operational`, `brand`, or `custom`. + + + + + +Personas and Policies are reusable across Harnesses. For balanced coverage, combine a benign and an adversarial Persona with the Policies that matter most to your Agent. See [Define Personas](/owner-guide/simulate-environment/personas) and [Define Policies](/owner-guide/simulate-environment/policies) for design guidance. + + ### Create a Harness +Pass the Persona and Policy IDs from the previous step as `persona_ids` and `policy_ids`. Both are optional — omit them to generate a Harness from the Agent description alone. + ```bash vijil harness custom-create \ --name "Customer Support Harness" \ - --agent-id "$AGENT_ID" + --agent-id "$AGENT_ID" \ + --persona-ids '[""]' \ + --policy-ids '[""]' ``` | Flag | Description | Required | @@ -96,6 +189,28 @@ To view the prompts, [Personas](/owner-guide/simulate-environment/personas) and | `description` | Human-readable description | No | | `system_prompt` | System prompt used during Harness generation; defaults to the Agent description if omitted | No | + + + + ```python + harness = client.harnesses.create( + name="Customer Support Harness", + agent_id="", + persona_ids=[""], # optional + policy_ids=[""], # optional + system_prompt="You are a helpful customer support assistant.", # optional + ) + print(harness.id) + ``` + + | Parameter | Description | Required | + |-----------|-------------|----------| + | `name` | Harness display name | Yes | + | `agent_id` | UUID of the Agent under test | Yes | + | `persona_ids` | List of Persona UUID values | No | + | `policy_ids` | List of Policy UUID values | No | + | `system_prompt` | System prompt used during generation; defaults to the Agent description | No | + @@ -114,6 +229,11 @@ vijil harness custom-get curl -s "$VIJIL_URL/v1/custom-harnesses/$HARNESS_ID?team_id=$TEAM_ID" \ -H "Authorization: Bearer $TOKEN" ``` + +```python title="SDK" +harness = client.harnesses.show("") +print(harness.status) +``` | `status` value | Meaning | @@ -137,9 +257,17 @@ vijil harness custom-prompts --json curl -s "$VIJIL_URL/v1/custom-harnesses/$HARNESS_ID/prompts?team_id=$TEAM_ID" \ -H "Authorization: Bearer $TOKEN" ``` + +```python title="SDK" +# No dedicated resource method yet — reach the endpoint through the client's HTTP transport +prompts = client._http.get( + f"/v1/custom-harnesses/{harness_id}/prompts", + params={"team_id": team_id}, +) +``` -Returns an empty list if the Harness is still in `draft` status. +Returns an empty list if the Harness is still in `draft` status. The SDK has no dedicated `prompts` method yet, so the example above calls the endpoint through `client._http`, the same transport the high-level methods use. ### List Custom Harnesses @@ -153,9 +281,14 @@ vijil harness custom-list --agent-id "$AGENT_ID" --status active curl -s "$VIJIL_URL/v1/custom-harnesses/?team_id=$TEAM_ID&status=active" \ -H "Authorization: Bearer $TOKEN" ``` + +```python title="SDK" +for harness in client.harnesses.list().items: + print(harness.id, harness.name, harness.status) +``` -Both support filtering by `agent_id` and `status`, with `limit` and `offset` for pagination. +Both support filtering by `agent_id` and `status`, with `limit` and `offset` for pagination. The SDK's `client.harnesses.list()` returns standard and custom Harnesses together. ### Cancel Generation @@ -170,8 +303,18 @@ vijil harness custom-cancel curl -s -X POST "$VIJIL_URL/v1/custom-harnesses/$HARNESS_ID/cancel?team_id=$TEAM_ID" \ -H "Authorization: Bearer $TOKEN" ``` + +```python title="SDK" +# No dedicated resource method yet — reach the endpoint through the client's HTTP transport +client._http.post( + f"/v1/custom-harnesses/{harness_id}/cancel", + params={"team_id": team_id}, +) +``` +The SDK has no dedicated `cancel` method yet, so the example above calls the endpoint through `client._http`, the same transport the high-level methods use. + ### Delete a Harness @@ -184,6 +327,10 @@ vijil harness custom-delete --yes curl -s -X DELETE "$VIJIL_URL/v1/custom-harnesses/$HARNESS_ID?team_id=$TEAM_ID" \ -H "Authorization: Bearer $TOKEN" ``` + +```python title="SDK" +client.harnesses.delete("") +``` ## Next Steps diff --git a/developer-guide/evaluate/running-evaluations.mdx b/developer-guide/evaluate/running-evaluations.mdx index ebb65c2..6206e08 100644 --- a/developer-guide/evaluate/running-evaluations.mdx +++ b/developer-guide/evaluate/running-evaluations.mdx @@ -1,13 +1,13 @@ --- title: 'Run Evaluations' -description: 'Start, monitor, and retrieve trust Evaluations programmatically using the CLI, REST API, or MCP.' +description: 'Start, monitor, and retrieve trust Evaluations programmatically using the CLI, MCP, REST API, or Python SDK.' --- [Vijil Evaluate](https://vijil.ai/evaluate) is a quality assurance framework that automates the testing of LLM applications. An **Evaluation** in Vijil is an automated test run where you select one or more AI agents and a test [Harness](/concepts/evaluation-components/harness) (covering [Security](/concepts/trust-score/security), [Reliability](/concepts/trust-score/reliability), and [Safety](/concepts/trust-score/safety)) to systematically assess the quality, safety, and reliability of LLM applications. ## Prerequisites -- CLI configured and authenticated, or a Bearer token from `POST /auth/jwt/login`, see the [Quickstart](/developer-guide/agentic/quickstart) +- CLI configured and authenticated, a Bearer token from `POST /auth/jwt/login`, or the SDK authenticated with a Vijil API key, see the [Quickstart](/developer-guide/agentic/quickstart) - A registered [Agent](/tutorials/manage-agents) and its UUID ## Start an Evaluation @@ -45,6 +45,32 @@ description: 'Start, monitor, and retrieve trust Evaluations programmatically us Claude calls `eval_run` with `wait=True` and reports back Trust Scores when the evaluation finishes. + + + + `client.evaluate()` starts the Evaluation and polls until it completes, then returns the result: + + ```python + from vijil import Vijil + + client = Vijil() + evaluation = client.evaluate("", baseline=True) + + print(evaluation.trust_score) # 0.82 + print(evaluation.dimensions.reliability) + print(evaluation.dimensions.security) + print(evaluation.dimensions.safety) + ``` + + | Parameter | Description | Required | + |-----------|-------------|----------| + | `agent_id` | Agent ID or alias | Yes | + | `baseline` | Run the standard trust Harnesses (reliability, security, safety) | | + | `harness_id` | Run a specific custom Harness instead of the baseline | | + | `_poll_interval` | Seconds between status checks (default `5.0`) | | + + Pass either `baseline=True` or a `harness_id`. To start without blocking, call `client.evaluations.create(agent_id="", baseline=True)` and poll `client.evaluations.show()` yourself. + @@ -101,6 +127,11 @@ vijil eval status curl -s "$VIJIL_URL/evaluations/$EVAL_ID" \ -H "Authorization: Bearer $TOKEN" ``` + +```python title="SDK" +evaluation = client.evaluations.show("") +print(evaluation.status) +``` When the status reaches `completed`, the response includes per-Harness Trust Scores: @@ -130,6 +161,11 @@ vijil eval results-detail --json | jq '.scores' curl -s "$VIJIL_URL/evaluation-results/$EVAL_ID/results?team_id=$TEAM_ID" \ -H "Authorization: Bearer $TOKEN" ``` + +```python title="SDK" +score = client.scores.show("") +print(score.trust_score, score.reliability, score.security, score.safety) +``` ## Generate a Report @@ -152,6 +188,12 @@ curl -s "$VIJIL_URL/evaluations/$EVAL_ID/pdf?team_id=$TEAM_ID" \ -H "Authorization: Bearer $TOKEN" \ -o report.pdf ``` + +```python title="SDK" +pdf_bytes = client.reports.download("") +with open("report.pdf", "wb") as f: + f.write(pdf_bytes) +``` ## List and Cancel Evaluations @@ -168,9 +210,15 @@ vijil eval cancel ```bash title="Delete" vijil eval delete ``` + +```python title="SDK" +page = client.evaluations.list(agent_id="") +for evaluation in page.items: + print(evaluation.id, evaluation.status) +``` -Use `GET /evaluations/` and `POST /evaluations/{evaluation_id}/cancel` for the REST API equivalents. +Use `GET /evaluations/` and `POST /evaluations/{evaluation_id}/cancel` for the REST API equivalents. The SDK exposes `client.evaluations.list()` and `client.evaluations.show()`; cancel and delete are available through the CLI and REST API. ## Next Steps diff --git a/developer-guide/sdk/models-errors.mdx b/developer-guide/sdk/models-errors.mdx index 51cba32..59caf35 100644 --- a/developer-guide/sdk/models-errors.mdx +++ b/developer-guide/sdk/models-errors.mdx @@ -87,11 +87,16 @@ dome.output_guards # list[str] | None ### `Harness` ```python -harness.id # str -harness.name # str | None -harness.type # str | None -harness.category # str | None -harness.version # str | None +harness.id # str +harness.name # str | None +harness.type # str | None +harness.category # str | None +harness.version # str | None +harness.agent_id # str | None +harness.persona_ids # list[str] | None +harness.policy_ids # list[str] | None +harness.status # str | None — e.g. "draft", "active" +harness.prompt_count # int | None ``` ### `Report` @@ -117,8 +122,8 @@ report.created_at # str | None | `Detection` | `id`, `agent_id`, `type`, `direction`, `blocked`, `timestamp` | `client.monitor.detections()` | | `Trace` | `trace_id`, `agent_id`, `duration_ms`, `spans`, `timestamp` | `client.monitor.traces()` | | `LogEntry` | `agent_id`, `level`, `message`, `timestamp` | `client.monitor.logs()` | -| `Persona` | `id`, `name`, `description`, `agent_id`, `created_at` | `client.personas.*` | -| `Policy` | `id`, `name`, `content`, `agent_id`, `created_at` | `client.policies.*` | +| `Persona` | `id`, `name`, `role`, `intent`, `knowledge_level`, `skill_level` | `client.personas.*` | +| `Policy` | `id`, `name`, `category`, `status`, `source_text`, `tags` | `client.policies.*` | ## Errors diff --git a/developer-guide/sdk/resources.mdx b/developer-guide/sdk/resources.mdx index 1699cc8..f353d2f 100644 --- a/developer-guide/sdk/resources.mdx +++ b/developer-guide/sdk/resources.mdx @@ -11,7 +11,7 @@ List methods return a [`Page[T]`](/developer-guide/sdk/models-errors); iterate ` |----------|---------| | `client.agents` | `list`, `show`, `create`, `update`, `delete` | | `client.evaluations` | `list`, `show`, `create` | -| `client.harnesses` | `list`, `show`, `create`, `update` | +| `client.harnesses` | `list`, `show`, `create`, `delete` | | `client.scores` | `show`, `history` | | `client.reports` | `list`, `show`, `download` | | `client.dome` | `list`, `show`, `default`, `update` | @@ -51,10 +51,18 @@ List and manage [Harnesses](/concepts/evaluation-components/harness). The list m ```python page = client.harnesses.list() harness = client.harnesses.show("harness-id") -harness = client.harnesses.create(name="Custom Harness") -harness = client.harnesses.update("harness-id", name="Updated") +harness = client.harnesses.create( + name="Custom Harness", + agent_id="agent-id", + persona_ids=["persona-id"], # optional + policy_ids=["policy-id"], # optional + system_prompt="...", # optional +) +client.harnesses.delete("harness-id") ``` +`create()` generates a [custom Harness](/developer-guide/evaluate/custom-harnesses) for the given Agent; `persona_ids` and `policy_ids` shape the Probes it produces. Custom Harnesses are immutable, so there is no `update()` — create a new Harness and delete the old one to change its configuration. + ## `client.scores` Read Trust Scores for an Agent. @@ -126,24 +134,32 @@ proposal = client.proposals.reject("proposal-id") ## `client.policies` -Manage safety and security policies. +Manage the [Policies](/owner-guide/simulate-environment/policies) that constrain Agent behavior. `category` is required — one of `privacy`, `ethics`, `security`, `compliance`, `operational`, `brand`, or `custom`. Provide the policy text with `source_text`, `content`, or a `file_path` to a `.txt`/`.pdf` document. ```python page = client.policies.list(agent_id="optional-agent-id") policy = client.policies.show("policy-id") -policy = client.policies.create(name="Safety Policy", file_path="policy.yaml") +policy = client.policies.create( + name="Data Handling Policy", + category="privacy", + source_text="The agent must not store or repeat personal data.", +) policy = client.policies.update("policy-id", name="Updated") client.policies.delete("policy-id") ``` ## `client.personas` -Manage test personas used to build custom Harnesses. +Manage the [Personas](/owner-guide/simulate-environment/personas) used to build custom Harnesses. `role` is required; `intent` is one of `benign`, `curious`, `adversarial`, or `malicious`. ```python page = client.personas.list(agent_id="optional-agent-id") persona = client.personas.show("persona-id") -persona = client.personas.create(name="Malicious User", description="Attempts injections") +persona = client.personas.create( + name="Frustrated Customer", + role="End user", + intent="adversarial", +) persona = client.personas.update("persona-id", name="Updated") client.personas.delete("persona-id") ``` diff --git a/developer-guide/sdk/setup.mdx b/developer-guide/sdk/setup.mdx index 4985061..3d968b1 100644 --- a/developer-guide/sdk/setup.mdx +++ b/developer-guide/sdk/setup.mdx @@ -81,6 +81,10 @@ vijil auth login # paste the token; saved to ~/.vijil/credentials.json The `api_key` argument takes precedence over `VIJIL_API_KEY`, which takes precedence over the saved credentials file. + +A bearer access token is a JWT that expires 24 hours after it is issued. For long-lived automation, authenticate with the client ID and secret instead — the SDK exchanges them for a fresh token automatically whenever the current one expires. + + In CI/CD, set `VIJIL_CLIENT_ID` and `VIJIL_CLIENT_SECRET` as secrets rather than committing them or running an interactive login. From 8b8108f8c7416802ec2dcebccd9cdadfdc830514 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dejan=20Luki=C4=87?= Date: Thu, 9 Jul 2026 17:19:25 +0200 Subject: [PATCH 8/8] docs: Diagram styling changes --- styles.css | 115 +++++++++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 108 insertions(+), 7 deletions(-) diff --git a/styles.css b/styles.css index 92f91b1..c2edf87 100644 --- a/styles.css +++ b/styles.css @@ -4,20 +4,121 @@ /* Target ul elements that are direct children or descendants of Card components */ [class*="card"] ul, [class*="Card"] ul { - list-style-type: disc !important; - list-style-position: outside !important; - padding-left: 20px; - margin-top: 12px; + list-style-type: disc !important; + list-style-position: outside !important; + padding-left: 20px; + margin-top: 12px; } [class*="card"] ul li, [class*="Card"] ul li { - margin-left: 0; - padding-left: 0; + margin-left: 0; + padding-left: 0; } /* Opt-out: some Card lists are used as structured text (no bullets). */ [class*="card"] ul.card-no-bullets, [class*="Card"] ul.card-no-bullets { - list-style-type: none !important; + list-style-type: none !important; +} + +/* ============================================================ + Mermaid diagrams + ============================================================ */ + +/* Render diagrams larger and centered in both themes. The SVGs ship with a + small intrinsic max-width; override it so they scale up to the content width. */ +.mermaid, +[class*="mermaid"] { + text-align: center; +} + +.mermaid svg, +svg[id^="mermaid-"], +svg[id^="flowchart-"] { + width: 100% !important; + max-width: 46rem !important; + height: auto !important; + margin-inline: auto; +} + +/* Dark mode: the diagrams hard-code a light-theme palette, so recolor the + structural strokes (arrows, node borders, subgraph borders) and structural + text to white for contrast on dark backgrounds. Node fills and node label + colors are intentionally left as authored. */ +html.dark .mermaid svg .flowchart-link, +html.dark .mermaid svg .edgePath > path, +html.dark svg[id^="mermaid-"] .flowchart-link, +html.dark svg[id^="mermaid-"] .edgePath > path, +html.dark svg[id^="flowchart-"] .flowchart-link, +html.dark svg[id^="flowchart-"] .edgePath > path, +html[data-theme="dark"] .mermaid svg .flowchart-link, +html[data-theme="dark"] svg[id^="mermaid-"] .flowchart-link, +html[data-theme="dark"] svg[id^="mermaid-"] .edgePath > path { + stroke: #ffffff !important; +} + +/* Arrowheads */ +html.dark .mermaid svg marker path, +html.dark .mermaid svg .arrowMarkerPath, +html.dark svg[id^="mermaid-"] marker path, +html.dark svg[id^="mermaid-"] .arrowMarkerPath, +html.dark svg[id^="flowchart-"] marker path, +html[data-theme="dark"] svg[id^="mermaid-"] marker path, +html[data-theme="dark"] svg[id^="mermaid-"] .arrowMarkerPath { + fill: #ffffff !important; + stroke: #ffffff !important; +} + +/* Node borders */ +html.dark .mermaid svg .node rect, +html.dark .mermaid svg .node circle, +html.dark .mermaid svg .node ellipse, +html.dark .mermaid svg .node polygon, +html.dark .mermaid svg .node path, +html.dark svg[id^="mermaid-"] .node rect, +html.dark svg[id^="mermaid-"] .node circle, +html.dark svg[id^="mermaid-"] .node ellipse, +html.dark svg[id^="mermaid-"] .node polygon, +html.dark svg[id^="mermaid-"] .node path, +html[data-theme="dark"] svg[id^="mermaid-"] .node rect, +html[data-theme="dark"] svg[id^="mermaid-"] .node circle, +html[data-theme="dark"] svg[id^="mermaid-"] .node ellipse, +html[data-theme="dark"] svg[id^="mermaid-"] .node polygon, +html[data-theme="dark"] svg[id^="mermaid-"] .node path { + stroke: #ffffff !important; +} + +/* Subgraph / cluster borders and titles */ +html.dark .mermaid svg .cluster rect, +html.dark svg[id^="mermaid-"] .cluster rect, +html[data-theme="dark"] svg[id^="mermaid-"] .cluster rect { + stroke: #ffffff !important; + fill: transparent !important; +} + +html.dark .mermaid svg .cluster text, +html.dark .mermaid svg .cluster .nodeLabel, +html.dark .mermaid svg .cluster span, +html.dark svg[id^="mermaid-"] .cluster text, +html.dark svg[id^="mermaid-"] .cluster .nodeLabel, +html.dark svg[id^="mermaid-"] .cluster span, +html[data-theme="dark"] svg[id^="mermaid-"] .cluster text, +html[data-theme="dark"] svg[id^="mermaid-"] .cluster .nodeLabel { + fill: #ffffff !important; + color: #ffffff !important; +} + +/* Edge labels, if a diagram uses them */ +html.dark .mermaid svg .edgeLabel, +html.dark .mermaid svg .edgeLabel span, +html.dark .mermaid svg .edgeLabel p, +html.dark svg[id^="mermaid-"] .edgeLabel, +html.dark svg[id^="mermaid-"] .edgeLabel span, +html.dark svg[id^="mermaid-"] .edgeLabel p, +html[data-theme="dark"] svg[id^="mermaid-"] .edgeLabel, +html[data-theme="dark"] svg[id^="mermaid-"] .edgeLabel span { + color: #ffffff !important; + fill: #ffffff !important; + background: transparent !important; }