diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c2ca35eb6..99c2cc45e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -256,9 +256,15 @@ jobs: - name: Run Python e2e suites 1-13 working-directory: sdk/python + # These suites drive a real server + real LLM, so individual tests + # flake on transient latency/tool-call stalls (workflow still RUNNING + # at timeout, tool batch not returning, etc.). Auto-retry transient + # failures up to twice — a genuinely broken test still fails all 3 + # attempts, while a one-off flake recovers. See #277 thread. run: | uv run pytest e2e/ -v --tb=short \ --junitxml=../../e2e-results/junit.xml \ + --reruns 2 --reruns-delay 5 \ -n 3 --dist=loadgroup - name: Generate Python HTML report diff --git a/.github/workflows/deploy-docs.yml b/.github/workflows/deploy-docs.yml new file mode 100644 index 000000000..98c4105a5 --- /dev/null +++ b/.github/workflows/deploy-docs.yml @@ -0,0 +1,25 @@ +name: Publish docs via GitHub Pages + +permissions: + contents: write + +on: + push: + branches: + - main + workflow_dispatch: + +jobs: + build: + name: Deploy docs + runs-on: ubuntu-latest + steps: + - name: Checkout main + uses: actions/checkout@v7 + + - name: Deploy docs + uses: mhausenblas/mkdocs-deploy-gh-pages@master + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + CONFIG_FILE: mkdocs.yml + REQUIREMENTS: requirements.txt diff --git a/.github/workflows/release-csharp-sdk.yml b/.github/workflows/release-csharp-sdk.yml index 3ef5f6c0a..f0d186463 100644 --- a/.github/workflows/release-csharp-sdk.yml +++ b/.github/workflows/release-csharp-sdk.yml @@ -35,10 +35,16 @@ jobs: - name: Pack run: | - dotnet pack src/Agentspan/Agentspan.csproj \ - --configuration Release \ - -p:Version=${{ steps.version.outputs.version }} \ - --output ./nupkg + for proj in \ + src/Conductor.AI/Conductor.AI.csproj \ + src/Conductor.AI.OpenAI/Conductor.AI.OpenAI.csproj \ + src/Conductor.AI.SemanticKernel/Conductor.AI.SemanticKernel.csproj \ + src/Conductor.AI.GoogleADK/Conductor.AI.GoogleADK.csproj; do + dotnet pack "$proj" \ + --configuration Release \ + -p:Version=${{ steps.version.outputs.version }} \ + --output ./nupkg + done - name: Publish to NuGet run: | diff --git a/.github/workflows/release-server-maven.yml b/.github/workflows/release-server-maven.yml index bbcd488d2..6f68f636d 100644 --- a/.github/workflows/release-server-maven.yml +++ b/.github/workflows/release-server-maven.yml @@ -1,6 +1,8 @@ name: Publish Server to Maven Central on: + release: + types: [created] workflow_dispatch: inputs: version: @@ -31,12 +33,17 @@ jobs: - name: Determine version id: version run: | - VERSION="${{ inputs.version }}" + if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then + VERSION="${{ inputs.version }}" + else + TAG="${{ github.event.release.tag_name }}" + VERSION="${TAG#v}" + fi echo "version=${VERSION}" >> "$GITHUB_OUTPUT" echo "Publishing version: ${VERSION}" # Publishes both modules (conductor-agentspan, conductor-agentspan-server) - # under org.conductoross.conductor. The runnable fat jar is released + # under org.conductoross. The runnable fat jar is released # separately via the S3/GitHub workflow, not to Maven Central. - name: Publish to Maven Central run: | @@ -48,3 +55,49 @@ jobs: ORG_GRADLE_PROJECT_signingInMemoryKeyId: ${{ secrets.SIGNING_KEY_ID }} ORG_GRADLE_PROJECT_signingInMemoryKey: ${{ secrets.SIGNING_KEY }} ORG_GRADLE_PROJECT_signingInMemoryKeyPassword: ${{ secrets.SIGNING_PASSWORD }} + + publish-github-packages: + runs-on: ubuntu-latest + # Least privilege: GitHub Packages only needs to write packages with the + # built-in GITHUB_TOKEN. No Sonatype/signing secrets or protected + # environment required, so this is a separate job from Maven Central. + permissions: + contents: read + packages: write + defaults: + run: + working-directory: server + + steps: + - uses: actions/checkout@v4 + + - name: Set up Zulu JDK 21 + uses: actions/setup-java@v5 + with: + distribution: "zulu" + java-version: "21" + + - name: Determine version + id: version + run: | + if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then + VERSION="${{ inputs.version }}" + else + TAG="${{ github.event.release.tag_name }}" + VERSION="${TAG#v}" + fi + echo "version=${VERSION}" >> "$GITHUB_OUTPUT" + echo "Publishing version: ${VERSION}" + + # Publishes both modules to GitHub Packages + # (https://maven.pkg.github.com/${{ github.repository }}). Unsigned — + # GitHub Packages does not require GPG signatures, so no signing secrets + # are passed here (the vanniktech block only signs when a signing key is + # present). The runnable fat jar is released via the S3/GitHub workflow. + - name: Publish to GitHub Packages + run: | + ./gradlew publishAllPublicationsToGitHubPackagesRepository --no-configuration-cache \ + -Pversion=${{ steps.version.outputs.version }} + env: + GITHUB_ACTOR: ${{ github.actor }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/release-server-s3.yml b/.github/workflows/release-server-s3.yml index e4c7a93d9..1bb056092 100644 --- a/.github/workflows/release-server-s3.yml +++ b/.github/workflows/release-server-s3.yml @@ -43,7 +43,7 @@ jobs: run: ./gradlew bootJar -PbuildUI=true - name: Prepare release asset - run: cp build/libs/agentspan-runtime.jar ../agentspan-server-${{ inputs.version }}.jar + run: cp conductor-agentspan-server/build/libs/agentspan-runtime.jar ../agentspan-server-${{ inputs.version }}.jar - name: Upload to S3 (versioned and latest) env: diff --git a/.gitignore b/.gitignore index 03dcb169a..2b5ea6d8f 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,6 @@ +# OCG smoke test local venv +/e2e/ocg/venv/ + .DS_Store .idea/ __pycache__/ @@ -51,3 +54,7 @@ obj/ *.user *.suo .vs/ + +# Editor / tooling local config +.vscode/ +.claude/ diff --git a/AGENTS.md b/AGENTS.md index b5198adea..f386b9657 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -4,11 +4,11 @@ This file provides context for AI coding agents (Claude Code, Copilot, Cursor, e ## Project Overview -The `agentspan` Python SDK compiles Python `Agent` definitions into durable [Conductor](https://github.com/conductor-oss/conductor) executions. Agents survive process crashes, tools scale as distributed workers, and human-in-the-loop approvals can pause for days. +The Agentspan Python SDK compiles Python `Agent` definitions into durable [Conductor](https://github.com/conductor-oss/conductor) executions. Agents survive process crashes, tools scale as distributed workers, and human-in-the-loop approvals can pause for days. -**Package name (PyPI):** `agentspan` -**npm package:** `@agentspan-ai/agentspan` -**Import path:** `from agentspan.agents import ...` +**Package name (PyPI):** `conductor-agent-sdk` +**npm package:** `@conductor-oss/conductor-agent-sdk` +**Import path:** `from conductor.ai.agents import ...` **Python:** 3.10+ **License:** MIT @@ -38,28 +38,28 @@ When `run(agent, prompt)` is called: | File | Purpose | |---|---| -| `src/agentspan/agents/agent.py` | `Agent` class — the single orchestration primitive | -| `src/agentspan/agents/tool.py` | `@tool` decorator, `ToolDef`, `http_tool()`, `mcp_tool()` | -| `src/agentspan/agents/run.py` | Top-level `run()`, `start()`, `stream()`, `run_async()`, `plan()` with singleton runtime | -| `src/agentspan/agents/result.py` | `AgentResult`, `AgentHandle`, `AgentStatus`, `AgentEvent`, `EventType` | -| `src/agentspan/agents/guardrail.py` | `Guardrail`, `GuardrailResult`, `RegexGuardrail`, `LLMGuardrail` | -| `src/agentspan/agents/memory.py` | `ConversationMemory` — session message history | -| `src/agentspan/agents/semantic_memory.py` | `SemanticMemory`, `MemoryStore`, `MemoryEntry` — long-term memory | -| `src/agentspan/agents/termination.py` | `TerminationCondition` and composable subclasses (`&`, `|` operators) | -| `src/agentspan/agents/handoff.py` | `HandoffCondition`, `OnToolResult`, `OnTextMention`, `OnCondition` | -| `src/agentspan/agents/code_executor.py` | `CodeExecutor` — Local, Docker, Jupyter, Serverless | -| `src/agentspan/agents/ext.py` | `GPTAssistantAgent` | -| `src/agentspan/agents/tracing.py` | Optional OpenTelemetry integration | -| `src/agentspan/agents/__init__.py` | Public API surface — all exports | -| `src/agentspan/agents/compiler/agent_compiler.py` | Single agent compilation (DoWhile loops, tool dispatch) | -| `src/agentspan/agents/compiler/multi_agent_compiler.py` | Multi-agent strategies (handoff, sequential, parallel, router) | -| `src/agentspan/agents/compiler/tool_compiler.py` | `@tool` → TaskDef + ToolSpec + dispatch registration | -| `src/agentspan/agents/compiler/_dispatch.py` | Universal dispatch worker (fuzzy parsing, circuit breaker) | -| `src/agentspan/agents/runtime/runtime.py` | `AgentRuntime` — compile + execute + stream | -| `src/agentspan/agents/runtime/worker_manager.py` | Auto-register `@tool` as Conductor workers | -| `src/agentspan/agents/runtime/config.py` | `AgentConfig` — environment variable configuration | -| `src/agentspan/agents/_internal/model_parser.py` | Parse `"provider/model"` strings | -| `src/agentspan/agents/_internal/schema_utils.py` | JSON Schema generation from type hints | +| `src/conductor/ai/agents/agent.py` | `Agent` class — the single orchestration primitive | +| `src/conductor/ai/agents/tool.py` | `@tool` decorator, `ToolDef`, `http_tool()`, `mcp_tool()` | +| `src/conductor/ai/agents/run.py` | Top-level `run()`, `start()`, `stream()`, `run_async()`, `plan()` with singleton runtime | +| `src/conductor/ai/agents/result.py` | `AgentResult`, `AgentHandle`, `AgentStatus`, `AgentEvent`, `EventType` | +| `src/conductor/ai/agents/guardrail.py` | `Guardrail`, `GuardrailResult`, `RegexGuardrail`, `LLMGuardrail` | +| `src/conductor/ai/agents/memory.py` | `ConversationMemory` — session message history | +| `src/conductor/ai/agents/semantic_memory.py` | `SemanticMemory`, `MemoryStore`, `MemoryEntry` — long-term memory | +| `src/conductor/ai/agents/termination.py` | `TerminationCondition` and composable subclasses (`&`, `|` operators) | +| `src/conductor/ai/agents/handoff.py` | `HandoffCondition`, `OnToolResult`, `OnTextMention`, `OnCondition` | +| `src/conductor/ai/agents/code_executor.py` | `CodeExecutor` — Local, Docker, Jupyter, Serverless | +| `src/conductor/ai/agents/ext.py` | `GPTAssistantAgent` | +| `src/conductor/ai/agents/tracing.py` | Optional OpenTelemetry integration | +| `src/conductor/ai/agents/__init__.py` | Public API surface — all exports | +| `src/conductor/ai/agents/compiler/agent_compiler.py` | Single agent compilation (DoWhile loops, tool dispatch) | +| `src/conductor/ai/agents/compiler/multi_agent_compiler.py` | Multi-agent strategies (handoff, sequential, parallel, router) | +| `src/conductor/ai/agents/compiler/tool_compiler.py` | `@tool` → TaskDef + ToolSpec + dispatch registration | +| `src/conductor/ai/agents/compiler/_dispatch.py` | Universal dispatch worker (fuzzy parsing, circuit breaker) | +| `src/conductor/ai/agents/runtime/runtime.py` | `AgentRuntime` — compile + execute + stream | +| `src/conductor/ai/agents/runtime/worker_manager.py` | Auto-register `@tool` as Conductor workers | +| `src/conductor/ai/agents/runtime/config.py` | `AgentConfig` — environment variable configuration | +| `src/conductor/ai/agents/_internal/model_parser.py` | Parse `"provider/model"` strings | +| `src/conductor/ai/agents/_internal/schema_utils.py` | JSON Schema generation from type hints | ### Conductor Primitive Mapping @@ -88,14 +88,14 @@ When `run(agent, prompt)` is called: ### Module-Level Patterns -- Every module uses `logging.getLogger("agentspan.agents.xxx")` for structured logging +- Every module uses `logging.getLogger("conductor.ai.agents.xxx")` for structured logging - The dispatch worker (`_dispatch.py`) deliberately does NOT use `from __future__ import annotations` because Conductor's worker framework needs real type objects for parameter resolution - The dispatch worker uses `object` type annotations (not `dict`/`list`) to avoid Conductor's `convert_from_dict_or_list()` issues - Tool functions, error counts, and approval flags are stored in module-level registries (`_tool_registry`, `_tool_error_counts`, `_tool_approval_flags`) ### Public API -All public exports are listed in `src/agentspan/agents/__init__.py` and its `__all__` list. When adding a new public class or function, add it to both the imports and `__all__`. +All public exports are listed in `src/conductor/ai/agents/__init__.py` and its `__all__` list. When adding a new public class or function, add it to both the imports and `__all__`. ### Agent Strategies @@ -127,7 +127,7 @@ python3 -m pytest tests/integration/ -v ruff check src/ # Type check -mypy src/agentspan/agents/ --ignore-missing-imports --no-strict-optional +mypy src/conductor/ai/agents/ --ignore-missing-imports --no-strict-optional ``` ### Test Files @@ -157,14 +157,14 @@ This is not negotiable and not subject to per-session interpretation: When a test reveals non-determinism that the test itself caused (timing-sensitive assertions, ordering assumptions), fix the **test** so it's robust. When the non-determinism is in the system under test (real race, real instability), fix the **system**. Don't add retries to mask either case. -**The narrow exception — upstream LLM provider variability.** Some e2e tests validate a non-LLM property (a strategy compiles, a sub-workflow fires, a worker registers) but depend on the LLM to drive the scenario (call a tool, pick a route). When gpt-4o-mini occasionally skips a tool call or paraphrases away a number, that's external provider variability — not Agentspan's bug and not the test's bug. For these cases: +**The narrow exception — upstream LLM provider variability.** Some e2e tests validate a non-LLM property (a strategy compiles, a sub-workflow fires, a worker registers) but depend on the LLM to drive the scenario (call a tool, pick a route). When gpt-4o-mini occasionally skips a tool call or paraphrases away a number, that's external provider variability — not Agentspan' bug and not the test's bug. For these cases: - Strongly prefer asserting on deterministic server-side state (workflow status, task names, `outputData` shapes from `@tool` stubs that return fixed data). - When that's not enough, `{ retry: 2 }` is acceptable, but only with a comment explaining *which* property is the real subject of the test and *why* LLM variability is incidental. See the pattern in `test_suite20_plan_execute.test.ts`. - Never use retries to paper over a real race in the system or a brittle assertion in the test. The retry is a coping mechanism for upstream variability, not for our own bugs. ### Writing Tests -- Unit tests must run without an Agentspan server (mock all external calls) +- Unit tests must run without a Agentspan server (mock all external calls) - Place unit tests in `tests/unit/`, integration tests in `tests/integration/` - Follow existing naming: `test_{module}.py` - Use `pytest` fixtures and parametrize where appropriate @@ -190,7 +190,7 @@ Before merging any change: 1. **Unit tests pass:** `python3 -m pytest tests/unit/ -v` 2. **Lint clean:** `ruff check src/` -3. **Type check clean:** `mypy src/agentspan/agents/ --ignore-missing-imports --no-strict-optional` +3. **Type check clean:** `mypy src/conductor/ai/agents/ --ignore-missing-imports --no-strict-optional` 4. **Public API unchanged** (or intentionally extended): check `__init__.py` `__all__` 5. **Examples still work** for affected features (run against a live Agentspan server) 6. **Docs updated when needed:** `mkdocs build --strict` @@ -269,7 +269,7 @@ cd server && ./gradlew build ## CLI (Go) -The `cli/` directory contains the AgentSpan CLI — a Go binary built with Cobra that manages the server and agents. +The `cli/` directory contains the Agentspan CLI — a Go binary built with Cobra that manages the server and agents. ### CLI Key Source Files @@ -388,7 +388,7 @@ Environment variables: | Variable | Description | Default | |---|---|---| -| `AGENTSPAN_SERVER_URL` | AgentSpan server API URL | `http://localhost:6767/api` | +| `AGENTSPAN_SERVER_URL` | Agentspan server API URL | `http://localhost:6767/api` | | `AGENTSPAN_AUTH_KEY` | Auth key (Orkes Cloud) | None | | `AGENTSPAN_AUTH_SECRET` | Auth secret (Orkes Cloud) | None | | `AGENTSPAN_AGENT_TIMEOUT` | Default execution timeout (seconds) | 300 | diff --git a/README.md b/README.md index 9133d2892..501f7f6c6 100644 --- a/README.md +++ b/README.md @@ -1,16 +1,16 @@

- - - Agentspan + + + Agentspan

AI agents that don't die when your process does.

- PyPI - Downloads + PyPI + Downloads Stars License Discord @@ -26,7 +26,7 @@

- +

--- @@ -38,7 +38,15 @@ https://github.com/user-attachments/assets/dd4b720d-d11c-42e8-93a6-875c5a740fd8 -**Agentspan** is a distributed, durable runtime for AI agents that survive crashes, scale across machines, and pause for human approval for days — not minutes. +**Agentspan** is a durable runtime for AI agents, built for Conductor. Three pillars: + +**Long-running agents** — Write an agent; it runs as long as it needs to. Minutes, hours, or until a human approves the next step. No timeout by default. If your worker process crashes, the server resumes from the last completed step when a new worker connects. + +**Dynamic agents (Plan-Execute)** — The LLM decides what to do at runtime; Conductor locks it in and executes it deterministically. The planner emits a JSON plan once; the server compiles it into an immutable Conductor sub-workflow — no LLM randomness in orchestration, retries, or parallelism. Dynamic agents can call existing Conductor workflows as steps, bridging AI with your existing automation. +→ `Strategy.PLAN_EXECUTE` · works across Python, TypeScript, Java, C# + +**Event-driven agents** — Trigger agents from cron schedules, Kafka topics, SQS queues, AMQP messages, webhooks, and database events. Agentspan runs on Conductor, so every event source Conductor supports is available to agents. Each trigger is a durable execution with full history. +→ `deploy(agent, schedules=[Schedule(cron="0 0 9 * * MON-FRI")])` · Conductor event handlers ## Quickstart (60 seconds) @@ -51,15 +59,18 @@ irm https://raw.githubusercontent.com/agentspan-ai/agentspan/main/cli/install.ps ``` ## Install SDKs + +Build on Agentspan with the Conductor agent SDK, available for Python, TypeScript/JavaScript, and C#/.NET: + ```bash # Python -pip install agentspan +pip install conductor-agent-sdk # TypeScript / JavaScript -npm install @agentspan-ai/sdk +npm install @conductor-oss/conductor-agent-sdk # C# / .NET -dotnet add package Agentspan +dotnet add package conductor-agent-sdk ``` ```bash @@ -69,7 +80,7 @@ agentspan server start # runs on localhost:6767 with UI ```python # hello.py — run with: python hello.py -from agentspan.agents import Agent, AgentRuntime, tool +from conductor.ai.agents import Agent, AgentRuntime, tool @tool def get_weather(city: str) -> str: @@ -129,7 +140,7 @@ agentspan doctor ## Why Agentspan? -Agentspan is the execution layer, not the replacement. Use native Agentspan agents, or bring LangGraph, the OpenAI Agents SDK, or Google ADK — pass your existing agent to `runtime.run()` and it gains crash recovery, human-in-the-loop pauses, and full execution history. Your definitions stay unchanged. +Agentspan is the execution layer, not the replacement. Use native Agentspan, or bring LangGraph, the OpenAI Agents SDK, or Google ADK — pass your existing agent to `runtime.run()` and it gains crash recovery, human-in-the-loop pauses, and full execution history. Your definitions stay unchanged. | | CrewAI | LangChain | AutoGen | OpenAI Agents | **Agentspan** | |---|---|---|---|---|---| @@ -170,7 +181,7 @@ Agentspan is the execution layer, not the replacement. Use native Agentspan agen ### Agent with Tools ```python -from agentspan.agents import Agent, AgentRuntime, tool +from conductor.ai.agents import Agent, AgentRuntime, tool @tool def get_weather(city: str) -> dict: @@ -198,7 +209,7 @@ with AgentRuntime() as runtime: ```python from pydantic import BaseModel -from agentspan.agents import Agent, AgentRuntime, tool +from conductor.ai.agents import Agent, AgentRuntime, tool class WeatherReport(BaseModel): city: str @@ -234,7 +245,7 @@ Credentials are encrypted at rest (AES-256-GCM). List them with `agentspan crede **Step 2: Declare which credentials a tool needs** ```python -from agentspan.agents import Agent, AgentRuntime, tool, get_credential +from conductor.ai.agents import Agent, AgentRuntime, tool, get_credential # Default: tool runs in isolated subprocess with credentials as env vars @tool(credentials=["GITHUB_TOKEN"]) @@ -270,7 +281,7 @@ with AgentRuntime() as runtime: **Credentials work with every tool type:** ```python -from agentspan.agents import http_tool, mcp_tool +from conductor.ai.agents import http_tool, mcp_tool # HTTP tools: server substitutes ${NAME} in headers at runtime api = http_tool( @@ -289,7 +300,7 @@ No credentials leave the server unencrypted. Workers resolve them via scoped exe ### Multi-Agent Handoffs ```python -from agentspan.agents import Agent, AgentRuntime, tool +from conductor.ai.agents import Agent, AgentRuntime, tool @tool def check_balance(account_id: str) -> dict: @@ -316,7 +327,7 @@ with AgentRuntime() as runtime: ### Pipeline Composition ```python -from agentspan.agents import Agent, AgentRuntime +from conductor.ai.agents import Agent, AgentRuntime researcher = Agent(name="researcher", model="openai/gpt-4o", instructions="Research the topic and provide key facts.") @@ -335,7 +346,7 @@ with AgentRuntime() as runtime: ### Parallel Agents ```python -from agentspan.agents import Agent, AgentRuntime +from conductor.ai.agents import Agent, AgentRuntime market = Agent(name="market", model="openai/gpt-4o", instructions="Analyze market size, growth, key players.") @@ -353,7 +364,7 @@ with AgentRuntime() as runtime: ### Human-in-the-Loop (Durable) ```python -from agentspan.agents import Agent, AgentRuntime, tool +from conductor.ai.agents import Agent, AgentRuntime, tool @tool(approval_required=True) def transfer_funds(from_acct: str, to_acct: str, amount: float) -> dict: @@ -374,7 +385,7 @@ if status.is_waiting: ### Guardrails ```python -from agentspan.agents import Agent, AgentRuntime, Guardrail, GuardrailResult, OnFail, guardrail +from conductor.ai.agents import Agent, AgentRuntime, Guardrail, GuardrailResult, OnFail, guardrail @guardrail def word_limit(content: str) -> GuardrailResult: @@ -396,7 +407,7 @@ with AgentRuntime() as runtime: ### Streaming ```python -from agentspan.agents import Agent, AgentRuntime +from conductor.ai.agents import Agent, AgentRuntime agent = Agent(name="writer", model="openai/gpt-4o") @@ -413,7 +424,7 @@ with AgentRuntime() as runtime: ### Server-Side Tools (No Workers Needed) ```python -from agentspan.agents import Agent, AgentRuntime, api_tool, http_tool, mcp_tool +from conductor.ai.agents import Agent, AgentRuntime, api_tool, http_tool, mcp_tool # Point to any OpenAPI/Swagger spec — all endpoints auto-discovered stripe = api_tool( @@ -448,8 +459,8 @@ Three ways to connect APIs — all server-side, no workers needed: ### Code Execution ```python -from agentspan.agents import Agent, AgentRuntime -from agentspan.agents.code_executor import DockerCodeExecutor +from conductor.ai.agents import Agent, AgentRuntime +from conductor.ai.agents.code_executor import DockerCodeExecutor executor = DockerCodeExecutor(image="python:3.12-slim", timeout=30) agent = Agent( @@ -466,7 +477,7 @@ with AgentRuntime() as runtime: ### Shared State (Tool Context) ```python -from agentspan.agents import Agent, AgentRuntime, tool, ToolContext +from conductor.ai.agents import Agent, AgentRuntime, tool, ToolContext @tool def add_item(item: str, context: ToolContext) -> str: @@ -499,7 +510,7 @@ Hook into agent, model, and tool lifecycle events with `CallbackHandler` classes ```python import time -from agentspan.agents import Agent, AgentRuntime, CallbackHandler +from conductor.ai.agents import Agent, AgentRuntime, CallbackHandler class TimingHandler(CallbackHandler): def on_agent_start(self, **kwargs): @@ -515,7 +526,7 @@ class LoggingHandler(CallbackHandler): agent = Agent( name="my_agent", - model="openai/gpt-4o-mini", + model="anthropic/claude-sonnet-4-6", instructions="You are a helpful assistant.", callbacks=[TimingHandler(), LoggingHandler()], ) @@ -730,4 +741,4 @@ See [API Reference](docs/python-sdk/api-reference.md) for the complete API refer ## License -[MIT](LICENSE) +[MIT](LICENSE) \ No newline at end of file diff --git a/SKILL.md b/SKILL.md index c1e90b81b..6fc98a855 100644 --- a/SKILL.md +++ b/SKILL.md @@ -11,7 +11,7 @@ Agentspan is a distributed, durable runtime for AI agents. Agents survive crashe ## Quickstart (Ephemeral — for autonomous agents) ```python -from agentspan.agents import Agent, AgentRuntime +from conductor.ai.agents import Agent, AgentRuntime agent = Agent(name="helper", model="openai/gpt-4o", instructions="You are a helpful assistant.") @@ -26,7 +26,7 @@ with AgentRuntime() as rt: ## Production Pattern (for developers) ```python -from agentspan.agents import Agent, AgentRuntime +from conductor.ai.agents import Agent, AgentRuntime agent = Agent(name="helper", model="openai/gpt-4o", instructions="...") @@ -47,7 +47,7 @@ Trigger from outside: `agentspan run helper "What is quantum computing?"` rt = AgentRuntime() # Explicit: -from agentspan.agents import AgentConfig +from conductor.ai.agents import AgentConfig config = AgentConfig(server_url="http://localhost:6767/api", api_key="...") rt = AgentRuntime(config=config) ``` @@ -79,7 +79,7 @@ Model formats: `"openai/gpt-4o"`, `"anthropic/claude-sonnet-4-6"`, `"google_gemi ### @agent Decorator ```python -from agentspan.agents import agent +from conductor.ai.agents import agent @agent(model="openai/gpt-4o", tools=[search]) def researcher(): @@ -126,7 +126,7 @@ For autonomous agents building ephemeral agents — always check `result.is_succ ## Tools ```python -from agentspan.agents import tool +from conductor.ai.agents import tool @tool def search(query: str) -> str: @@ -145,7 +145,7 @@ Tool functions must have type hints and a docstring. The schema is generated aut ### ToolContext (dependency injection + shared state) ```python -from agentspan.agents import tool, ToolContext +from conductor.ai.agents import tool, ToolContext @tool def lookup(query: str, context: ToolContext) -> str: @@ -176,7 +176,7 @@ def get_cart(context: ToolContext) -> list: ### Server-side tools (no local worker needed) ```python -from agentspan.agents import http_tool, mcp_tool, api_tool +from conductor.ai.agents import http_tool, mcp_tool, api_tool weather = http_tool( name="get_weather", @@ -205,7 +205,7 @@ All multi-agent compositions use one `Agent(...)` as the *parent* with `agents=[ ### Strategy Enum ```python -from agentspan.agents import Strategy +from conductor.ai.agents import Strategy Strategy.SEQUENTIAL # Run in order; output of one feeds the next Strategy.PARALLEL # Run concurrently; results aggregated @@ -257,7 +257,7 @@ Agent( ### SWARM (peer-to-peer handoff) ```python -from agentspan.agents.handoff import OnTextMention +from conductor.ai.agents.handoff import OnTextMention coder = Agent(name="coder", model="openai/gpt-4o", instructions="Code. Say HANDOFF_TO_QA when done.") qa = Agent(name="qa", model="openai/gpt-4o", instructions="Test. Say HANDOFF_TO_CODER if bugs found.") @@ -277,11 +277,11 @@ Agent( ### Scatter-Gather (fan-out/fan-in) ```python -from agentspan.agents import scatter_gather +from conductor.ai.agents import scatter_gather coordinator = scatter_gather( name="multi_search", - worker=Agent(name="searcher", model="openai/gpt-4o-mini", instructions="Search and summarize."), + worker=Agent(name="searcher", model="anthropic/claude-sonnet-4-6", instructions="Search and summarize."), timeout_seconds=300, ) # Spawns multiple copies of worker agent in parallel, aggregates results @@ -333,7 +333,7 @@ Mix and match — `parallel >> sequential`, `router → swarm`, etc. The `>>` op ### Agent as Tool ```python -from agentspan.agents import agent_tool +from conductor.ai.agents import agent_tool specialist = Agent(name="math_expert", model="openai/gpt-4o", instructions="Solve math problems.") @@ -348,7 +348,7 @@ orchestrator = Agent( ## Guardrails ```python -from agentspan.agents import RegexGuardrail, LLMGuardrail, Guardrail, GuardrailResult +from conductor.ai.agents import RegexGuardrail, LLMGuardrail, Guardrail, GuardrailResult # Regex: block emails in output RegexGuardrail( @@ -362,7 +362,7 @@ RegexGuardrail( # LLM: policy-based check LLMGuardrail( name="safety", - model="openai/gpt-4o-mini", + model="anthropic/claude-sonnet-4-6", policy="Reject responses with medical advice.", on_fail="raise", ) @@ -379,7 +379,7 @@ Guardrail(no_ssn, position="output", on_fail="retry", max_retries=3) ## Termination Conditions ```python -from agentspan.agents import TextMentionTermination, MaxMessageTermination +from conductor.ai.agents import TextMentionTermination, MaxMessageTermination Agent( name="worker", @@ -394,7 +394,7 @@ Agent( ## Gates (Conditional Pipelines) ```python -from agentspan.agents.gate import TextGate +from conductor.ai.agents.gate import TextGate checker = Agent(name="checker", model="openai/gpt-4o", instructions="Output NO_ISSUES if everything is fine.", @@ -408,7 +408,7 @@ pipeline = checker >> fixer # fixer only runs if checker finds issues ## Memory ```python -from agentspan.agents import ConversationMemory, SemanticMemory +from conductor.ai.agents import ConversationMemory, SemanticMemory # Conversation memory (chat history with windowing) agent = Agent( @@ -427,7 +427,7 @@ results = memory.search("What language does the user prefer?") ## Claude Code Agents ```python -from agentspan.agents import Agent, ClaudeCode +from conductor.ai.agents import Agent, ClaudeCode # Simple: slash syntax reviewer = Agent( @@ -467,10 +467,10 @@ Agent( Attach one or more cron triggers to an agent at deploy time. The scheduler fires the agent on its cadence with the supplied input. ```python -from agentspan.agents import Agent, AgentRuntime -from agentspan.agents.schedule import Schedule +from conductor.ai.agents import Agent, AgentRuntime +from conductor.ai.agents.schedule import Schedule -agent = Agent(name="hello", model="openai/gpt-4o-mini", instructions="Say hi.") +agent = Agent(name="hello", model="anthropic/claude-sonnet-4-6", instructions="Say hi.") with AgentRuntime() as rt: rt.deploy( @@ -526,7 +526,7 @@ Agent( ## Callbacks ```python -from agentspan.agents import CallbackHandler +from conductor.ai.agents import CallbackHandler class MyCallbacks(CallbackHandler): def on_agent_start(self, **kwargs): pass @@ -542,7 +542,7 @@ Agent(name="agent", model="openai/gpt-4o", callbacks=[MyCallbacks()]) Use `rt.start()` + `handle.stream()` to react to events as they fire — required for interactive HITL (approval tools). ```python -from agentspan.agents import EventType +from conductor.ai.agents import EventType with AgentRuntime() as rt: handle = rt.start(agent, "Transfer $500 from A to B") @@ -569,7 +569,7 @@ with AgentRuntime() as rt: For LLM-generated plans that should run deterministically (DAG compiled into a Conductor workflow): ```python -from agentspan.agents import plan_execute, Agent, tool +from conductor.ai.agents import plan_execute, Agent, tool planner = Agent(name="planner", model="openai/gpt-4o", instructions="Produce a JSON plan...") fallback = Agent(name="fixer", model="openai/gpt-4o", instructions="Repair failed plan.") @@ -615,7 +615,7 @@ Agent(name="analyzer", model="openai/gpt-4o", output_type=Analysis) ```python from langgraph.prebuilt import create_react_agent from langchain_openai import ChatOpenAI -from agentspan.agents import AgentRuntime +from conductor.ai.agents import AgentRuntime llm = ChatOpenAI(model="gpt-4o") graph = create_react_agent(llm, tools=[my_tool]) @@ -629,7 +629,7 @@ with AgentRuntime() as rt: ```python from agents import Agent as OpenAIAgent -from agentspan.agents import AgentRuntime +from conductor.ai.agents import AgentRuntime agent = OpenAIAgent(name="helper", instructions="...", model="gpt-4o") @@ -697,12 +697,12 @@ A research-and-publish pipeline showing tools, server-side tools, shared state, ```python from pydantic import BaseModel -from agentspan.agents import ( +from conductor.ai.agents import ( Agent, AgentRuntime, Strategy, tool, http_tool, agent_tool, RegexGuardrail, LLMGuardrail, EventType, ToolContext, ) -from agentspan.agents.gate import TextGate -from agentspan.agents.schedule import Schedule +from conductor.ai.agents.gate import TextGate +from conductor.ai.agents.schedule import Schedule # 1. Tools — local Python + server-side HTTP @@ -731,17 +731,17 @@ class Article(BaseModel): tags: list[str] # 3. Parallel research phase (two specialists run concurrently) -market = Agent(name="market", model="openai/gpt-4o-mini", tools=[fetch_news, remember], +market = Agent(name="market", model="anthropic/claude-sonnet-4-6", tools=[fetch_news, remember], instructions="Research market signals. Save findings with remember().") -risk = Agent(name="risk", model="openai/gpt-4o-mini", tools=[remember], +risk = Agent(name="risk", model="anthropic/claude-sonnet-4-6", tools=[remember], instructions="Identify risks. Save findings with remember().") -research = Agent(name="research_phase", model="openai/gpt-4o-mini", +research = Agent(name="research_phase", model="anthropic/claude-sonnet-4-6", agents=[market, risk], strategy=Strategy.PARALLEL) # 4. Quality gate — skip writing if research is empty quality_check = Agent( - name="quality_check", model="openai/gpt-4o-mini", + name="quality_check", model="anthropic/claude-sonnet-4-6", instructions="Output NO_SIGNAL if the research lacks substance. Otherwise summarize.", gate=TextGate("NO_SIGNAL"), ) @@ -755,7 +755,7 @@ writer = Agent( guardrails=[ RegexGuardrail(name="no_emails", patterns=[r"[\w.+-]+@[\w-]+\.[\w.-]+"], message="Remove emails.", on_fail="retry", max_retries=2), - LLMGuardrail(name="safety", model="openai/gpt-4o-mini", + LLMGuardrail(name="safety", model="anthropic/claude-sonnet-4-6", policy="Reject content with PII or medical advice.", on_fail="raise"), ], ) diff --git a/cli/cmd/server.go b/cli/cmd/server.go index d65a55f98..1f2397bf2 100644 --- a/cli/cmd/server.go +++ b/cli/cmd/server.go @@ -78,7 +78,7 @@ func init() { serverStartCmd.Flags().StringVarP(&serverModel, "model", "m", "", "Default LLM model (e.g. openai/gpt-4o)") serverStartCmd.Flags().StringVar(&serverVersion, "version", "", "Specific server version to download (e.g. 0.1.0)") serverStartCmd.Flags().StringVar(&serverJar, "jar", "", "Path to a local JAR file to use directly") - serverStartCmd.Flags().BoolVar(&serverLocal, "local", false, "Use locally built JAR from server/build/libs/") + serverStartCmd.Flags().BoolVar(&serverLocal, "local", false, "Use locally built JAR from server/conductor-agentspan-server/build/libs/") serverLogsCmd.Flags().BoolVarP(&followLogs, "follow", "f", false, "Follow log output") serverLogsCmd.Flags().IntVarP(&tailLines, "lines", "n", 20, "Number of lines to show before following (with -f)") @@ -409,7 +409,8 @@ func lastNLinesOffset(f *os.File, n int) (int64, error) { // --- Local JAR helpers --- func findLocalJAR() (string, error) { - // Try CWD first, then walk up to find 'server/build/libs/agentspan-runtime.jar' + // Try CWD first, then walk up to find + // 'server/conductor-agentspan-server/build/libs/agentspan-runtime.jar' cwd, err := os.Getwd() if err != nil { return "", fmt.Errorf("get working directory: %w", err) @@ -417,15 +418,16 @@ func findLocalJAR() (string, error) { // Check common relative paths from likely CWD locations candidates := []string{ - filepath.Join(cwd, "server", "build", "libs", jarName), + filepath.Join(cwd, "server", "conductor-agentspan-server", "build", "libs", jarName), + filepath.Join(cwd, "conductor-agentspan-server", "build", "libs", jarName), filepath.Join(cwd, "build", "libs", jarName), - filepath.Join(cwd, "..", "server", "build", "libs", jarName), + filepath.Join(cwd, "..", "server", "conductor-agentspan-server", "build", "libs", jarName), } - // Also walk up from CWD looking for server/build/libs/ + // Also walk up from CWD looking for server/conductor-agentspan-server/build/libs/ dir := cwd for i := 0; i < 5; i++ { - candidate := filepath.Join(dir, "server", "build", "libs", jarName) + candidate := filepath.Join(dir, "server", "conductor-agentspan-server", "build", "libs", jarName) candidates = append(candidates, candidate) parent := filepath.Dir(dir) if parent == dir { diff --git a/cli/cmd/server_test.go b/cli/cmd/server_test.go index 27ee27e72..d8bbcb61d 100644 --- a/cli/cmd/server_test.go +++ b/cli/cmd/server_test.go @@ -175,6 +175,44 @@ func TestServerStartUsesRequestedVersion(t *testing.T) { } } +// After the server split (#271) the runnable jar lives at +// server/conductor-agentspan-server/build/libs/, not server/build/libs/. +// findLocalJAR must locate it both from the repo root and from a nested CWD. +func TestFindLocalJARNewSplitPath(t *testing.T) { + root := t.TempDir() + jarDir := filepath.Join(root, "server", "conductor-agentspan-server", "build", "libs") + if err := os.MkdirAll(jarDir, 0o755); err != nil { + t.Fatalf("mkdir jar dir: %v", err) + } + jarPath := filepath.Join(jarDir, jarName) + if err := os.WriteFile(jarPath, []byte("fake jar"), 0o644); err != nil { + t.Fatalf("write jar: %v", err) + } + wantPath, err := filepath.Abs(jarPath) + if err != nil { + t.Fatalf("abs jar path: %v", err) + } + + // Nested CWD exercises the walk-up loop; repo root exercises the candidates list. + nested := filepath.Join(root, "cli", "cmd") + if err := os.MkdirAll(nested, 0o755); err != nil { + t.Fatalf("mkdir nested: %v", err) + } + + for _, cwd := range []string{root, nested} { + t.Run(cwd, func(t *testing.T) { + t.Chdir(cwd) + got, err := findLocalJAR() + if err != nil { + t.Fatalf("findLocalJAR from %q returned error: %v", cwd, err) + } + if got != wantPath { + t.Fatalf("findLocalJAR from %q = %q, want %q", cwd, got, wantPath) + } + }) + } +} + func freeTCPPort(t *testing.T) string { t.Helper() ln, err := net.Listen("tcp", "127.0.0.1:0") diff --git a/design/agentspan-design.md b/design/agentspan-design.md new file mode 100644 index 000000000..a1ad55fd3 --- /dev/null +++ b/design/agentspan-design.md @@ -0,0 +1,309 @@ +# Agentspan Design + +**Status:** Consolidated 2026-06-26 + +**Scope:** This is the canonical platform architecture and server-feature reference for Agentspan. It covers the core model ("everything is an agent"), how an `AgentConfig` compiles to a Conductor `WorkflowDef`, how those workflows execute (worker dispatch, durability), the library/server module split and its SPIs, multi-agent orchestration with pipeline context passing, and the server-side feature endpoints (HITL, dynamic DAG injection, agent signals) plus the `agentspan deploy` CLI. SDK-authoring detail (per-language idioms, serialization rules, worker registration mechanics) lives in [sdk-design.md](sdk-design.md); the REST/SSE contract in [api-design.md](api-design.md); and adjacent subsystems in [guardrails-design.md](guardrails-design.md), [tool-execution-and-credentials-design.md](tool-execution-and-credentials-design.md), [framework-integration.md](framework-integration.md), [sentinel-agents.md](sentinel-agents.md), and [stateful-agents.md](stateful-agents.md). + +--- + +## 1. Overview & "Everything Is an Agent" + +Agentspan is a server-first agent execution platform built on [Conductor](https://conductor-oss.org). An SDK (Python is the reference; TypeScript, Java, Go, Kotlin, C#, Ruby mirror it) defines agents, tools, guardrails, and callbacks as language-native constructs, serializes them to a single **`AgentConfig` JSON**, and posts that to the server. The server **compiles** the config into a durable Conductor `WorkflowDef`, executes it on the Conductor engine, and streams events back over SSE. The SDK's remaining job at runtime is to run **workers** that the engine dispatches tool/guardrail/callback work to. + +``` +┌─────────────────────────────────────────────────┐ +│ SDK (any language) │ +│ Agent definition → serialize → AgentConfig JSON │ +│ Worker poll loop → tool execution → results │ +│ SSE client → event stream → AgentStream │ +└──────────────────────┬──────────────────────────┘ + │ REST + SSE (JSON) +┌──────────────────────▼──────────────────────────┐ +│ Agentspan (Java, on Conductor) │ +│ Compiler → Conductor WorkflowDef │ +│ Executor → Conductor workflow engine │ +│ Stream → SSE events │ +│ Secrets → AES-256-GCM store + exec tokens │ +└─────────────────────────────────────────────────┘ +``` + +**Everything is an agent.** There is one unifying domain object — the `Agent` — and a single serialization format (`AgentConfig`). A bare LLM call, a tool-using ReAct loop, a multi-agent swarm, a sequential pipeline, a router, and a plan-and-execute planner are all just `AgentConfig`s that differ in which fields are populated. Composition is recursive: the `agents` array of an `AgentConfig` holds nested `AgentConfig`s, and a nested agent invoked as a tool (`agent_tool`) compiles to a `SUB_WORKFLOW`. This recursion is what lets every orchestration strategy reduce to the same compile → execute → dispatch path. + +**No passthrough.** Framework agents (LangGraph, LangChain, OpenAI, Google ADK, Vercel AI SDK) are not run as black boxes. Each is decomposed into a proper `AgentConfig` and compiled through the same pipeline so that durability, per-tool observability, HITL, and distributed worker execution all apply. See [framework-integration.md](framework-integration.md). + +**Core principle for server features:** add no new Conductor primitives. HITL, signal injection, dynamic DAG injection, and context passing are all built on existing Conductor capabilities — `HUMAN` tasks, `updateVariables`, `SET_VARIABLE`/`INLINE`, workflow variables, and direct `ExecutionDAO` access. + +--- + +## 2. Compilation Model (`AgentConfig` JSON → Conductor `WorkflowDef`) + +The SDK serializes an agent tree to `AgentConfig` JSON and posts it to `POST /agent/start` (compile + register + execute) or `POST /agent/compile` (compile only, returns the `WorkflowDef` without running). **Producing identical `AgentConfig` JSON for equivalent agent definitions across SDKs is the primary correctness criterion** — the wire JSON must match byte-for-byte for round-tripping with the server compiler. + +### 2.1 `AgentConfig` shape (abridged) + +All keys are camelCase; null-valued keys are omitted; `strategy` is set only when `agents` is non-empty. + +```json +{ + "name": "agent_name", + "model": "provider/model_name", + "strategy": "handoff|sequential|parallel|router|round_robin|random|swarm|manual|plan_execute", + "maxTurns": 25, + "instructions": "string | { prompt_template } | null", + "tools": [ ToolConfig... ], + "agents": [ AgentConfig... ], + "router": "AgentConfig | { taskName }", + "guardrails": [ GuardrailConfig... ], + "outputType": { "schema": {...}, "className": "MyModel" }, + "callbacks": [ { "position": "before_agent", "taskName": "..." } ], + "credentials": ["GITHUB_TOKEN", "OPENAI_API_KEY"] +} +``` + +(Full field reference, `ToolConfig`/`GuardrailConfig` schemas, and per-SDK serialization rules: [sdk-design.md](sdk-design.md).) + +### 2.2 Compiler dispatch + +The server-side `AgentCompiler` (plus `ToolCompiler`, `GuardrailCompiler`, `MultiAgentCompiler`) inspects the config and dispatches by shape: + +| Config shape | Compiles to | +|---|---| +| No tools, no sub-agents | a single `LLM_CHAT_COMPLETE` task; output = `${llm.output.result}` | +| Tools, no sub-agents | a `DO_WHILE` ReAct loop (§2.3) | +| Sub-agents (`agents` set) | a multi-agent strategy graph (§5) | +| Tools **and** sub-agents (hybrid) | a `DO_WHILE` loop whose tool set includes `transfer_to_{name}` handoff tools, followed by a `SWITCH` | + +The compilers emit only stable `conductor-common` models (`WorkflowDef`, `WorkflowTask`, `TaskDef`) — no engine internals — which is what makes the library/server split in §4 clean. + +### 2.3 The ReAct loop (single agent with tools) + +The canonical compiled shape is a Conductor `DO_WHILE`: + +``` +[SET_VARIABLE: init messages] + │ + ▼ +[DO_WHILE] + ├─ [LLM_CHAT_COMPLETE] reads ${workflow.variables.messages}, json_output=true + ├─ [dispatch_worker] routes tool calls, updates messages + │ llm_response=${llm.output.result} messages=${workflow.variables.messages} + ├─ [SET_VARIABLE] messages=${dispatch.output.messages} + └─ [stop_when_worker] (optional) + condition: $.loop.iteration < maxTurns + && $.dispatch.continue_loop == true + [&& $.stop_when.should_continue == true] + │ + ▼ +Output: ${dispatch.output.result} +``` + +> **Conductor quirk:** in `DO_WHILE` conditions, task references map directly to `outputData` with **no** `.output` wrapper — `$.dispatch.continue_loop`, not `$.dispatch.output.continue_loop`. + +Tool calls produced by the LLM are routed by an **enrichment script** (an `INLINE` GraalJS task) into a `FORK_JOIN_DYNAMIC` + `JOIN` so that all tool calls in a turn run in parallel, each mapped to its task type (`SIMPLE`/`HTTP`/`CALL_MCP_TOOL`/`SUB_WORKFLOW`/`INLINE`). Output guardrails compile into the loop body as durable tasks with a `SWITCH` on the result; input guardrails are an SDK-side pre-check. See [guardrails-design.md](guardrails-design.md). + +### 2.4 Task-def registration (server-side, at compile time) + +Conductor task definitions (timeout/retry config) are registered **by the server during compilation**, not by SDKs. After `compile()`, `AgentService.registerAllTaskDefs(WorkflowDef)` walks the entire workflow tree and registers a `TaskDef` for every `SIMPLE` task. This eliminated a class of bugs where the same timeout (`120s`) had been hardcoded independently in the Python SDK, the TS SDK, and the server. + +- **Defaults:** `timeoutSeconds: 0` (no overall timeout), `responseTimeoutSeconds: 3600`, `retryCount: 2`, `retryDelaySeconds: 2`, `retryLogic: LINEAR_BACKOFF`. `responseTimeoutSeconds` is currently hardcoded to `3600` in `registerTaskDef` — there is no per-tool `timeoutSeconds` override. +- **SDKs do not register task defs.** They only poll for and execute tasks (`register_task_def=False` in Python; the TS `registerTaskDef()` is a no-op kept for backward compat). + +--- + +## 3. Execution Model (Conductor workflows, worker dispatch, durability) + +Once compiled, an `AgentConfig` runs as a normal Conductor workflow — every benefit of durable execution comes for free. + +### 3.1 Runtime lifecycle + +``` +runtime.run/start/stream(agent, prompt) + └─ _compile_agent(agent) # cached per agent.name + └─ serialize → POST /agent/compile (server AgentCompiler dispatches by shape) + └─ ToolRegistry.register_tool_workers() # start local Conductor workers + └─ POST /agent/start # engine executes the WorkflowDef + └─ SSE / poll for events and result +``` + +A module-level **singleton `AgentRuntime`** is shared by `run`/`start`/`stream`/`run_async` so Conductor clients and worker processes are created once, not per call. + +### 3.2 Worker dispatch + +Native `@tool` functions (and guardrails/callbacks with local implementations) compile to `SIMPLE` tasks. The SDK runs a poll loop (thread/goroutine/fiber) that: + +1. Polls Conductor for tasks by name. +2. Executes the registered worker function. +3. Returns a `TaskResult`. + +The universal **`dispatch_worker`** is the tool-execution router: it receives the LLM response, parses tool calls, invokes the matching local functions (handling approval flags, circuit-breaker error counts, `ToolContext`), updates the message history, and signals `continue_loop`. Because it is shared across all agents and registered once per task name, tool functions and per-tool state live in module-level registries (`_tool_registry`, `_tool_error_counts`, `_tool_approval_flags`). + +**External / by-reference work:** when a `worker` tool (or guardrail/agent) has no local function, the SDK emits only the task name. A remote worker — possibly in another language on another machine — picks the task up off Conductor's queue. The SDK registers no local worker for it. + +### 3.3 Durability + +Because state lives in Conductor (workflow variables, task I/O, the message history in `${workflow.variables.messages}`), an agent execution survives worker crashes and restarts, is fully inspectable and replayable, and supports long pauses. HITL (`HUMAN` tasks) makes **long-paused executions routine, not rare** — an execution can sit paused for days awaiting human input and resume cleanly. This durability is the entire reason for the no-passthrough rule (§1): a black-box framework task would forfeit crash recovery, per-tool visibility, and HITL. + +--- + +## 4. Library / Server Split & SPIs + +Agentspan is structured as a **library that Conductor depends on**, not a standalone app that bundles Conductor. The dependency direction is inverted: `orkes-conductor` (and the OSS standalone) depend on the `conductor-agentspan` artifact; Agentspan compiles against Conductor APIs as `compileOnly`/provided so the **host owns the Conductor version**. + +### 4.1 Two modules + +- **`conductor-agentspan`** (plain jar) — **SPI interfaces + core logic only.** Agent domain (`model/`, `normalizer/`, `compiler/*`), the services that operate on Conductor and on the SPIs (`AgentService`, `AgentDagService`, `AgentStreamRegistry`), custom system tasks, the AI provider, REST controllers, and the credential-resolution/masking logic. **No concrete store/DAO/crypto implementations.** +- **`conductor-agentspan-server`** (bootJar + Docker) — the OSS runtime and standalone app. Bundles the **default SPI implementations**, the OSS Conductor runtime (persistence, scheduler, rest, http-task, json-jq-task), the launcher (`AgentRuntime` main), web/UI config, and the standalone-only auth scaffolding. + +Why two and not three: the compilers emit Conductor `WorkflowDef`/`WorkflowTask`, and there is no consumer of the agent logic *without* an engine, so an engine-free "core" module buys nothing. Conductor execution is itself **not** an SPI — Conductor's `WorkflowService`/`MetadataService`/`ExecutionService` (and the DAOs beneath) already are that interface, and the engine is a non-goal to swap; wrapping them adds a redundant layer with no override value. `AgentService` injects them directly. + +### 4.2 The SPI layer + +Interfaces live in `conductor-agentspan` (`dev.agentspan.runtime.spi`); they cover **Agentspan-owned data**, not execution. The library holds no impls — a host contributes one impl bean per SPI (via `@ConditionalOnMissingBean`, the same pattern orkes uses for http-task/DAOs/security). A context missing an impl **fails fast at startup** — intentional, so a missing secret store cannot silently no-op. + +The directory (`dev.agentspan.runtime.spi`) contains exactly five interfaces: + +| SPI (library) | OSS default (`conductor-agentspan-server`) | Enterprise (orkes) | +|---|---|---| +| `CredentialStoreProvider` | `EncryptedDbCredentialStoreProvider` (JDBC `credentials_store` + AES-256-GCM) | secrets manager / Vault / KMS | +| `SecretOutputMasker` | **no-op** (payload unchanged) | disclosure-tracking masker | +| `SkillPackageStore` (+ `StoredSkillPackage` value type) | `FileSystemSkillPackageStore` / `ConductorPayloadSkillPackageStore` | S3 / object store | +| `SkillMetadataDAO` | `FileSystemSkillMetadataDAO` | DB-backed | + +```java +// OSS default returns payload unchanged; enterprise redacts disclosed secret values. +public interface SecretOutputMasker { String mask(String executionId, String userId, String payload); } + +public interface SkillMetadataDAO { + SkillDetail save(SkillDetail detail); + List list(boolean allVersions, String ownerId); + Optional get(String ownerId, String name, String version); + void delete(String ownerId, String name, String version); +} +``` + +**Execution tokens are not an SPI.** Worker-boundary tokens (§4.4) are minted/validated by a concrete `@Service` `ExecutionTokenService` (`dev.agentspan.runtime.credentials`) using HMAC-SHA256 over the server master key — not a pluggable interface. + +> Secrets resolution is a direct `(userId, name)` lookup with dotted-JSONPath into JSON-valued secrets (`GCP_SVC.project_id`) and prefix-permissive declared-name bounding — implemented in `CredentialResolutionService` over `CredentialStoreProvider`. There is **no** binding/alias store. There is **no** `UserStore`/`ApiKeyStore`: identity is the host's (orkes supplies it; OSS Conductor has none → anonymous). The library only needs the current principal (`userId`) for secret scoping, carried by `RequestContextHolder`; *who populates it* is the host's job. Full secret/credential mechanics: [tool-execution-and-credentials-design.md](tool-execution-and-credentials-design.md). + +### 4.3 Spring wiring + +The library registers via **Spring Boot auto-configuration** (`META-INF/spring/...AutoConfiguration.imports` → `AgentSpanAutoConfiguration`), not a component scan — orkes-conductor's `@ComponentScan` does not cover `dev.agentspan.*`. Two layers: (a) the library auto-config wires the *logic* beans, each `@ConditionalOnBean` on the SPIs it needs; (b) the host contributes one impl bean per SPI (`conductor-agentspan-server` ships the OSS `AgentSpanDefaultImplConfiguration`; orkes contributes its own). + +**`@Primary` landmines must become opt-in.** Beans that override Conductor's own (`CredentialAwareHttpTask` as `HTTP`, `CredentialAwareMcpService` extending `MCPService`, `AgentHumanTask` as `HUMAN`, the agent status listener, the JDBC `DataSource`) must be property-gated / `@ConditionalOnMissingBean` (default on standalone, off embedded) so they do not hijack host behavior. The JDBC `DataSource` is qualified (`agentspanDataSource`), never `@Primary`. + +### 4.4 Two auth boundaries + +1. **User boundary** — `/api/secrets`, `/api/agent/*`, `/api/skill`. Agentspan' own `AuthFilter` is **standalone-only** (ships in the server module, off by default); when embedded, the host owns authN/authZ and an adapter populates the principal. +2. **Worker boundary** — `/api/workers/secrets`, gated by HMAC **execution tokens**, independent of user auth, so in-flight workers can always reach it. The host's security chain must not block `/api/workers/**`. + +### 4.5 Consumption modes & version alignment + +`compileOnly` means nothing bundles or enforces a Conductor version — the host's classpath wins. Drift is scoped to three modes: + +| Mode | Takes | Conductor version | Drift risk | Owner | +|---|---|---|---|---| +| **A — Standalone** | `conductor-agentspan-server` bootJar/Docker | fixed, bundled | none (one `conductorVersion` for lib+server) | us | +| **B — Self-embed (external OSS)** | `conductor-agentspan` library | host-supplied | real | the host → **build from source** against your engine (eliminates drift), or take the jar + self-certify via the SDK conformance suite | +| **C — Enterprise embed** | `orkes-conductor` | orkes pins | single certified pair | orkes | + +No declared compatibility *range* — an unverified range is a false promise; the SDK conformance suite (black-box HTTP, parameterized by server URL) is the only interoperability oracle, and the interop surface is kept tiny precisely because execution is not an SPI (§4.1). + +--- + +## 5. Orchestration (multi-agent strategies + pipeline context) + +`MultiAgentCompiler` compiles the strategies. All reduce to Conductor control-flow over `SUB_WORKFLOW`s, which is why composition is recursive. + +| Strategy | Compiled shape | +|---|---| +| `handoff` | router `LLM_CHAT_COMPLETE` → `SWITCH` → one sub-agent `SUB_WORKFLOW` per case | +| `sequential` | chain of `SUB_WORKFLOW`s; each step's prompt = prior step's `output.result` | +| `parallel` | `FORK` → N `SUB_WORKFLOW`s → `JOIN`; output namespaced by agent | +| `router` | agent- or function-based selector → `SWITCH` → chosen `SUB_WORKFLOW` | +| `swarm` / `manual` / `round_robin` / `random` | shared `DO_WHILE` loop; `active_agent` + `conversation` in `SET_VARIABLE`, agents handed off in place | +| `plan_execute` (PAC/PAE) | planner agent emits a JSON DAG; the server compiles that JSON into a deterministic sub-workflow | +| hybrid (tools + sub-agents) | tool `DO_WHILE` with `transfer_to_{name}` tools → `SWITCH` | + +### 5.1 Pipeline context passing + +Only LLM text used to flow between agents, so concrete artifacts (repo paths, branch names, PR URLs) produced by tools were lost across boundaries — a real failure mode (a 3-step pipeline once had 3 agents working on 3 different repos). The fix: a **context dict** flows alongside the text output through every boundary. + +- **Structure:** a single-level key-value map; values are any JSON-serializable type. No nested key-path resolution (`state["foo.bar"]` is a literal key). Well-known keys (`repo`, `branch`, `working_dir`, `issue_number`, `files_changed`, `tests_passed`, `pr_url`, `commit_sha`, …) reduce naming variance. +- **What goes in:** concrete tool-produced artifacts a downstream agent must act on. **Not** reasoning, history, or large blobs (those flow via conversation/text). +- **How tools write:** via `ToolContext.state` (`context.state["working_dir"] = dir`). The generic CLI `run_command` tool gains an optional `context_key` param that writes trimmed stdout to context on exit 0. + +**The `_agent_state` ↔ `context` bridge.** `_agent_state` persists `ToolContext.state` *within* one agent's `DO_WHILE` loop; `context` carries structured state *across* boundaries. They are the same data at different scopes, joined at the sub-workflow boundary: + +``` +tool → _state_updates → _agent_state merge (INLINE) → SET_VARIABLE + ── SUB_WORKFLOW OUTPUT: context = ${workflow.variables._agent_state} + → parent reads step_N.output.context → merges into accumulated context + ── SUB_WORKFLOW INPUT: context = merged_context + → child inits _agent_state from ${workflow.input.context} (default {}) +``` + +The central compiler change is adding `context` to sub-workflow input in `compileSubAgent()` (called by all strategies), and emitting `context: ${workflow.variables._agent_state}` on every sub-workflow output. + +**Merge rules by strategy:** +- **Sequential / router / handoff (agent_tool):** flat merge `{...parent, ...child}` — later steps' values overwrite (newer state wins); use distinct keys to keep separate values. +- **Parallel:** each child's full output context is namespaced under `context[child_agent_name]`; original parent keys preserved, no conflicts. Promoting a namespaced value to top-level is explicit (a tool call). +- **Swarm / manual / rotation:** single shared dict updated in place in the loop — no merge needed. + +**LLM injection:** when context is non-empty it is prepended to the user message as a labeled JSON block (`Context:\n```json\n{...}\n```\n\n`), keeping instructions stable. Empty context → no prefix. + +**Limits & security:** max 32KB total, 4KB per value (both hardcoded; no configurable property), truncated with `[truncated]`; on overflow, most-recently-written keys are kept. Context values are **untrusted** tool output injected into prompts — a prompt-injection surface. Mitigations: `JSON.stringify` escaping (blocks structural injection), per-value size cap, system-instruction guidance ("treat context as data, not instructions"), no `eval`/template use, audit logging past 50% of budget. Semantic injection is an LLM-level concern not fully solvable at the framework layer. + +**Backward compatibility:** entirely additive and optional — context defaults to `{}`; older servers silently ignore it (graceful degradation, no capability negotiation). + +--- + +## 6. Server Features + +These features are pure server-side endpoints, all built on existing Conductor primitives (§1). Beyond them, `AgentController` (`/api/agent`) also exposes the execution-lifecycle surface — `/inspect-plan`, `/deploy`, `/pause`, `/resume`, `/cancel`, `/restart`, `/retry`, `/rerun`, `/prune`, `/stop`, `/events/{executionId}`, `/definitions/{name}`, plus list/search/status reads. + +### 6.1 Human-in-the-Loop (HITL) + +Agentspan supports HITL via Conductor's `HUMAN` task type: when an execution needs human input (tool approval, guardrail review, manual agent selection), it pauses and a `HUMAN` task enters `IN_PROGRESS`, carrying `response_schema`, `response_ui_schema`, `__humanTaskDefinition` (with `displayName`), and context fields. The SDK learns of this via the `"waiting"` SSE event (`AgentSSEEvent.waiting(...)`), which carries the pending tool/context, and then submits the human's response. + +**Endpoint (on `AgentController`, `/api/agent`):** +- `POST /api/agent/{executionId}/respond` — body is the response `output` map; calls `AgentService.respond(executionId, output)` to complete the paused `HUMAN` task and resume the execution. Returns void. + +### 6.2 Dynamic DAG task injection + +The SDK's Dynamic DAG feature needs to display tool/sub-agent activity in the Conductor DAG of a running execution. Two endpoints back this, served by `AgentDagService`, which injects `ExecutionDAO` **directly** to mutate live execution/task state — bypassing the `WorkflowExecutor` decide loop (injected tasks have no counterpart in the `WorkflowDef`; they are display-only, so calling `decide()` would try and fail to advance the execution). `ExecutionDAOFacade` is avoided because its external-payload logic is unneeded for small tool-arg inputs. + +- **`POST /api/agent/{executionId}/tasks`** → `injectTask`: loads the `WorkflowModel` (404 if absent), builds a `TaskModel` (`IN_PROGRESS`, `SIMPLE` or `SUB_WORKFLOW`, `seq = tasks.size()+1`, `subWorkflowId` from the param for sub-workflows), and `executionDAO.createTasks(...)`. The task appears in `getExecutionStatus` via its `workflowInstanceId`. When the SDK later completes it via native `POST /api/task`, `decide()` runs but the main worker task is still `IN_PROGRESS`, so the execution stays `RUNNING` — no disruption. +- **`POST /api/agent/execution`** → `createTrackingWorkflow`: builds a minimal `WorkflowDef` + a `RUNNING` `WorkflowModel` and `executionDAO.createWorkflow(...)`, returning the new executionId for sub-agent display. (Static segment resolves before `GET /api/agent/{name}`.) A companion **`POST /api/agent/execution/{executionId}/complete`** finalizes a tracking workflow. + +**Concurrency:** duplicate `seq` from concurrent hooks is harmless (no uniqueness constraint on display-only tasks). A tracking execution is finalized explicitly by the SDK calling `POST /api/agent/execution/{executionId}/complete` (which marks the `WorkflowModel` `COMPLETED`); injected task-def names (`Bash`, `Read`) need not be registered since the tasks are display-only. + +### 6.3 Agent signals (injecting context into running workflows) + +Signals let a caller inject context into a running agent workflow. The current implementation is a **single-variable injection** on existing primitives — no disposition state machine. + +**Mechanism:** `AgentService.signalAgent(executionId, message)` loads the `WorkflowModel`, sets one workflow variable `_signal_injection` to the message string (or `""` if null), and persists it via `executionDAO.updateWorkflow(...)`. On each `DO_WHILE` iteration the context-injection script reads `_signal_injection` and prepends it to the LLM's user message (as a `[SIGNALS]...[/SIGNALS]` block). There is no accept/reject flow, no per-signal status tracking, and no recursive propagation to sub-workflows. + +**Endpoint (on `AgentController`, `/api/agent`):** +- `POST /api/agent/{executionId}/signal` — body `{ "message": "..." }`; calls `signalAgent(executionId, message)`. Returns void. This is the only signal endpoint. + +> Richer signaling (durable per-signal disposition, accept/reject tools, name-based broadcast, urgent pause/resume, recursive propagation, dedicated SSE events) is roadmap, not implemented. + +--- + +## 7. CLI Deploy + +`agentspan deploy` discovers agents from user code and registers them on the server, bridging the Go CLI with the Python/TS SDK `deploy()` paths. + +``` +agentspan deploy [--agents foo,bar] [--language python|typescript] [--package myapp] [--yes] [--json] [--server URL] +``` + +**Flow:** auto-detect language (marker files: `pyproject.toml`/`setup.py`/`requirements.txt` vs `package.json`+`tsconfig.json`; `--language` overrides; ambiguous/none → error) → verify runtime (venv-preferred `python3`/`python`, or `npx`) → infer package (Python dotted module, TS directory; `--package` overrides) → **discover** → filter (`--agents`) → **confirm** (skipped by `--yes`) → **deploy** → format output. Exit 1 on any failure. + +**Shell-out design:** the Go CLI delegates discovery and deployment to the SDK via subprocess (`exec.CommandContext`, 120s timeout), forwarding `AGENTSPAN_SERVER_URL`, `AGENTSPAN_API_KEY`, and `AGENTSPAN_AUTH_KEY`/`_SECRET` as **environment variables** (not args, to avoid leaking secrets in process lists). The SDK entry points print JSON to stdout, stderr to the user: +- **Discover** — `python -m agentspan.cli.discover --package ` / `npx tsx .../discover.ts --path ` → `[{name, framework}]`. (Python uses a dotted module; TS uses a filesystem path.) +- **Deploy** — `python -m agentspan.cli.deploy --package [--agents ...]` / `.../deploy.ts --path ` → `[{agent_name, registered_name, success, error}]`. Deployment calls `deploy()` **per agent** with individual try/except so one failure doesn't crash the batch — the Go CLI always gets parseable JSON. + +Subprocess non-zero exit with valid JSON on stdout → partial-failure results; non-zero with no JSON → stderr is the error. + +**Known TS limitations:** discovery finds only native `Agent` instances (no framework-agent discovery) and scans only the top-level directory (no recursion). diff --git a/design/api-design.md b/design/api-design.md new file mode 100644 index 000000000..e75ee8229 --- /dev/null +++ b/design/api-design.md @@ -0,0 +1,450 @@ +# API Design + +**Status:** Consolidated 2026-06-26 + +**Scope.** This is the canonical reference for the **SDK-facing API surface** — how every +language SDK lets a user declare tools and agents — and for the **wire schema** those SDKs +serialize to and POST to the server. It covers the `AgentConfig` JSON contract, the tool +declaration conventions (`tool` / `httpTool` / `mcpTool` / `apiTool`), the `api_tool()` +auto-discovery feature, and the `Agent(model=...)` model conventions including +`Agent(model="claude-code")`. Detailed REST server endpoints (start/compile/poll, HITL, etc.) +live in [agentspan-design.md](agentspan-design.md); this doc is the SDK + wire API only. See +also [sdk-design.md](sdk-design.md) (multi-language SDK surface), +[framework-integration.md](framework-integration.md) (framework-bridged agents), and +[tool-execution-and-credentials-design.md](tool-execution-and-credentials-design.md) +(runtime tool execution and credential resolution). + +--- + +## 1. The API contract + +Every SDK exposes the same conceptual surface, regardless of language: + +- **Agents** — a single `Agent(...)` constructor declares one agent (model, instructions, + tools, turn/token limits, guardrails) or a multi-agent group (`agents=[...]` + a + `strategy`). Agents nest recursively. +- **Tools** — small declarative factory functions (`tool` / `httpTool` / `mcpTool` / + `apiTool`) attach capabilities to an agent. Each produces a tool descriptor that serializes + into the `tools` array of `AgentConfig`. +- **Wire schema** — every SDK serializes the above into one JSON document, `AgentConfig`, + and POSTs it under the `agentConfig` key of the start/compile request. The server + deserializes it into its `AgentConfig` model and compiles it into a Conductor workflow. + +The wire schema is the contract that makes the SDKs interchangeable: a config emitted by the +Python SDK and one emitted by the Java SDK are the same document and compile identically. + +--- + +## 2. AgentConfig wire schema + +The canonical wire contract is **[`../sdk/java/docs/agent-schema.json`](../sdk/java/docs/agent-schema.json)** +(JSON Schema Draft 2020-12), documented in +[`../sdk/java/docs/agent-schema.md`](../sdk/java/docs/agent-schema.md). Treat that JSON as the +source of truth; the summary below is a guide, not a redefinition. + +**Conventions** + +- camelCase keys; absent = unset (server uses `@JsonInclude(NON_NULL)`). +- `additionalProperties: false` at the root — the schema is the *complete* set of recognized + top-level keys (intentionally stricter than the server, which ignores unknown keys). +- Recursive: `agents`, `planner`, `fallback`, and `router` each nest a full `AgentConfig` + (`$ref: "#"`). +- The schema describes **native** agent configs. Framework-bridged agents (`openai`, + `google_adk`, `skill`, …) take a different path — they are sent as an opaque `rawConfig` + under a `framework` key in the request wrapper and are out of scope here (see + [framework-integration.md](framework-integration.md)). + +**Top-level fields (selected).** `name` is the only required field. + +| Field | Type | Purpose | +|---|---|---| +| `name` | string (required) | Agent name (`^[a-zA-Z_][a-zA-Z0-9_-]*$`). | +| `model` | string\|null | `"provider/model"` identifier. Null/omitted for external agents. | +| `external` | boolean | True when the agent has no model and is driven externally. | +| `baseUrl` | string | Per-agent LLM provider endpoint override. | +| `instructions` | string\|object\|null | System prompt — plain string or a prompt-template ref. | +| `tools` | array→`tool` | Tool descriptors (see §3). | +| `agents` | array→`#` | Sub-agents (recursive); requires a `strategy`. | +| `strategy` | string\|null (enum) | Multi-agent orchestration; null for a single agent. | +| `router` | `#`\|`workerRef` | ROUTER strategy router (nested agent or worker task). | +| `guardrails` | array→`guardrail` | Input/output guardrails (see below). | +| `maxTurns` / `maxTokens` / `temperature` / `timeoutSeconds` | int/num | Run limits. | +| `reasoningEffort` | string (enum) | `minimal\|low\|medium\|high` — OpenAI reasoning models only. | +| `contextWindowBudget` | integer | Token threshold for proactive context condensation. | +| `thinkingConfig` / `memory` / `termination` / `outputType` | object | Extended-thinking, message memory, termination conditions, structured-output type. | +| `handoffs` / `allowedTransitions` | array / object | Handoff conditions; SWARM transition map. | +| `callbacks` | array→`callback` | Lifecycle callbacks (before/after agent/model/tool). | +| `gate` / `stopWhen` | object / workerRef | Sequential-pipeline gate; stop condition. | +| `enablePlanning` / `planner` / `fallback` / `fallbackMaxTurns` / `plannerContext` / `planSource` | mixed | PLAN_EXECUTE planning slots. | +| `requiredTools` / `prefillTools` | array | Force-call tools; prefilled tool calls. | +| `credentials` | array | Credential names to resolve for this agent. | +| `codeExecution` / `cliConfig` | object | Sandboxed code execution; CLI execution config. | +| `metadata` / `maskedFields` / `synthesize` / `stateful` / `includeContents` | mixed | Misc orchestration flags. | + +**Strategy enum:** `handoff`, `sequential`, `parallel`, `router`, `round_robin`, `random`, +`swarm`, `manual`, `plan_execute` (or `null` for a single agent). + +**Tool kinds.** A tool descriptor (`$defs.tool`) carries `name`, `description`, +`inputSchema`/`outputSchema`, a `toolType` discriminator, and a freeform `config` map for +type-specific settings. `toolType` is one of `worker | http | mcp | api | agent_tool | …` +(see §3 for the SDK conventions that produce each). The `tool` definition keeps +`additionalProperties: true` because `config` is freeform and the Java serializer may emit +extra retry fields (`retryCount`, `retryDelaySeconds`, `retryPolicy`). + +> **Note.** `toolType` is a free `string` in the schema (it has no `enum`), so any value +> validates. The schema's `toolType` *description* string now lists `api` explicitly +> (`"worker | http | mcp | api | agent_tool | …"`); `api` is the real wire literal emitted by +> `api_tool` (see §3 / §4.1). The description remains illustrative, not exhaustive. + +**Guardrails.** A guardrail (`$defs.guardrail`) has a `guardrailType` +(`regex | llm | custom | external | …`), a `position` (`input | output`), an `onFail` policy +(closed enum `retry | raise | fix | human`), and type-specific keys (`patterns`, `mode`, +`model`, `policy`, …). See [guardrails-design.md](guardrails-design.md). + +**Nested config models.** The schema defines 16 nested `$defs`: `promptTemplate`, `tool`, +`guardrail`, `termination`, `handoff`, `callback`, `memory`, `message`, `codeExecution`, +`cliConfig`, `thinkingConfig`, `prefillTool`, `plannerContextEntry`, `outputType`, `gate`, +`workerRef`. Most are `additionalProperties: false`; consult the JSON for exact fields. + +**Known cross-SDK divergences** (both forms validate against the schema): + +- **Static plan channel.** Python places the static plan in `agentConfig.planSource`; Java + sends it in the request wrapper as `static_plan`. +- **Session id channel.** Java echoes `sessionId` into `agentConfig` *and* the wrapper; + Python sends it only in the wrapper. The server reads it from the wrapper. +- **`stateful` / `localCodeExecution` / `cliConfig.workingDir`.** SDK-emitted extras the + server does not model on `AgentConfig` directly but the schema tolerates so SDK output + validates. + +--- + +## 3. Tool declaration API + +Tools are declared with small factory functions. Names below use the Python form; each SDK +mirrors the convention idiomatically (camelCase methods in Java/TS, etc.). All of them +produce a tool descriptor that lands in the `tools` array of `AgentConfig` with a `toolType` +discriminator and a `config` map. + +| SDK factory | `toolType` | Declares | Discovery | +|---|---|---|---| +| `tool` / `@tool` | `worker` | A native function/worker that runs in the SDK process. | Static — the function signature defines `inputSchema`. | +| `http_tool` | `http` | A single HTTP endpoint (name, URL, method, headers, input schema). | Static — you define the one endpoint. | +| `mcp_tool` | `mcp` | An MCP server; all its tools become agent tools. | Auto — discovered at workflow startup via `LIST_MCP_TOOLS`. | +| `api_tool` | `api` | An OpenAPI/Swagger spec, Postman collection, or base URL; all operations become agent tools. | Auto — discovered at workflow startup via `LIST_API_TOOLS` (see §4). | + +**Conventions shared across kinds** + +- **Credentials.** Headers may reference credentials with `${NAME}` placeholders; the + `credentials=[...]` list names which to resolve. Resolution happens server-side at runtime + (see [tool-execution-and-credentials-design.md](tool-execution-and-credentials-design.md)). +- **Uniform LLM view.** The model sees one flat tool list and cannot tell which tools are + native, HTTP, MCP-discovered, or API-discovered — they are all just callable tools. +- **Auto-discovered kinds** (`mcp_tool`, `api_tool`) support a `max_tools` cap; when the + discovered set exceeds it, a filter LLM selects the most relevant subset at startup. + +> **SDK availability.** `api_tool` now ships in **all four SDKs** (Python, TypeScript, Java, +> and C#) — Java added the factory, and C# gained the `${NAME}` credential-placeholder +> validation it previously lacked. All four tool factories (`tool` / `http_tool` / +> `mcp_tool` / `api_tool`) are available in every SDK. + +```python +from conductor.ai.agents import Agent, api_tool, http_tool, mcp_tool, tool + +@tool +def calculate(expression: str) -> dict: # toolType=worker (native) + return {"result": eval(expression)} + +weather = http_tool(name="getWeather", url="https://api.weather.com/now", method="GET") +github = mcp_tool(server_url="http://localhost:3001/mcp", credentials=["GITHUB_TOKEN"]) +stripe = api_tool(url="https://api.stripe.com/openapi.json", credentials=["STRIPE_KEY"]) + +agent = Agent(name="assistant", model="openai/gpt-4o", + tools=[calculate, weather, github, stripe]) +``` + +--- + +## 4. `api_tool` — auto-discovery from OpenAPI / Swagger / Postman + +`api_tool()` points at an OpenAPI spec, Swagger spec, Postman collection, or bare base URL +and automatically discovers every API operation as an agent tool. It mirrors the `mcp_tool` +pattern (discover at startup, filter with an LLM if too many, execute as standard HTTP tasks), +removing the need to hand-define dozens or hundreds of endpoints with `http_tool`. + +### 4.1 SDK API + +```python +from conductor.ai.agents import api_tool + +# OpenAPI 3.x spec +stripe = api_tool( + url="https://api.stripe.com/openapi.json", + headers={"Authorization": "Bearer ${STRIPE_KEY}"}, + credentials=["STRIPE_KEY"], + max_tools=20, +) + +# Swagger 2.0 spec +legacy = api_tool(url="https://petstore.swagger.io/v2/swagger.json", max_tools=10) + +# Postman collection +slack = api_tool( + url="https://api.getpostman.com/collections/12345", + headers={"Authorization": "Bearer ${SLACK_TOKEN}"}, + credentials=["SLACK_TOKEN"], +) + +# Base URL — auto-discovers spec at known paths +weather = api_tool(url="https://api.weather.com", + tool_names=["getCurrentWeather", "getForecast"]) +``` + +**Parameters** + +| Parameter | Type | Default | Description | +|---|---|---|---| +| `url` | str | required | URL to OpenAPI spec, Postman collection, or base URL. | +| `name` | str | None | Override name (default: from spec `info.title`). | +| `description` | str | None | Override description (default: from spec `info.description`). | +| `headers` | dict | None | Global headers applied to ALL discovered endpoints. | +| `credentials` | list | None | Credential names for `${NAME}` header substitution. | +| `tool_names` | list | None | Whitelist — only include these operation IDs. | +| `max_tools` | int | 64 | If operations exceed this, a filter LLM selects the most relevant. | + +**Serialization** — produces a tool descriptor with `toolType: "api"` and a `config` map: + +```json +{ + "name": "stripe_api", + "description": "Stripe payment API", + "toolType": "api", + "inputSchema": null, + "config": { + "url": "https://api.stripe.com/openapi.json", + "headers": {"Authorization": "Bearer ${STRIPE_KEY}"}, + "tool_names": null, + "max_tools": 20, + "credentials": ["STRIPE_KEY"] + } +} +``` + +### 4.2 Server-side discovery: `LIST_API_TOOLS` + +A new Conductor system task, inserted before the agent loop (same position as +`LIST_MCP_TOOLS`). It HTTP-GETs the spec URL with the resolved headers, auto-detects the +format, parses operations, and returns normalized tool descriptors plus the base URL. + +**Format auto-detection** + +| Signal | Format | +|---|---| +| JSON with `"openapi"` field starting `"3."` | OpenAPI 3.x | +| JSON with `"swagger"` field `"2.0"` | Swagger 2.0 | +| JSON with `"info"."_postman_id"` or root `"item"` array | Postman Collection v2.1 | +| URL returns HTML or 404 | Base URL — try known spec paths | + +**Base URL auto-discovery** — tries, in order: +`{url}/openapi.json`, `{url}/swagger.json`, `{url}/v3/api-docs`, +`{url}/swagger/v1/swagger.json`, `{url}/api-docs`, `{url}/.well-known/openapi.json`. First +success wins; if none succeed, the task fails with a descriptive error. + +**Spec → tool mapping** + +- **OpenAPI 3.x:** `operationId` → `name` (fallback `{method}_{path_slug}`); `summary`/ + `description` → `description`; `parameters` (path/query/header) + `requestBody` → merged + `inputSchema`; `servers[0].url` + `path` → `baseUrl` + `path`; HTTP method → `method`. +- **Swagger 2.0:** as above, except `host` + `basePath` + `path` → `baseUrl` + `path`; + `in: body` parameters → request body schema; `consumes`/`produces` → content-type headers. +- **Postman:** `item[].name` → slugified `name`; `request.description` → `description`; + `request.url` → `baseUrl` + `path`; `request.method` → `method`; + `request.body.raw` (JSON Schema inferred) → `inputSchema`. Nested folders + (`item[].item[]`) flatten as `{folder}_{item}`. + +### 4.3 Compilation pipeline + +Reuses the MCP discovery chain (`ToolCompiler.java`): + +``` +Workflow Start +├─ LIST_MCP_TOOLS (for mcp_tool defs) ← existing +├─ LIST_API_TOOLS (for api_tool defs) ← NEW +├─ INLINE prepare task +│ - Merge MCP + API + static tools (http_tool, worker) +│ - Build mcpConfig (existing) + apiConfig: {toolName → {baseUrl, method, path, headers}} +│ - Check total_tools > maxTools +├─ SWITCH threshold (if exceeded) → filter LLM picks top N ← reused +├─ INLINE resolve task → {tools, mcpConfig, apiConfig} +└─ Agent Loop (LLM sees one unified tool list) +``` + +`apiConfig` is keyed by tool name, each entry `{baseUrl, method, path, headers}` with +credentials already resolved into the headers. + +### 4.4 Tool enrichment & execution + +API tools execute as standard Conductor **`HTTP`** tasks — there is no new execution task +type, only `LIST_API_TOOLS` for discovery. At enrichment time (`enrichToolsScript` in +`JavaScriptBuilder.java`), an `apiCfg[toolName]` entry is routed by: + +- Substituting path params into the URI template (`/users/{id}` → `/users/123`); consumed + params are removed from the body. +- For `GET`/`DELETE`/`HEAD`: remaining params become the query string. +- For `POST`/`PUT`/`PATCH`: remaining params become the JSON body. +- Merging `header` params and the global `headers` into the request headers. + +| OpenAPI `in` | Enrichment behavior | +|---|---| +| `path` | Substituted into the URI template. | +| `query` | Query string for GET/DELETE/HEAD. | +| `header` | Merged into request headers. | +| `body` / `requestBody` | JSON body for POST/PUT/PATCH. | + +Enriched runtime task: + +```json +{ + "type": "HTTP", + "taskReferenceName": "tool_createCustomer_0", + "inputParameters": { + "http_request": { + "uri": "https://api.stripe.com/v1/customers", + "method": "POST", + "headers": {"Authorization": "Bearer sk-resolved-key"}, + "body": {"email": "user@example.com", "name": "Alice"}, + "accept": "application/json", + "contentType": "application/json" + } + } +} +``` + +### 4.5 Error handling + +| Error | Behavior | +|---|---| +| Spec URL unreachable | `LIST_API_TOOLS` fails → workflow fails with descriptive error. | +| Invalid/undetectable format | Fail: "Could not detect format at {url}". | +| Base URL — no spec at any known path | Fail: "No OpenAPI/Swagger spec found at {url}". | +| Spec parses but 0 operations | Warning logged; empty tools list (agent works with other tools). | +| Credential resolution fails | Task fails with `CredentialNotFoundError`. | +| Filter LLM fails (max_tools exceeded) | Fallback: use all tools (log warning). | + +--- + +## 5. Agent model conventions + +The `model` field is a `"provider/model"` string (e.g. `"openai/gpt-4o"`, +`"anthropic/claude-sonnet-4-5"`). Null/omitted marks an **external** agent driven outside the +server. Beyond standard providers, the SDK supports a **Claude Code** convention that lets +Claude Agent SDK agents use the same `Agent(...)` interface as native agents — so they +compose as sub-agents, participate in handoffs, and work with sequential/parallel/router +strategies. + +> **SDK availability.** The Claude Code convention — the `ClaudeCode` config object and +> `Agent(model="claude-code"|"claude-code/...")` — exists **only in the Python and TypeScript +> SDKs**. It is **not available in the Java or C# SDKs**. Everything in §5 is scoped to those +> two SDKs. + +### 5.1 `Agent(model="claude-code")` + +```python +from conductor.ai.agents import Agent, ClaudeCode + +# Slash syntax (alias resolved to a full model ID) +reviewer = Agent( + name="reviewer", + model="claude-code/opus", + instructions="Review Python code for quality and security", + tools=["Read", "Glob", "Grep"], + max_turns=10, +) + +# Default model (CLI default) +reviewer = Agent(name="reviewer", model="claude-code", instructions="...", tools=["Read"]) + +# Config object for permission_mode +reviewer = Agent( + name="reviewer", + model=ClaudeCode("opus", permission_mode=ClaudeCode.PermissionMode.ACCEPT_EDITS), + instructions="Review code", + tools=["Read", "Edit", "Bash"], + max_turns=10, +) + +# Composition — a native orchestrator with Claude Code sub-agents +pipeline = Agent(name="pipeline", model="anthropic/claude-sonnet-4-5", + agents=[reviewer, writer, tester], strategy="sequential") +``` + +**`ClaudeCode` config** carries a minimal surface — model name + permission mode only: + +```python +@dataclass +class ClaudeCode: + class PermissionMode(str, Enum): + DEFAULT = "default" + ACCEPT_EDITS = "acceptEdits" + PLAN = "plan" + BYPASS = "bypassPermissions" + + model_name: str = "" # "opus"/"sonnet"/"haiku"/full ID; "" = CLI default + permission_mode: PermissionMode = PermissionMode.ACCEPT_EDITS +``` + +No `mcp_servers` and no `hooks` on this config: agentspan injects observability hooks +internally, and the `ClaudeCodeOptions` escape hatch remains for power users who need raw +MCP, hooks, etc. **Phase 1 supports only string (Claude built-in) tools** — passing a custom +`@tool` callable to a `claude-code` agent raises `ValueError`. (Phase 2 will add an MCP bridge +that auto-converts `@tool` functions to MCP servers.) + +### 5.2 Model alias resolution + +| Input | Resolved model | +|---|---| +| `"claude-code"` | `None` (CLI default) | +| `"claude-code/opus"` | `"claude-opus-4-6"` | +| `"claude-code/sonnet"` | `"claude-sonnet-4-6"` | +| `"claude-code/haiku"` | `"claude-haiku-4-5"` | +| `"claude-code/claude-opus-4-6"` | `"claude-opus-4-6"` (passthrough) | +| `ClaudeCode("opus")` | `"claude-opus-4-6"` | +| `ClaudeCode()` | `None` (CLI default) | + +Short aliases map to full model IDs via a dict lookup; unknown aliases pass through as-is. + +### 5.3 Where the config lives (architecture) + +**The server only ever sees a passthrough stub for a `claude-code` agent.** All real +configuration — instructions, tools, `max_turns`, `permission_mode` — is consumed locally in +the SDK worker closure, not serialized to JSON. The server's role is to create a minimal +workflow with a single SIMPLE task; the worker does the rest. + +- Serialization emits a minimal `{name, _worker_name}` raw_config (identical for an `Agent` + and for a raw `ClaudeCodeOptions`). +- The worker builder converts `Agent(model="claude-code/...")` → a `ClaudeCodeOptions` + dataclass (`agent_to_claude_code_options()`) before invoking the worker. This conversion is + **load-bearing**: the worker calls `dataclasses.replace(options, hooks=...)` to merge + observability hooks, which would crash on a non-dataclass (e.g. an `Agent`). +- **Routing is NOT by framework passthrough.** A native `Agent` is *always* serialized + natively — even when its `model` is `"claude-code"` / `"claude-code/..."`. `detect_framework()` + returns `None` for any native `Agent` instance (serializer.py: "Native Agent instances are + always native, even with claude-code models. The server handles claude-code model routing + during execution."). `detect_framework()` returns `"claude_agent_sdk"` **only** for a raw + `ClaudeCodeOptions` / `ClaudeAgentOptions` object (the escape hatch), not for an `Agent`. The + server routes a `claude-code`-model `Agent` server-side **by model**, not via the framework + passthrough path. + +**Sub-agent composition** requires three coordinated pieces so a `claude-code` agent can sit +inside `agents=[...]`: + +1. **Worker prep** — when recursing into sub-agents, detect a `claude-code` sub-agent and + register a passthrough worker instead of recursing into its (string) tools. +2. **Config serialization** — emit passthrough metadata for the sub-agent + (`metadata._framework_passthrough = true`, a single `worker`-type tool entry; do *not* + serialize instructions/tools), matching the shape the framework normalizer produces. +3. **Server compile** — `AgentCompiler` detects a `claude-code` model prefix on a sub-agent + as a safety net and forces the passthrough compilation path even if metadata was missing. + +This convention extends to the framework-bridge machinery documented in +[framework-integration.md](framework-integration.md); the `ClaudeCodeOptions` escape hatch +(`runtime.run(ClaudeCodeOptions(...))`) continues to work unchanged. diff --git a/design/framework-integration.md b/design/framework-integration.md new file mode 100644 index 000000000..93724b229 --- /dev/null +++ b/design/framework-integration.md @@ -0,0 +1,682 @@ +# Framework Integration + +**Status:** Consolidated 2026-06-26 + +**Scope.** This is the single canonical reference for running agents authored in third-party frameworks on the Agentspan platform. Framework graphs and agents become Conductor tasks: depending on what the SDK can introspect, a framework agent either decomposes into native server-side tasks (model + tool tasks, nodes/edges) or runs **passthrough** — the whole graph/agent executes inside one durable Conductor SIMPLE worker while pushing thinking/tool-call/tool-result events to the server non-blocking. Either way the user keeps their framework's authoring API and the call is always the same: `runtime.run(frameworkAgentOrGraph, prompt)`. This doc covers the passthrough execution model, the serialization reference for each framework (LangGraph being the definitive one), and the OCG retrieval integration. + +**Siblings.** Platform model: [agentspan-design.md](agentspan-design.md). SDK surface: [sdk-design.md](sdk-design.md). HTTP API: [api-design.md](api-design.md). Credential resolution and tool dispatch: [tool-execution-and-credentials-design.md](tool-execution-and-credentials-design.md). Per-SDK usage docs: [Python framework-agents.md](../sdk/python/docs/framework-agents.md), [TypeScript framework-agents.md](../sdk/typescript/docs/framework-agents.md). + +--- + +## 1. Scope and the passthrough execution model + +A framework agent reaches the server as a `raw_config` dict plus a set of worker closures. The server normalizes the `raw_config` into a canonical `AgentConfig` and compiles it into a Conductor `WorkflowDef`. Two broad outcomes: + +- **Decomposed** — the SDK introspects the agent and the server compiles native tasks: an AI_MODEL agentic loop with one SIMPLE task per tool, or a node/edge workflow of typed tasks. The server controls model selection, tool dispatch, retries, and step-level orchestration. +- **Passthrough** — the SDK cannot (or should not) decompose the agent, so the entire framework runtime runs inside one SIMPLE worker. The server sees a single durable task; the worker forwards events so observability and durability still apply, but step-level orchestration does not. + +The detection rule is per-framework (see each section), and the exact mechanism differs by SDK — but the shared property is: **no framework is imported by Agentspan, and framework packages are optional peer dependencies.** In the **Python** SDK (`serializer.py`), LangGraph/LangChain/Claude are detected by **type-name** checks (`type(obj).__name__` in `{CompiledStateGraph, Pregel, CompiledGraph}`, `AgentExecutor`, `{ClaudeCodeOptions, ClaudeAgentOptions}`) and OpenAI/ADK by **module-prefix** matching (`_FRAMEWORK_DETECTION = {"agents": "openai", "google.adk": "google_adk"}`). Only the **TypeScript** SDK uses **duck-typed marker** detection (`detect.ts`) for OpenAI/ADK. Whichever path is chosen, events are pushed non-blocking from the worker to the server so the calling code never waits on instrumentation. + +| Framework | SDK availability | Primary path | Falls back to | +|---|---|---|---| +| OpenAI Agents SDK | Python & TS | Full extraction (AI_MODEL + tools) | — | +| LangGraph | Python & TS | Full extraction / graph-structure | Passthrough | +| LangChain (`create_agent`) | Python & TS | Full extraction via LangGraph | Passthrough (legacy `AgentExecutor`) | +| Google ADK | Python & TS | Full extraction (AI_MODEL + tools / orchestration agents) | — | +| Claude Agent SDK | **Python only** | Passthrough with injected per-tool/subagent tasks | — | +| Vercel AI SDK | **TypeScript only** | Full extraction (`tool()` → native tool defs) | — | +| OCG retrieval | **Python only** | HTTP tasks (SDK-baked) | — | + +> **Per-SDK availability.** LangGraph, LangChain, OpenAI Agents SDK, and Google ADK are bridged in **both** the Python and TypeScript SDKs. The **Claude Agent SDK** framework bridge is **Python only** (TypeScript exposes only the native `ClaudeCode` *model* on a native Agent — no Claude framework bridge). The **Vercel AI SDK** bridge is **TypeScript only**. **OCG** retrieval is **Python only**. + +--- + +## 2. Common passthrough architecture + +All passthrough bridges share the same shape, so it is documented once here and referenced from each framework section. + +**Single durable task.** The server's `compileFrameworkPassthrough()` produces a `WorkflowDef` with a single SIMPLE task. The normalizer sets `metadata._framework_passthrough = true` and emits one `ToolConfig` with `toolType = "worker"`. The task receives `prompt`, `session_id`, `media`, and `cwd` and hands them to the worker. + +**Worker closure, not JSON.** Framework objects often contain callables (hooks, custom tools, compiled graphs) that cannot be JSON-serialized. The SDK therefore keeps the object in a worker closure and sends only a minimal `raw_config = {name, _worker_name}` to the server. `_build_passthrough_func()` builds the worker per framework; `_register_passthrough_worker()` registers it as a Conductor task def (default 600s timeout). + +**Callback handlers → SSE events.** Each framework exposes an instrumentation hook (LangChain/LangGraph callback handler, Claude Agent SDK hooks, etc.). Agentspan attaches its own handler that maps framework events to Agentspan stream events and pushes them via fire-and-forget HTTP `POST /api/agent/events/{executionId}` using a module-level `ThreadPoolExecutor(max_workers=4)`. User-supplied handlers/hooks are preserved and run first; Agentspan handlers are additive and defensive (try/except — instrumentation must never crash the agent). Typical event types: `tool_call`, `tool_result`, `tool_error`, `thinking`, `subagent_start`/`subagent_stop`, `notification`, `agent_stop`. + +**Credential injection contract.** The passthrough worker resolves execution-level credentials from the `_workflow_credentials` registry by execution token, injects them into `os.environ` before running, and removes them in a `finally` block. This is the same contract every tool worker follows — see [tool-execution-and-credentials-design.md](tool-execution-and-credentials-design.md) for the full credential resolution model (names never leave the server, placeholders resolved at dispatch). Credentials are declared per-agent (`credentials=[...]`) or per-run. + +**Async in a sync worker.** Conductor workers are sync functions in `ThreadPoolExecutor` threads. Async frameworks (Claude Agent SDK, OpenAI streaming) are driven with `asyncio.run(...)`, which creates a fresh event loop — safe because worker threads have no existing loop. Known limitation: this does not work from inside an already-running loop (e.g. Jupyter); workaround is `nest_asyncio` or a separate thread. + +--- + +## 3. LangGraph (definitive serialization reference) + +Agentspan compiles LangGraph `StateGraph` and `create_react_agent`/`create_agent` graphs into Conductor workflow definitions. Three phases: + +1. **Serialization** (Python SDK) — introspect the graph, extract nodes/edges/tools, produce a `raw_config` dict + worker functions. +2. **Normalization** (Server) — convert `raw_config` into a canonical `AgentConfig`. +3. **Compilation** (Server) — transform the `AgentConfig` into a Conductor `WorkflowDef` with typed tasks. + +The serializer chooses one of three paths automatically based on graph structure: + +| Path | When | Conductor Pattern | +|------|------|-------------------| +| **Full extraction** | `create_agent`/`create_react_agent` (with or without tools) | AI_MODEL + SIMPLE per tool | +| **Graph-structure** | Custom `StateGraph` with detectable model | Node/edge workflow with typed tasks | +| **Passthrough** | Fallback (no model found, multi-arg nodes) | Single SIMPLE task running graph locally (see §2) | + +### 3.1 Serialization paths + +#### Path 1: Full Extraction + +**Trigger:** Model found in graph — either with tools (ToolNode) or without (pure LLM call). Covers both `create_react_agent` with tools and `create_agent` with no tools. + +The serializer: +1. Finds the LLM object via `_find_model_in_graph()` — walks `graph.nodes` for objects with `model_name` / `model` attributes. +2. Infers the provider from the class name (ChatOpenAI → `openai`, ChatAnthropic → `anthropic`, etc.). +3. Finds tools via `_find_tools_in_graph()` — searches nodes for a `tools_by_name` dict (ToolNode pattern). +4. For each tool, extracts name, description, JSON schema, and callable. +5. Extracts the system prompt via `_extract_system_prompt_from_graph()` — walks node closures for `system_message` (set by `create_agent`'s `system_prompt`). +6. Registers one worker per tool (may be zero for pure LLM agents). + +```python +raw_config = { + "name": "my_agent", + "model": "anthropic/claude-sonnet-4-6", + "instructions": "You are a helpful pirate.", # from system_prompt param, if present + "tools": [ + {"_worker_ref": "search", "description": "Search the web", "parameters": {...}}, + ] +} +``` + +**Conductor result:** compiled as an AI_MODEL task (agentic loop with tool calling) — identical to OpenAI agents. With no tools, the AI_MODEL task runs a single LLM call with the system prompt and user message. + +#### Path 2: Graph-Structure + +**Trigger:** Model found BUT no ToolNode tools (custom StateGraph with explicit nodes/edges). + +The serializer introspects the compiled graph to extract: +- **Nodes**: function references from `graph.nodes`. +- **Edges**: `(source, target)` from `graph.builder.edges`. +- **Conditional edges**: `(source, router_func, target_map, is_dynamic)` from `graph.builder.branches`. +- **State reducers**: from `graph.channels` (e.g. `Annotated[list, operator.add]`). +- **Retry policies**: per-node metadata from `graph.builder._nodes`. +- **Recursion limit**: from `graph.config` or default 25. + +Each node is classified as: **regular** (plain function → SIMPLE worker), **LLM node** (uses a detected LLM variable → prep + finish workers), or **human** (`@human_task` → Conductor HUMAN task). + +```python +raw_config = { + "name": "my_workflow", + "model": "anthropic/claude-sonnet-4-6", + "_graph": { + "nodes": [ + {"name": "fetch", "_worker_ref": "my_workflow_fetch"}, + {"name": "analyze", "_llm_node": True, + "_llm_prep_ref": "my_workflow_analyze_prep", + "_llm_finish_ref": "my_workflow_analyze_finish"}, + {"name": "review", "_human_node": True, "_human_prompt": "Review the analysis"}, + ], + "edges": [{"source": "fetch", "target": "analyze"}], + "conditional_edges": [ + {"source": "review", "_router_ref": "my_workflow_review_router", + "targets": {"approve": "__end__", "revise": "analyze"}} + ], + "_reducers": {"results": "add"}, + "_retry_policies": {"fetch": {"max_attempts": 3}}, + "_recursion_limit": 25 + } +} +``` + +#### Path 3: Passthrough + +**Trigger:** No model detected in the graph (introspection cannot find the LLM object). The entire graph runs inside one SIMPLE worker that calls `graph.stream(...)` locally and forwards thinking/tool_call/tool_result events as SSE (see §2). + +```python +raw_config = {"name": "my_agent", "_worker_name": "my_agent"} +``` + +### 3.2 Feature-by-feature translation + +#### Sequential nodes + +```python +graph.add_edge("node_a", "node_b") +graph.add_edge("node_b", "node_c") +``` + +Sequential SIMPLE tasks; state threaded via Conductor expressions (`${node_a.output.state}` → node_b input). + +#### LLM nodes (server-side LLM calls) + +```python +def analyze(state): + response = llm.invoke([SystemMessage(...), HumanMessage(state["text"])]) + return {"analysis": response.content} +``` + +Three-task pipeline with a conditional bypass: + +``` +prep (SIMPLE) + → SWITCH(_skip_llm) + case "true": INLINE passthrough (pre-computed result) + default: LLM_CHAT_COMPLETE → finish (SIMPLE) + → coalesce (INLINE) +``` + +1. **Prep worker** swaps the module-level `llm` for a `_LLMCaptureProxy`. When the function calls `llm.invoke(messages)`, the proxy raises `_CapturedLLMCall`, intercepting the messages without an API call. The worker serializes and returns them. +2. **LLM_CHAT_COMPLETE** is a native Conductor system task that calls the provider server-side with the captured messages (server controls model selection, rate limiting, cost tracking). +3. **Finish worker** swaps `llm` for a `_LLMMockProxy` returning the server's response; the function runs to completion, producing the state update as if the call happened normally. +4. **Conditional bypass (SWITCH):** if the function completes *without* calling `llm.invoke()` (e.g. early return), prep sets `_skip_llm: true` and returns the pre-computed result; the SWITCH skips the LLM task. + +Thread safety: all LLM variable swaps are protected by `_llm_intercept_lock`. + +#### Conditional routing + +```python +def route(state): + if state["sentiment"] == "positive": + return "celebrate" + return "console" + +graph.add_conditional_edges("analyze", route, {"celebrate": "celebrate", "console": "console"}) +``` + +``` +router (SIMPLE) → returns {decision: "celebrate", state: {...}} + → SWITCH (value-param evaluator on decision) + case "celebrate": celebrate_tasks... + case "console": console_tasks... + → coalesce (INLINE) — unifies branch outputs +``` + +#### Parallel branches (FORK_JOIN) + +```python +graph.add_edge(START, "pros") +graph.add_edge(START, "cons") +graph.add_edge("pros", "merge") +graph.add_edge("cons", "merge") +``` + +``` +FORK_JOIN + ├─ branch 0: pros_tasks... + └─ branch 1: cons_tasks... +JOIN (waits for both) + → INLINE merge (reducer-aware state combination) +``` + +State merge: fields with `Annotated[list, operator.add]` are concatenated across branches; all other fields are last-write-wins. + +#### Dynamic fan-out (Send API / FORK_JOIN_DYNAMIC) + +```python +from langgraph.types import Send + +def fan_out(state): + return [Send("summarize", {"document": doc}) for doc in state["documents"]] + +graph.add_conditional_edges("generate", fan_out, ["summarize"]) +``` + +``` +router (SIMPLE) → returns {dynamic_tasks: [{node: "summarize", input: {...}}, ...]} + → INLINE enrich → Conductor FORK_JOIN_DYNAMIC format + → FORK_JOIN_DYNAMIC (N parallel SIMPLE tasks at runtime) + → JOIN + → INLINE merge (reducer-aware, iterates over join output keys) +``` + +**Detection:** the serializer inspects the routing function's bytecode (`co_names`) for `Send` references; the router worker checks for a list of objects with `.node`/`.arg`. The enrich INLINE maps each `node` to its worker ref and builds the Conductor task format; merge handles a runtime-determined branch count. + +#### Cycles and loops (DO_WHILE) + +```python +def should_continue(state): + if state["iterations"] < 3: + return "refine" # back-edge → cycle + return "__end__" # exit + +graph.add_conditional_edges("refine", should_continue, {"refine": "refine", "__end__": END}) +``` + +``` +DO_WHILE + condition: iteration < recursion_limit AND decision in back_edges + body: + state_bridge (INLINE) — iter 1 uses pre-loop state, later iters use router output + ...loop body tasks... + router (SIMPLE) — evaluates continue/exit +``` + +**Cycle detection:** during topological traversal, a conditional edge target already visited is a back-edge; tasks between cycle start and the router form the loop body. **State bridge:** selects pre-loop state on iteration 1, router output on iteration 2+. **Recursion limit:** LangGraph `recursion_limit` (default 25) → DO_WHILE iteration cap. + +#### Human-in-the-loop + +```python +from conductor.ai.agents.frameworks.langgraph import human_task + +@human_task(prompt="Review the draft and provide verdict + feedback.") +def review(state): + pass +``` + +``` +HUMAN task (pauses, waits for external input via API/UI) + → validation (INLINE) + → normalization (INLINE) + → process (SIMPLE) — merges human input into state +``` + +The decorator marks the function with `_agentspan_human_task = True`. No worker is registered for human nodes; the Conductor HUMAN task type handles input natively, and the server auto-generates the response form schema from the workflow context and prompt. See [agentspan-design.md](agentspan-design.md). + +#### State reducers + +```python +class State(TypedDict): + results: Annotated[list, operator.add] # concatenate across branches + topic: str # last-write-wins +``` + +The serializer inspects `graph.channels` for `BinaryOperatorAggregate` and maps `operator.add` → `"add"`. Applied in every FORK_JOIN / FORK_JOIN_DYNAMIC merge INLINE: + +```javascript +for (var k in branch_state) { + if (k === 'results') { + merged[k] = (merged[k] || []).concat(branch_state[k]); // "add" reducer + } else { + merged[k] = branch_state[k]; // last-write-wins + } +} +``` + +#### Retry policies + +```python +graph.add_node("fetch", fetch_data, retry=RetryPolicy(max_attempts=3, initial_interval=1.0)) +``` + +- `max_attempts` → `retryCount` (minus 1; Conductor counts retries not attempts) +- `initial_interval` → `retryDelaySeconds` +- `backoff_factor` → `backoffScaleFactor` +- `max_interval` → capped via backoff calculation + +#### Agent-as-tool (SUB_WORKFLOW) + +```python +from conductor.ai.agents.tool import AgentTool + +research_tool = AgentTool(name="researcher", agent=research_graph, description="Research a topic") +main_graph = create_react_agent(llm, tools=[calculator, research_tool]) +``` + +The child agent is recursively compiled into its own workflow def; the parent invokes it as a SUB_WORKFLOW task. `LangGraphNormalizer` detects `AgentTool` (via `_type: "AgentTool"`) and recursively calls `normalize()` on the embedded config. + +#### Subgraphs + +```python +inner_compiled = inner.compile() + +def run_inner(state): + result = inner_compiled.invoke({"text": state["analysis_text"]}) + return {"sentiment": result["sentiment"], ...} + +outer.add_node("analysis", run_inner) +``` + +Compiled as `SUB_WORKFLOW` with the same intercept pattern as LLM nodes: + +1. **Detection:** `_find_subgraph_in_func()` checks node bytecode (`co_names`) against globals for `CompiledStateGraph` objects. +2. **Serialization:** subgraph recursively serialized via `_serialize_graph_structure()` with a `{parent}_{node}` name prefix. +3. **Prep worker:** runs the node with `_SubgraphCaptureProxy`, capturing the `.invoke()` input. +4. **SUB_WORKFLOW:** server compiles the subgraph into a nested `WorkflowDef`, receiving `state` directly via `${workflow.input.state}` and returning both `state` and `result`. +5. **Finish worker:** runs the node with `_SubgraphMockProxy` (returns the SUB_WORKFLOW output state), producing the parent state update. +6. **SWITCH for skip:** handles the case where the function completes without calling `subgraph.invoke()`. + +``` +prep SIMPLE → SWITCH(_skip_subgraph) → [passthrough INLINE | SUB_WORKFLOW → finish SIMPLE] → coalesce INLINE +``` + +Subgraph workflows differ from regular graph-structure workflows: input is `${workflow.input.state}` (full state dict), output includes `state` alongside `result`, and the `_graph` metadata is marked `_is_subgraph: true`. + +#### State reconstitution + +Conductor's JSON serialization loses type information. `_reconstitute_state()` runs before every worker: +- **LangChain Documents:** dicts with a `page_content` key → `Document(page_content=..., metadata=...)`. +- **Stringified dicts:** a single string field containing a dict literal (e.g. `str(state)` used as the prompt) is parsed back via `ast.literal_eval`. + +### 3.3 Conductor construct mapping + +| LangGraph Concept | Conductor Task Type | Notes | +|---|---|---| +| Node function | SIMPLE | Worker polls and executes | +| LLM call in node | Prep (SIMPLE) → LLM_CHAT_COMPLETE → Finish (SIMPLE) | Server-side LLM with conditional bypass | +| `add_edge(a, b)` | Sequential task ordering | State threaded via `${ref.output.state}` | +| `add_conditional_edges` | Router (SIMPLE) → SWITCH | Value-param evaluator | +| Parallel from START | FORK_JOIN → JOIN → INLINE merge | Reducer-aware | +| `Send()` API | FORK_JOIN_DYNAMIC → JOIN → INLINE merge | Runtime-determined parallelism | +| Cycles (back-edges) | DO_WHILE + state bridge | Iteration cap from recursion_limit | +| `@human_task` | HUMAN system task | Pauses for external input | +| `AgentTool` | SUB_WORKFLOW | Recursive agent compilation | +| Subgraph `.invoke()` | Prep (SIMPLE) → SUB_WORKFLOW → Finish (SIMPLE) | Subgraph compiled as nested workflow | +| State reducers | INLINE merge JavaScript | `operator.add` → array concat | +| `RetryPolicy` | Task-level retry settings | max_attempts, backoff, interval | +| `create_agent`/`create_react_agent` | AI_MODEL agentic loop | Server-side LLM, with or without tools; system prompt from closure | +| Entire graph (fallback) | Single SIMPLE task | Passthrough: `graph.stream()` locally | + +### 3.4 Limitations and unsupported features + +This is the source of truth for LangGraph parity. + +#### Not supported + +| Feature | LangGraph API | Status | Notes | +|---------|--------------|--------|-------| +| `Command` construct | `Command(goto=..., update=...)` | Not implemented | Dynamic routing with state updates. Planned (Task #42). | +| Custom reducers | `Annotated[list, my_custom_fn]` | Warning logged | Only `operator.add` is mapped. Custom callables fall back to last-write-wins in FORK_JOIN merge (possible data loss). | +| Functional API | `@entrypoint`, `@task` | Not implemented | Different programming model entirely. | +| `CachePolicy` | `CachePolicy(ttl=...)` | Not implemented | No equivalent in Conductor task model. | +| Managed values | `RemainingSteps`, `IsLastStep` | Not implemented | Depend on LangGraph internal recursion tracking. | +| Private state channels | `PrivateAttr`, channel-level access control | Not implemented | Conductor state is a flat JSON dict. | +| `InputState` / `OutputState` distinction | Separate TypedDict for input vs output | Not implemented | Single state schema; no input validation / output filtering. | +| Time travel / replay | `get_state_history()`, replay from checkpoint | Not implemented | No checkpoint storage. | +| Cross-thread persistence | `BaseStore`, `InMemoryStore` | Not implemented | No cross-execution memory store. | +| `InjectedState` / `InjectedStore` | Tool parameter injection | Not implemented | Tools receive explicit inputs only. | +| `ValidationNode` | Built-in validation node type | Not implemented | Use regular nodes with validation logic. | +| Middleware | Request/response middleware hooks | Not implemented | No equivalent in Conductor. | +| Deferred nodes | `defer=True` | Not implemented | All nodes execute eagerly. | +| LangGraph Platform features | Cron jobs, double texting, assistants API | Not applicable | LangGraph Cloud features, not graph features. | +| CompiledStateGraph as tool parameter | Passing a graph object directly as a tool | Not supported | `ToolNode` rejects non-callable tools. Wrap in a `@tool` that calls `.invoke()`. | +| Server-side token streaming | Real-time token streaming from LLM nodes | Not supported | `LLM_CHAT_COMPLETE` returns the full response. | + +#### Passthrough only (local execution) + +These run, but the whole graph executes inside one SIMPLE worker — the server has no per-node visibility, cannot control LLM calls, and cannot orchestrate steps. See §2. + +| Feature | Why Passthrough | What Triggers It | +|---------|----------------|------------------| +| Graphs where no model can be detected | Serializer can't find LLM object via introspection | No object with `model_name`/`model` attribute in graph nodes or globals | +| Nodes with >1 positional arg in custom StateGraphs | Cannot run as standalone SIMPLE workers | Function signature like `(state, config)` | + +**Previously passthrough, now server-side:** `create_agent` graphs (with or without tools/system prompt) are now detected as full extraction — the model is extracted from graph nodes and the system prompt from `model_node`'s closure (`system_message` free variable). + +#### Known limitations of supported features + +**Bytecode inspection for detection (LLM, subgraph, Send API).** Detection relies on CPython bytecode (`func.__code__.co_names` + `func.__globals__`). It breaks with: aliased imports (`ChatOpenAI as MyLLM`), variables captured in closures, decorators that replace `__code__`, and non-CPython runtimes (PyPy, GraalPy). Mitigation: use straightforward module-level LLM/subgraph variable assignments; avoid aliasing or wrapping. + +**Global variable mutation for LLM/subgraph interception.** Prep/finish workers swap module-level globals with proxies under a process-wide lock (`_llm_intercept_lock`): one node function at a time per process; functions sharing an LLM variable share the lock; on error the `finally` restores the original, but a brief window exists where another thread could see the proxy. Safe for the current single-threaded worker model; would need redesign for concurrent execution. + +**State reducers.** Only `operator.add` maps to array concat. Other fields use last-write-wins. Custom reducer callables are detected and a warning logged, but the server cannot run arbitrary Python in JavaScript INLINE tasks. + +**Retry policies.** `max_attempts`, `initial_interval`, `backoff_factor` are mapped. `max_interval` (backoff is unbounded) and `jitter` are not mapped and log a warning. + +**Multiple conditional edges from the same source node.** Targets are merged but the last router function wins (Conductor SWITCH evaluates one decision per node); a warning is logged. + +**Result extraction heuristic.** The workflow extracts "result" from the final node's state using `result`, `final_report`, `output` in that order. A different field name yields empty workflow output (full state is always available via the state output). + +**INLINE JavaScript in Conductor tasks.** Merge/bridge/coalesce/enrich logic runs as GraalJS, string-concatenated in Java with no compile-time validation; covered by integration tests, no unit tests for the generated JavaScript. + +### 3.5 Data flow + +``` +User Code Python SDK Server +───────── ────────── ────── +StateGraph / create_agent serialize_langgraph() + │ ├─ Introspect graph + │ ├─ Extract nodes/edges + │ ├─ Build worker functions + │ ├─ Produce raw_config + │ AgentRuntime.run() + │ ├─ POST /agent/start ────────► LangGraphNormalizer.normalize() + │ │ (raw_config + framework) ├─ Detect path (full/graph/passthrough) + │ │ ├─ Build AgentConfig + │ │ AgentCompiler.compile() + │ │ ├─ Build Conductor WorkflowDef + │ │ ├─ Register workflow + │ │ └─ Start execution + │ ├─ Register workers ◄─ Conductor polls workers + │ ├─ Workers execute: + │ │ node_func / router_func / llm_prep/finish + │ ├─ Poll for completion + ◄────────────────────────────┤ Return result +``` + +--- + +## 4. LangChain (passthrough via LangGraph) + +Modern LangChain (v1.2+) uses `create_agent()` from `langchain.agents`, which returns a `CompiledStateGraph`. Agentspan detects this as a LangGraph object and routes it through the LangGraph pipeline (§3) — so LangChain agents get the same server-side LLM orchestration, tool extraction, and system-prompt support as native LangGraph agents. + +``` +create_agent(llm, tools=[...], system_prompt="...") + → CompiledStateGraph ──detect_framework()──► "langgraph" + → serialize_langgraph() + ├─ _find_model_in_graph() → "anthropic/claude-sonnet-4-6" + ├─ _find_tools_in_graph() → [tool1, tool2, ...] + └─ _extract_system_prompt_from_graph() → "You are a helpful assistant." + → Full Extraction raw_config: { name, model, instructions, tools: [...] } + → Server: LangGraphNormalizer → AgentCompiler → Conductor WorkflowDef (AI_MODEL) +``` + +| Path | When | Conductor Pattern | +|------|------|-------------------| +| **Full extraction (with tools)** | `create_agent(llm, tools=[...])` | AI_MODEL loop + SIMPLE per tool | +| **Full extraction (no tools)** | `create_agent(llm, tools=[])` | AI_MODEL single LLM call | +| **Passthrough** | Legacy `AgentExecutor` (if model/tools undetectable) | Single SIMPLE task running executor locally | + +### 4.1 Feature support + +- **System prompts** passed via `create_agent(llm, system_prompt="...")` are extracted from the `model_node` closure (`_extract_system_prompt_from_graph()` finds the `system_message` free variable) and sent as `instructions`. +- **Tools** — `@tool` functions and `StructuredTool` objects are extracted and registered as individual workers; the server orchestrates tool calling through the AI_MODEL loop. Name, description, and JSON schema (type hints or Pydantic `args_schema`) are included. +- **Structured output** — `with_structured_output()` works inside `@tool` functions; the structured LLM call runs locally within the tool worker while the outer loop is server-side. +- **Prompt templates** — `ChatPromptTemplate`/`PromptTemplate` work by formatting the system prompt before passing it to `create_agent`; the formatted string is sent as `instructions`. +- **Multi-turn** — handled via Agentspan session management; each `runtime.run()` call is independent (no checkpointer). + +Since `create_agent` returns a `CompiledStateGraph`, LangChain agents are a subset of the LangGraph integration — all §3 features and limitations apply. + +### 4.2 Legacy AgentExecutor support + +The `langchain.py` serializer handles legacy `AgentExecutor` objects two ways: +1. **Full extraction** — if model and tools are extractable (`executor.agent.llm`, `executor.tools`), delegates to the shared `_serialize_full_extraction()`. +2. **Passthrough** — fallback: the executor runs inside one SIMPLE worker with an `AgentspanCallbackHandler` streaming `tool_call`/`tool_result` events (see §2). `LangChainNormalizer` produces a passthrough `AgentConfig` with `_framework_passthrough: true`. + +Note: `AgentExecutor` is no longer importable from current LangChain (v1.2+). Use `create_agent`. + +### 4.3 LangChain-specific limitations + +Inherited: all §3.4 limitations (custom reducers, `Command`, functional API, time travel, cross-thread persistence). + +| Feature | Status | Notes | +|---------|--------|-------| +| `AgentExecutor` | Deprecated | No longer importable; use `create_agent`. | +| LCEL chains (non-agent) | Not supported | Only `CompiledStateGraph` is detected. Wrap plain LCEL (`prompt \| llm \| parser`) in a `@tool` or use inside `create_agent`. | +| `ConversationBufferMemory` | Not applicable | Legacy memory classes don't apply to `create_agent`; use tool-based memory. | +| LangServe | Not applicable | Agentspan replaces LangServe for deployment. | +| LangSmith tracing | Compatible | LangSmith callbacks work inside tool workers alongside `AgentspanCallbackHandler`. | + +--- + +## 5. OpenAI Agents SDK and Google ADK + +Both are first-class bridges that **decompose** to native server-side tasks, in **both the Python and TypeScript SDKs** via the generic serializer. No framework is imported by Agentspan, and the framework packages are optional peer dependencies. Detection differs by SDK: Python uses **module-prefix** matching (`agents` → openai, `google.adk` → google_adk in `_FRAMEWORK_DETECTION`); TypeScript uses **duck-typed marker** detection (`detect.ts`). See [Python framework-agents.md](../sdk/python/docs/framework-agents.md) and [TypeScript framework-agents.md](../sdk/typescript/docs/framework-agents.md) for full usage. + +### 5.1 OpenAI Agents SDK + +An `@openai/agents` (TS) / `agents` (Python) `Agent` is extracted into an AI_MODEL agentic loop plus one SIMPLE task per tool — identical to LangGraph full extraction. Detection (TS): `name` + string/function `instructions` + string `model` + `tools[]` + an OpenAI marker (`handoffs[]`, `inputGuardrails[]`, `asTool()`, `toolUseBehavior`, ...). + +Two authoring styles: +- **Drop-in `Runner`** (Python) — change one import to `from conductor.ai import Runner` and keep your existing `agents.Agent`. `Runner.run` / `run_sync` / `run_streamed` accept an OpenAI-Agents `Agent` or a native Agentspan `Agent`; `RunResult` exposes `.final_output` and `.execution_id` (`context` is accepted for compatibility and ignored). `from conductor.ai import function_tool` aliases `@tool`. +- **Pass to `runtime.run(...)`** (TS and Python) — hand the `Agent` straight to the runtime; same entry point as every other framework. + +### 5.2 Google ADK + +A `@google/adk` agent is bridged via the TypeScript SDK. Detection: `subAgents[]` (orchestration agents — `Sequential`/`Parallel`/`Loop`), or string `model` + ADK markers (`instruction`, `outputKey`, `generateContentConfig`, `beforeModelCallback`, ...). An `LlmAgent` extracts to an AI_MODEL loop + tool tasks; the orchestration agents map their structure onto Conductor tasks. Pass the agent straight to `runtime.run(...)`. + +```ts +import { LlmAgent } from '@google/adk'; +import { AgentRuntime } from '@conductor-oss/conductor-agent-sdk'; + +const agent = new LlmAgent({ name: 'greeter', model: 'gemini-2.5-flash', + instruction: 'You are a friendly assistant.' }); +const runtime = new AgentRuntime(); +const result = await runtime.run(agent, 'Say hello and a fun fact about ML.'); +``` + +> The TypeScript SDK additionally bridges the **Vercel AI SDK** (AI SDK `tool()` objects auto-convert to native tool defs; a drop-in `generateText`/`streamText` subpath builds an `Agent` under the hood). See [TypeScript framework-agents.md](../sdk/typescript/docs/framework-agents.md). + +--- + +## 6. Claude Agent SDK (passthrough by design) + +The Claude Agent SDK (PyPI package `claude-code-sdk`, imported as `claude_code_sdk`) is a full runtime — built-in tools (Read, Edit, Bash, ...), hooks, sessions, permissions. Extracting individual tools would lose most of its value, so Agentspan runs it **passthrough**: the full `query()` runs in one durable Conductor SIMPLE worker (the §2 passthrough architecture), instrumented through the SDK's hook system. This is **Python only** (TypeScript has only the native `ClaudeCode` *model* usable on a native Agent — there is no Claude framework bridge in TS). Users pass `ClaudeCodeOptions` / `ClaudeAgentOptions` (or use the native `ClaudeCode` model on a Agentspan `Agent`) to `runtime.run()` / `runtime.start()`. + +**Use cases:** (A) bring existing Claude Agent SDK agents in for durability/orchestration/observability; (C) invoke a Claude Agent SDK agent as a worker tool inside a larger Agentspan workflow. + +### 6.1 Execution model + +``` +runtime.run(options, prompt) + ├─ detect_framework() → "claude_agent_sdk" (type-name check accepting both ClaudeCodeOptions and ClaudeAgentOptions) + ├─ serialize_claude_agent_sdk(options) → (raw_config={name,_worker_name}, [WorkerInfo]) + ├─ _build_passthrough_func() → make_claude_agent_sdk_worker() (closure: options, server_url, auth) + ├─ _register_passthrough_worker() → Conductor task def (600s timeout) + └─ POST /api/agent/start {framework, rawConfig} + → ClaudeAgentSdkNormalizer → AgentConfig (_framework_passthrough=true) + → AgentCompiler.compileFrameworkPassthrough() → WorkflowDef (single SIMPLE task) + → start execution → Conductor → worker polls task: + 1. extract cwd from task input (set on options so file ops run in the right dir) + 2. inject execution credentials → os.environ (cleanup in finally) + 3. create metadata dict {tool_call_count, tool_error_count, subagent_count, tools_used} + 4. build agentspan hooks (close over metadata + execution_id) + 5. merge user hooks + agentspan hooks (user first) + 6. asyncio.run(_run_query(prompt, merged_options)) + └─ async for message in query(prompt, options): + ├─ hooks fire: PreToolUse, PostToolUse, SubagentStart, ... + │ ├─ push stream events: POST /api/agent/events/{executionId} + │ └─ mutate metadata + └─ collect ResultMessage → result text + token usage + 7. return TaskResult {result, tools_used, ...metadata, token_usage} +``` + +Although the agent runs in one durable worker, the Claude passthrough is **not opaque**: the worker dynamically injects child tasks as the run progresses, so the server gets per-tool and per-subagent visibility rather than a single black-box step. On `SubagentStart`, `_subagent_start` injects a SUB_WORKFLOW task for the subagent (`_create_tracking_workflow` builds the tracking workflow def); per-tool tracking tasks are injected via `_inject_tool_task`. These are added at runtime to the running execution (see `claude_agent_sdk.py`). + +### 6.2 Hooks (observability + metadata) + +All agentspan hooks are defensive (try/except) and return `{}` (no interference). User hooks run first; agentspan hooks are appended. Event delivery is fire-and-forget via the shared `ThreadPoolExecutor` (§2). + +| Hook Event | Stream Event | Metadata Mutation | +|---|---|---| +| `PreToolUse` | `{type: "tool_call", toolName, toolUseId}` | `tool_call_count += 1`, `tools_used.add(name)` | +| `PostToolUse` | `{type: "tool_result", toolName, toolUseId}` | — | +| `PostToolUseFailure` | `{type: "tool_error", toolName, error}` | `tool_error_count += 1` | +| `SubagentStart` | `{type: "subagent_start", agent_id}` | `subagent_count += 1` | +| `SubagentStop` | `{type: "subagent_stop", agent_id}` | — | +| `Notification` | `{type: "notification", message}` | — | +| `Stop` | `{type: "agent_stop"}` | — | + +The exact hook callback signature must be verified against the installed `claude-code-sdk` version (PyPI: `claude-code-sdk`, imported as `claude_code_sdk`; imports `query`, `ClaudeCodeOptions`, `AssistantMessage`, `ResultMessage`). The options object is kept in the worker closure, never JSON-serialized (it may contain callables). + +### 6.3 Components, design decisions, limitations + +| Component | File | +|---|---| +| Detection + serialize short-circuit | `sdk/python/src/conductor/ai/agents/frameworks/serializer.py` | +| Serializer, worker, hooks | `sdk/python/src/conductor/ai/agents/frameworks/claude_agent_sdk.py` (new) | +| `_build_passthrough_func()` branch | `sdk/python/src/conductor/ai/agents/runtime/runtime.py` | +| Passthrough normalizer | `server/.../normalizer/ClaudeAgentSdkNormalizer.java` (new) | + +Key decisions: passthrough over extraction (full runtime — extraction loses value); hooks for observability (exact instrumentation points, additive, defensive); `asyncio.run()` in the sync worker (fresh loop per worker thread); options in closure not JSON (callables); user hooks run first. + +**Use case C.** Phase 1 (ships with A): wrap the SDK in a Agentspan `@tool` that drives `query()` — works today, but no SUB_WORKFLOW and no inner-agent streaming. Phase 2 (follow-up): `runtime.register(options, name=...)` registers the agent by name for native handoffs (`HandoffCondition(target="claude_reviewer")`) with full SUB_WORKFLOW composition and streaming. + +**Limitations.** `asyncio.run()` fails inside an already-running loop (Jupyter) — use `nest_asyncio` or a separate thread. Phase 1 `@tool` produces no SUB_WORKFLOW / inner events. Hooks capture tool-level events but not individual LLM API calls (the SDK exposes no LLM-call hook). TypeScript support is a follow-up (Python first). + +--- + +## 7. OCG retrieval integration + +OCG (Open Context Graph) is a retrieval engine over a knowledge graph of entities — messages, channels, people, tickets — linked by claims and relationships. It is embedding/keyword search exposed as an HTTP API, **not** an LLM. + +The integration lives **entirely in the Python SDK** (`conductor.ai.agents.ocg`): the retrieval system prompt, tool schemas, endpoint routing, and instance binding. The tools compile to plain Conductor HTTP tasks, so **any Agentspan server runs them with zero OCG-specific configuration** — no properties, no task types. OCG is opt-in per agent; an agent that doesn't declare OCG tools never makes an OCG call. + +### 7.1 Two shapes + +**Sub-agent — delegate retrieval.** `ocg_agent()` returns an ordinary `Agent` carrying the canned retrieval prompt and the `ocg_*` tools. Wrap it with `agent_tool()` and the main agent's LLM sees a single tool; calling it runs the retriever as a sub-workflow with its own LLM loop, returning one synthesized, cited answer. The raw citations stay in the retriever's context; the main agent only sees the synthesized answer. Choose this when retrieval takes judgment (several queries, neighborhood walks, two-step aggregation). + +```python +from conductor.ai.agents import Agent, agent_tool +from conductor.ai.agents.ocg import ocg_agent + +retriever = ocg_agent( + model="anthropic/claude-sonnet-4-6", + url="https://test.contextgraph.io", + credential="OCG_PUBLIC_KEY", # secrets-store NAME, never the key +) +main = Agent( + name="support", model="openai/gpt-4o", + instructions="Call your retrieval tool exactly once with the user's full question; its answer is complete — write a concise cited brief.", + tools=[agent_tool(retriever)], max_turns=4, +) +``` + +**Direct tools — the main agent queries itself.** `ocg_tools()` returns the raw `ToolDef`s; attach them (or a subset) to your own agent and its LLM issues the queries directly — no sub-workflow hop, roughly half the tokens for simple lookups, but raw citations land in the main agent's context and the retrieval prompting is yours. + +```python +from conductor.ai.agents.ocg import ocg_tools + +main = Agent( + name="support", model="anthropic/claude-sonnet-4-6", + instructions="Answer using ocg_query (keyword/embedding retrieval, NOT an LLM). Query with specific keywords, at most one per topic, then write your brief.", + tools=ocg_tools(url="https://test.contextgraph.io", credential="OCG_PUBLIC_KEY", + entities=False, memory=False), # subset switches → ocg_query only + max_turns=6, +) +``` + +### 7.2 How a tool call executes + +There is no OCG code on the server. The SDK bakes everything the dispatch needs into each tool's config at definition time; the compiled workflow's **enrich script** (compile-time JavaScript, evaluated at dispatch) turns the LLM's arguments into a standard Conductor HTTP task. + +``` +SDK: ToolDef(tool_type="http", config={url, method, pathTemplate, queryParams, + headers:{Authorization:"Bearer ${OCG_PUBLIC_KEY}"}}) + → Compiler bakes config into workflow def (placeholder escaped for the host's resolver) + → LLM emits a tool call, e.g. ocg_get_entity(entity_id="entity_01...", depth=1) + → Enrich script: uri = url + pathTemplate filled from args (URL-encoded) + queryParams present in args; + body = remaining args (consumed args removed) + → HTTP task {uri, method, headers, body} + → Conductor resolves credential placeholder by NAME from secrets store (token in memory only) + → HTTPS request to OCG instance → JSON response → tool result for the LLM +``` + +Key properties: +- **Per-tool instance binding.** `url=` is required — every OCG tool set binds the instance it talks to. Different agents can target different graphs (e.g. a US retriever and a Canada retriever in one router agent); agents bound to different instances must have distinct `name`s. +- **Secrets never leave the server.** `credential="OCG_PUBLIC_KEY"` is a *name*; it compiles to a standard HTTP-tool header placeholder resolved from the server's secrets store at execution. Store it once (`PUT /api/secrets/OCG_PUBLIC_KEY`). This is the same credential contract as every other tool — see [tool-execution-and-credentials-design.md](tool-execution-and-credentials-design.md). +- **Path templating is generic.** `pathTemplate`/`queryParams` on an `http` tool config is a general Agentspan capability; OCG is its first user. + +### 7.3 The tools + +| Tool (LLM-visible) | Endpoint | Method | +| ---------------------- | ---------------------------------------- | -------- | +| `ocg_query` | `/api/v1/agent/query` | `POST` | +| `ocg_get_entity` | `/api/v1/entities/{entity_id}` | `GET` | +| `ocg_neighborhood` | `/api/v1/graph/neighborhood/{entity_id}` | `GET` | +| `ocg_memory_set` | `/api/v1/memories` | `POST` | +| `ocg_memory_reinforce` | `/api/v1/memories/{key}/reinforce` | `POST` | +| `ocg_memory_delete` | `/api/v1/memories/{key}` | `DELETE` | + +Path params (`{entity_id}`, `{key}`) are filled from the LLM's arguments and URL-encoded; listed query params are appended when present; everything else becomes the JSON body. Subset switches on `ocg_tools()` / `ocg_agent()`: `query`, `entities` (get_entity + neighborhood), `memory` (set / reinforce / delete). + +### 7.4 Keeping the LLM honest + +OCG responses are injected verbatim into the calling LLM's context, so schemas and the canned prompt enforce discipline: +- `max_results` is **fixed at 100** — the schema pins `default: 100`, `minimum: 100`, `maximum: 100`, and the prompt says to ALWAYS use 100 (it is both the hard maximum and the floor for decent context). +- `traversal_level` is **fixed at 1** — the schema pins `default: 1`, `minimum: 1`, `maximum: 1`; the prompt says ALWAYS 1 (pulls each citation's immediate neighborhood), never 0 (too shallow) and never higher. +- `start_time`/`end_time` must be full RFC3339 (`2026-06-04T00:00:00Z`); the OCG API rejects bare dates, and the schemas say so to prevent retry loops. +- The canned retrieval prompt budgets at most 3 distinct keyword queries per request, forbids rephrasing (embedding search returns the same results for the same intent), anchors relative dates on an execution-time `__today__`, and instructs keyword-style queries under ~15 content words. + +`ocg_agent()` defaults to `max_turns=10`; give your *main* agent explicit retrieval instructions and a small `max_turns` so it treats the retriever's answer as complete instead of paging for continuations. + +For full parameter tables see [Python SDK API Reference → ocg_agent() / ocg_tools()](../sdk/python/docs/api-reference.md). diff --git a/design/guardrails-design.md b/design/guardrails-design.md new file mode 100644 index 000000000..96b007b91 --- /dev/null +++ b/design/guardrails-design.md @@ -0,0 +1,690 @@ +# Guardrails Design + +**Status:** Consolidated 2026-06-26 + +**Scope.** Guardrails validate agent inputs and outputs, preventing unsafe, non-compliant, or malformed content from reaching users — and, just as importantly, preventing an agent from taking unsafe *actions* via its tools. This document is the canonical reference for the guardrails feature: the user-facing model and API (guardrail types, the five checkpoints, failure modes), how each guardrail compiles into Conductor workflow tasks (so retries, escalations, and fixes are durable and visible in the Conductor UI), worked recipes, and a condensed industry analysis explaining the design rationale. For the broader agent runtime see [agentspan-design.md](agentspan-design.md); for the language SDK surface see [sdk-design.md](sdk-design.md); for the REST/control-plane API see [api-design.md](api-design.md). + +--- + +## 1. Scope + +A guardrail answers one question: **"Should this content be allowed to proceed?"** — and, on failure, *what should we do about it*. Guardrails integrate directly into Conductor execution so that retries, escalations, and fixes are: + +- **Durable** — they survive worker and client crashes (they are workflow state, not in-memory state). +- **Visible** — each check appears as a task in the Conductor UI, with full status and logs. +- **Compatible** — they work with every execution mode (`run()`, `start()`, `stream()`). + +Guardrails attach to an **agent** (validate LLM input/output) or to a **tool** (validate tool I/O — the highest-risk checkpoint, because tools take real-world actions). + +Code samples below use the Python SDK. The model has **identical enums/types** across SDKs (`OnFail`, `Position`, `GuardrailResult`), but **construction is idiomatic per SDK**: Python `Guardrail(func, ...)`; Java `Guardrail.of(name, func)...build()` plus `Guardrail.external(name)`; TypeScript `guardrail(fn, {...})` plus `guardrail.external()` and the `@Guardrail` decorator; C# `[Guardrail]` attribute / options-style `Create(...)` factories (see [sdk-design.md](sdk-design.md)). + +```python +import re +from conductor.ai.agents import ( + Agent, AgentRuntime, Guardrail, GuardrailResult, + OnFail, Position, guardrail, tool, +) + +# 1. Define a guardrail with the @guardrail decorator +@guardrail +def no_pii(content: str) -> GuardrailResult: + """Reject responses containing credit card numbers.""" + if re.search(r"\b\d{4}[\s-]?\d{4}[\s-]?\d{4}[\s-]?\d{4}\b", content): + return GuardrailResult( + passed=False, + message="Redact all credit card numbers before responding.", + ) + return GuardrailResult(passed=True) + +# 2. Define a tool +@tool +def get_customer(customer_id: str) -> dict: + """Look up customer profile.""" + return {"name": "Alice", "card": "4532-0150-1234-5678"} + +# 3. Attach the guardrail to the agent +agent = Agent( + name="support", + model="openai/gpt-4o", + tools=[get_customer], + guardrails=[ + Guardrail(no_pii, position=Position.OUTPUT, on_fail=OnFail.RETRY), + ], +) + +# 4. Run — the guardrail retries automatically inside the execution +with AgentRuntime() as runtime: + result = runtime.run(agent, "Show me customer CUST-7's full profile.") + print(result.output) # Credit card number will be redacted +``` + +> **Note:** Plain strings (`"output"`, `"retry"`) still work — `OnFail` and `Position` are `str` enums for discoverability and IDE autocompletion. + +--- + +## 2. Guardrail model & API + +A guardrail is a function `(content: str) -> GuardrailResult` that returns pass/fail (and optionally a corrected output). The lifecycle: + +1. The LLM generates a response (or a tool produces output). +2. Each guardrail runs against that content in order. +3. On the first failure, the `on_fail` strategy decides what happens. + +### 2.1 The five checkpoints + +The agent execution loop has five natural checkpoints where guardrails can intercept. Two map to the SDK's `Position` values today; the others are realized through tool guardrails and pre-model context validation. + +| # | Checkpoint | When | What it catches | Cost of failure | +|---|-----------|------|-----------------|-----------------| +| 1 | **Input** | Before the agent loop starts | Prompt injection, malformed input, off-topic requests | Low (no work done yet) | +| 2 | **Pre-model** | Before each LLM call in the loop | Context poisoning, accumulated injection | Medium | +| 3 | **Post-model** | After the LLM responds, before tool dispatch | Hallucinated tool calls, unsafe reasoning | High (about to act) | +| 4 | **Tool** | Around each tool execution | Dangerous parameters, sensitive data in args/results | Critical (action taken) | +| 5 | **Output** | Before returning the final answer | PII, policy violations, quality issues | Medium (text only) | + +The `Position` enum exposes the two most-used checkpoints: + +```python +class Position(str, Enum): + INPUT = "input" # Before the LLM call (or before a tool runs) + OUTPUT = "output" # After the LLM call (or after a tool runs) +``` + +**Key insight (see §5):** most SDKs only implement checkpoints 1 and 5. For agents the highest risk is at 3 and 4 — where the model decided to call a dangerous tool, or the tool is about to execute with bad parameters. Tool guardrails (§3.3) cover these. + +### 2.2 Failure modes (`on_fail`) + +```python +class OnFail(str, Enum): + RETRY = "retry" # Ask the LLM to try again with feedback + RAISE = "raise" # Fail the execution immediately + FIX = "fix" # Use GuardrailResult.fixed_output + HUMAN = "human" # Pause for human review (output only) +``` + +**Default `on_fail` is now uniform `raise` across all four SDKs** (Python, TypeScript, Java, C#) — for every guardrail kind (`guardrail()`/custom, `guardrail.external()`, the `@Guardrail`/`[Guardrail]` decorator/attribute, `RegexGuardrail`, `LLMGuardrail`). This matches the server's routing, which also falls back to `raise` when `on_fail` is null. (Earlier divergence — Python/Java defaulting to `retry`, C# mixed — has been removed; every SDK serializes an explicit `raise` unless overridden.) + +| Mode | Behavior | Best for | +|------|----------|----------| +| `retry` | Feedback appended to the conversation; the LLM retries. After `max_retries` is exhausted, escalates to `raise`. | Quality/format/PII issues the LLM can self-correct. | +| `fix` | Uses `GuardrailResult.fixed_output` directly — no LLM retry. | Deterministic corrections (regex substitution, sanitization). Faster and cheaper. | +| `raise` | Terminates the execution with `FAILED` status and the guardrail message as the reason. | Hard security blocks, zero-tolerance policies, input validation. | +| `human` | Pauses at a HumanTask; a human approves, edits, or rejects. **Only valid for `position="output"`** — input guardrails run client-side and cannot pause an execution. (This constraint is now enforced at construction in **all four SDKs** — Python, TypeScript, Java, and C# — each rejecting the `human`+`input` combination.) | Compliance review, content moderation, sensitive decisions. | + +**Retry escalation.** `max_retries` controls how many times `retry` attempts before escalating to `raise` (default `3`). The minimum is `1`: the Python SDK rejects `max_retries < 1` with a `ValueError`, so `0` is **invalid**, not "equivalent to raise". Each guardrail carries its own `max_retries`. For client-side guardrails (simple agents without tools), the runtime uses the maximum across all output guardrails. This prevents infinite retry loops. + +#### `human` usage with `start()` + +`run()` would block, so use `start()` when an execution may pause: + +```python +with AgentRuntime() as runtime: + handle = runtime.start(agent, "Give me investment advice.") + + import time + while True: + status = handle.get_status() + if status.is_waiting: + print("Paused for human review") + runtime.approve(handle.execution_id) # accept as-is + # or: runtime.reject(handle.execution_id, reason="...") # terminate FAILED + # or: runtime.respond(handle.execution_id, {"edited_output": "..."}) # replace + break + if status.is_complete: + break + time.sleep(1) + + print(handle.get_status().output) +``` + +### 2.3 Guardrail types + +| Type | What it does | Compiles to (see §3) | Output path | +|------|--------------|----------------------|-------------| +| `Guardrail` (custom fn) | Wrap any Python function | SIMPLE worker + normalize InlineTask | `${ref}.output.result.*` | +| `RegexGuardrail` | Pattern block/allow lists | InlineTask (JavaScript, GraalVM) | `${ref}.output.result.*` | +| `LLMGuardrail` | Judge content with a second LLM against a policy | `LlmChatComplete` + InlineTask parser | `${ref}.output.result.*` | +| External | Reference a remote worker by name | SimpleTask | `${ref}.output.*` | + +#### `Guardrail` (custom function) + +```python +guard = Guardrail( + func=check_length, + position="output", # "input" or "output" + on_fail="retry", # "retry", "raise" (default), "fix", or "human" + name="length_check", # Optional, defaults to function name + max_retries=3, +) +``` + +| Parameter | Type | Default | Description | +|-----------|------|---------|-------------| +| `func` | `Callable[[str], GuardrailResult]` | *required* (unless external) | Validation function | +| `position` | `str` | `"output"` | `"input"` or `"output"` | +| `on_fail` | `str` | `"raise"` | `"retry"`, `"raise"`, `"fix"`, `"human"` (default uniform `raise` across all four SDKs) | +| `name` | `str` | function name | Human-readable identifier | +| `max_retries` | `int` | `3` | Max retries for `on_fail="retry"` | + +**External guardrails** — pass `name` without `func` to reference a guardrail worker running elsewhere (any language). Its `external` attribute is `True`. + +```python +Guardrail(name="compliance_checker", on_fail=OnFail.RETRY) +``` + +Worker contract: input `{"content": "", "iteration": }`, output `{"passed": bool, "message": str, "on_fail": str, "should_continue": bool}`. + +#### `RegexGuardrail` + +```python +# Block mode (default): reject content matching any pattern +no_emails = RegexGuardrail( + patterns=[r"[\w.+-]+@[\w-]+\.[\w.-]+"], + mode="block", + name="no_emails", + message="Do not include email addresses in your response.", +) + +# Allow mode: reject content that does NOT match at least one pattern +json_only = RegexGuardrail( + patterns=[r"^\s*[\{\[]"], + mode="allow", + name="json_only", + message="Response must be valid JSON.", +) +``` + +| Parameter | Type | Default | Description | +|-----------|------|---------|-------------| +| `patterns` | `str \| List[str]` | *required* | Regex patterns | +| `mode` | `str` | `"block"` | `"block"` (reject matches) or `"allow"` (reject non-matches) | +| `message` | `str` | auto-generated | Custom failure message | +| `position` | `str` | `"output"` | `"input"` or `"output"` | +| `on_fail` | `str` | `"raise"` | Failure strategy (default uniform `raise` across all four SDKs) | +| `max_retries` | `int` | `3` | Max retries | + +#### `LLMGuardrail` + +```python +safety = LLMGuardrail( + model="anthropic/claude-sonnet-4-6", # use a fast, cheap model + policy=( + "Reject any content that:\n" + "1. Contains medical or legal advice presented as fact\n" + "2. Makes promises or guarantees about outcomes\n" + "3. Includes discriminatory or biased language" + ), + name="content_safety", + on_fail="retry", +) +``` + +The judge LLM receives the policy + content and returns `{"passed": true/false, "reason": "..."}`. + +| Parameter | Type | Default | Description | +|-----------|------|---------|-------------| +| `model` | `str` | *required* | `"provider/model"` format | +| `policy` | `str` | *required* | Natural-language policy for the judge | +| `position` | `str` | `"output"` | `"input"` or `"output"` | +| `on_fail` | `str` | `"raise"` | Failure strategy (default uniform `raise` across all four SDKs) | +| `max_retries` | `int` | `3` | Max retries | +| `max_tokens` | `int` | SDK default | Max tokens for the judge LLM response (supported in Python, TypeScript, C#, and the server `GuardrailCompiler`) | + +> When compiled (§3.4) the judge call runs **server-side** as an `LlmChatComplete` task and needs no client dependency. When run client-side it uses `litellm` (`pip install litellm`); pick a fast model either way to avoid slowing the agent loop. + +### 2.4 `GuardrailResult` and the `@guardrail` decorator + +```python +@dataclass +class GuardrailResult: + passed: bool # True if content passes validation + message: str = "" # Feedback for the LLM (used on retry) + fixed_output: Optional[str] = None # Corrected output (used with on_fail="fix") +``` + +```python +@guardrail +def no_pii(content: str) -> GuardrailResult: + """Reject PII.""" + ... + +@guardrail(name="pii_checker") # custom name +def no_pii(content: str) -> GuardrailResult: ... +``` + +The decorator attaches a `_guardrail_def` attribute (a `GuardrailDef` dataclass) and keeps the function callable, so `@guardrail` functions are usable standalone — without an agent or a server: + +```python +result = no_pii("Some text to validate") +print(result.passed, result.message) +``` + +They can also be deployed as standalone Conductor workers, letting any agent in any language reference them by name (external guardrails, above). `Guardrail()` auto-detects decorated functions. + +### 2.5 Constructor signatures (reference) + +```python +class Guardrail: + def __init__( + self, + func: Optional[Callable[[str], GuardrailResult]] = None, + position: str = "output", + on_fail: str = "raise", # uniform default across all four SDKs + name: Optional[str] = None, + max_retries: int = 3, + ) -> None: ... + external: bool # True when func is None + def check(self, content: str) -> GuardrailResult: ... + +class RegexGuardrail(Guardrail): + def __init__(self, patterns, *, mode="block", position="output", + on_fail="raise", name=None, message=None, max_retries=3): ... + +class LLMGuardrail(Guardrail): + def __init__(self, model, policy, *, position="output", + on_fail="raise", name=None, max_retries=3, max_tokens=None): ... +``` + +```python +@tool(guardrails=[guard1, guard2]) +def my_tool(param: str) -> str: ... + +Agent(name="...", model="...", guardrails=[guard1, guard2]) +``` + +--- + +## 3. Conductor compilation + +Conductor already has every building block needed; the design is about composition. Guardrails behave differently depending on whether the agent has tools — but the user-facing API is the same. + +| Conductor construct | Guardrail role | +|---------------------|----------------| +| `worker_task` | Runs a custom/regex/LLM guardrail; result is durable workflow state. | +| `LlmChatComplete` | Server-side LLM guardrails (replaces client-side litellm). | +| `SwitchTask` | Routes on `{passed, message, on_fail}` to retry / raise / fix / human. | +| `DoWhileTask` | The agent loop; output guardrails insert into its body. | +| `SetVariableTask` | Appends retry feedback to `workflow.variables.messages` — no full re-execution. | +| `TerminateTask` | `on_fail="raise"` / tripwire — terminate `FAILED` with the guardrail reason. | +| `HumanTask` | `on_fail="human"` — durable, assignable, auditable escalation. | +| `ForkTask` + `JoinTask` | Run multiple guardrails in parallel (§3.5). | +| `InlineTask` | Regex eval, LLM-response parsing, score aggregation. | +| `SubWorkflowTask` | Package a guardrail chain for reuse across agents. | + +### 3.1 Output guardrails in the DoWhile loop (agents with tools) + +The agent loop body, before/after guardrails are compiled in: + +``` +DoWhile (before): + [1. LlmChatComplete] + [2. SwitchTask (tool_call vs final_answer)] + +DoWhile (with guardrails): + [1. LlmChatComplete] + [2. Guardrail check task] <-- NEW: evaluates LLM output + [3. SwitchTask on guardrail result] <-- NEW: routes on pass/fail + -> "pass": [original SwitchTask (tool_call vs final_answer)] + -> "retry": [SetVariable: append feedback to messages] -> loop continues + -> "raise": [TerminateTask(FAILED, reason)] + -> "fix": [SetVariable: use fixed_output] -> [original SwitchTask] + -> "human": [HumanTask] -> [SwitchTask on human decision] + -> approve: continue + -> edit: use edited output + -> reject: TerminateTask(FAILED) +``` + +The SwitchTask reads `on_fail` from a type-dependent path (tracked by an `is_inline` flag from `_compile_output_guardrail_tasks()`): + +| Guardrail type | Output path | +|----------------|-------------| +| RegexGuardrail / LLMGuardrail (InlineTask) | `$.{ref}.result.on_fail` | +| Custom function (SIMPLE worker + normalize INLINE) | `$.{ref}.result.on_fail` | +| External (SimpleTask) | `$.{ref}.on_fail` | + +Custom guardrails compile to a **SIMPLE worker task plus a normalize INLINE task**; the routing SWITCH reads the INLINE task's output, so the path is `output.result.on_fail` (not `output.on_fail`). + +**Retry via feedback injection.** On `on_fail="retry"` the guardrail returns: + +```json +{ "passed": false, "message": "Response contains a credit card number. Redact all PII.", + "on_fail": "retry", "should_continue": true } +``` + +A SetVariable appends a system message and the loop iterates back to the LLM — **no full workflow re-execution**: + +```python +set_retry = SetVariableTask(task_ref_name="guardrail_retry_feedback") +set_retry.input_parameter("messages", [ + ...existing_messages, + {"role": "system", + "message": "[Guardrail: ${guardrail.output.message}. Please revise your response.]"}, +]) +``` + +**Termination-condition integration.** When retry guardrails exist, their `should_continue` flag is ANDed into the loop condition so the loop keeps going on retry: + +```javascript +iteration < max_turns + && finishReason != 'LENGTH' + && (toolCalls != null || guardrail_should_continue) +``` + +### 3.2 Simple agents (no tools) and input guardrails + +**Simple agents (client-side output).** With no tools there is no DoWhile loop, so output guardrails run client-side after each execution: execute → check → on retry, modify the prompt and re-execute the whole agent. Simpler, but less efficient (full re-execution per retry). + +**Input guardrails (always client-side).** `position="input"` runs once, before workflow submission. Only `raise`/`human`-block semantics are meaningful — there is no LLM to retry against. This is intentional: fast rejection saves server resources (the workflow is never created), and there is no durability benefit to a one-shot pre-submission check. + +```python +# In runtime.run(), before workflow submission +for guard in agent.guardrails: + if guard.position == "input": + result = guard.check(prompt) + if not result.passed: + raise ValueError(f"Input guardrail '{guard.name}' failed: {result.message}") +``` + +### 3.3 Tool guardrails + +Tool calls are the highest-risk checkpoint because they take **real-world actions** — a hallucinated `send_email(to="all@company.com")`, PII flowing from a database into LLM context, SQL injection in a query parameter. Pre-tool guardrails catch dangerous inputs; post-tool guardrails sanitize dangerous outputs. + +```python +@tool(guardrails=[Guardrail(no_sql_injection, position="input", on_fail="raise")]) +def run_query(query: str) -> str: ... +``` + +Tool guardrails have **two execution paths**, and both exist in the codebase: + +**1. Client-side, in-process (Python-run tools).** When the tool runs as a Python worker, its guardrails are wrapped **inside the tool worker process** by `make_tool_worker()` (`runtime/_dispatch.py`) — the check happens within the existing tool task, not as a separate workflow task: + +- **`position="input"`** — runs before the tool. Receives a JSON string of all input kwargs. On failure with `raise`, raises `ValueError`; otherwise returns `{error: ..., blocked: True}` and the tool is skipped. +- **`position="output"`** — runs after the tool. Receives the result as a string. On `fix`, replaces the result with `fixed_output`; on `raise`, raises `ValueError`. + +**2. Server-compiled gate tasks (separate workflow tasks).** Python serializes each tool's guardrails into the tool config (`config_serializer.py`), so the **server compiler builds them as real, separate workflow tasks**. `ToolCompiler.java` (`collectToolGuardrails`, `buildToolGuardrailGate`, with `compileToolGuardrailTasks` in `GuardrailCompiler.java`) prepends a **guardrail gate before the `DynamicFork`** of tool workers. The gate is: + +``` +tool_call branch: + [format INLINE: _format_tool_calls] <-- build guardrail input from tool calls + [guardrail task(s)] <-- SIMPLE/INLINE/LLM per guardrail kind + [routing SWITCH] <-- pass / raise / fix per on_fail + -> pass: [DynamicFork(tool workers)] <-- the gate runs BEFORE the fork +``` + +So the same authoring API maps to client-side in-process checks for Python-run tools, and to server-compiled gate tasks (durable, visible in the Conductor UI) otherwise. + +### 3.4 Server-side vs client-side LLM guardrails + +Client-side (`litellm` in the worker process) requires a dependency, isn't visible in the UI, and gets no Conductor retry/timeout policies. The compiled form is a server-side `LlmChatComplete` task: + +```python +guardrail_llm = LlmChatComplete( + task_ref_name=f"{agent_name}_guardrail_llm", + llm_provider="openai", # server-configured provider — no extra keys + model="anthropic/claude-sonnet-4-6", + messages=[ + ChatMessage(role="system", message=guardrail_policy_prompt), + ChatMessage(role="user", message="${llm_output}"), + ], + temperature=0.0, max_tokens=200, json_output=True, +) +``` + +It is followed by an InlineTask that parses `passed`/`reason` and maps `on_fail`. Choosing a construct per guardrail kind: + +| Construct | When to use | +|-----------|-------------| +| `worker_task` (Python) | Custom logic, regex, DB lookups | +| `LlmChatComplete` (server) | Policy evaluation, content classification | +| `InlineTask` (JavaScript) | Threshold/pattern checks, score aggregation | + +### 3.5 Parallel guardrails via ForkTask + +Run independent guardrails (PII + toxicity + policy) concurrently, then aggregate: + +``` +[LlmChatComplete output] -> [ForkTask: PII | Toxicity | Policy] -> [JoinTask] + -> [InlineTask: aggregate] -> [SwitchTask] -> pass: continue / fail: on_fail handler +``` + +```javascript +(function() { + var results = [$.pii_guard.output, $.toxicity_guard.output, $.policy_guard.output]; + var failed = results.filter(function(r) { return !r.passed; }); + if (failed.length === 0) return { passed: true, on_fail: "pass" }; + // Priority: raise > human > retry > fix — return the most severe failure + var priority = { "raise": 4, "human": 3, "retry": 2, "fix": 1 }; + failed.sort(function(a, b) { return (priority[b.on_fail] || 0) - (priority[a.on_fail] || 0); }); + return failed[0]; +})() +``` + +### 3.6 Multi-agent guardrail wrapping + +When a multi-agent strategy workflow has output guardrails, the whole strategy is wrapped in an outer DoWhile, which re-runs the full strategy on retry: + +``` +DoWhile (guardrail_loop) + ├─ InlineSubWorkflow (strategy workflow) + ├─ [Guardrail check task(s)] + └─ [Guardrail routing SwitchTask(s)] +``` + +### 3.7 Why compiled beats client-side + +| Aspect | Client-side | Compiled into workflow | +|--------|-------------|------------------------| +| Durability | Lost on crash | Survives crashes | +| Visibility | Invisible | Tasks visible in Conductor UI | +| Retry efficiency | Re-executes entire workflow | Loop iteration only | +| `start()` / `stream()` | Skipped | Works automatically | +| Human escalation | Not possible | HumanTask with full state | +| Parallel guardrails | Sequential only | ForkTask parallelism | +| Audit / timeout / retry policy | None / hardcoded | Full history; per-task config | +| LLM guardrails | Needs litellm | Uses server LLM providers | + +The API is backward-compatible: the `@guardrail` decorator, `OnFail`/`Position` enums, external guardrails, and the new failure modes layer on without breaking existing code. What changes is internal — output guardrails compile into the loop, `LLMGuardrail` becomes an `LlmChatComplete` task, `human` becomes a HumanTask, retry becomes a SetVariable, and `start()`/`stream()` get guardrail support for free. + +--- + +## 4. Recipes / examples + +### PII detection with retry + +```python +import re +from conductor.ai.agents import Agent, Guardrail, GuardrailResult + +def no_pii(content: str) -> GuardrailResult: + patterns = { + "credit card": r"\b\d{4}[\s-]?\d{4}[\s-]?\d{4}[\s-]?\d{4}\b", + "SSN": r"\b\d{3}-\d{2}-\d{4}\b", + "email": r"[\w.+-]+@[\w-]+\.[\w.-]+", + } + for name, pat in patterns.items(): + if re.search(pat, content): + return GuardrailResult(passed=False, + message=f"Response contains {name}. Redact all PII.") + return GuardrailResult(passed=True) + +agent = Agent(name="safe_agent", model="openai/gpt-4o", tools=[...], + guardrails=[Guardrail(no_pii, on_fail="retry", max_retries=3)]) +``` + +### Automatic redaction with fix + +```python +import re +from conductor.ai.agents import Agent, Guardrail, GuardrailResult + +def redact_all_pii(content: str) -> GuardrailResult: + patterns = [ + (r"\b\d{4}[\s-]?\d{4}[\s-]?\d{4}[\s-]?\d{4}\b", "XXXX-XXXX-XXXX-XXXX"), + (r"\b\d{3}-\d{2}-\d{4}\b", "XXX-XX-XXXX"), + (r"[\w.+-]+@[\w-]+\.[\w.-]+", "[EMAIL REDACTED]"), + ] + fixed, found = content, False + for pat, replacement in patterns: + if re.search(pat, fixed): + found = True + fixed = re.sub(pat, replacement, fixed) + if found: + return GuardrailResult(passed=False, message="PII redacted.", fixed_output=fixed) + return GuardrailResult(passed=True) + +agent = Agent(name="redacting_agent", model="openai/gpt-4o", tools=[...], + guardrails=[Guardrail(redact_all_pii, on_fail="fix")]) +``` + +### JSON-only output enforcement + +```python +from conductor.ai.agents import Agent, RegexGuardrail + +agent = Agent( + name="json_agent", model="openai/gpt-4o", + instructions="Always respond with valid JSON.", + guardrails=[RegexGuardrail( + patterns=[r"^\s*[\{\[]"], mode="allow", name="json_only", + message="Response must start with { or [. Output only valid JSON.", + on_fail="retry", + )], +) +``` + +### Layered guardrails (lenient + strict) + +```python +from conductor.ai.agents import Agent, Guardrail, GuardrailResult, RegexGuardrail + +length_guard = Guardrail( + lambda c: GuardrailResult(passed=len(c) <= 1000, message="Too long. Be concise."), + on_fail="retry", name="length_check", +) +ssn_guard = RegexGuardrail(patterns=[r"\b\d{3}-\d{2}-\d{4}\b"], on_fail="raise", name="no_ssn") + +agent = Agent(name="layered_agent", model="openai/gpt-4o", tools=[...], + guardrails=[length_guard, ssn_guard]) +# Guardrails run in order. The first failure determines the action. +``` + +### Compliance review with human escalation + +```python +from conductor.ai.agents import Agent, Guardrail, GuardrailResult + +def compliance_check(content: str) -> GuardrailResult: + flagged = ["guaranteed returns", "risk-free", "investment advice"] + for term in flagged: + if term.lower() in content.lower(): + return GuardrailResult(passed=False, + message=f"Contains flagged term: '{term}'. Requires compliance review.") + return GuardrailResult(passed=True) + +agent = Agent(name="finance_agent", model="openai/gpt-4o", tools=[...], + guardrails=[Guardrail(compliance_check, on_fail="human", name="compliance")]) + +# Use start() since the execution may pause (see §2.2 for the poll/approve loop) +with AgentRuntime() as runtime: + handle = runtime.start(agent, "Should I invest in tech stocks?") +``` + +### SQL injection blocking on a tool + +```python +import re +from conductor.ai.agents import Guardrail, GuardrailResult, tool + +def no_sql_injection(content: str) -> GuardrailResult: + dangerous = [r"DROP\s+TABLE", r"DELETE\s+FROM", r";\s*--", r"UNION\s+SELECT"] + for pat in dangerous: + if re.search(pat, content, re.IGNORECASE): + return GuardrailResult(passed=False, message=f"Blocked: {pat}") + return GuardrailResult(passed=True) + +@tool(guardrails=[Guardrail(no_sql_injection, position="input", on_fail="raise")]) +def run_query(query: str) -> str: + """Execute a database query.""" + return f"Results: {query}" # never called with a dangerous query +``` + +### Tool output sanitization (redact secrets) + +```python +import re +from conductor.ai.agents import Guardrail, GuardrailResult, tool + +def redact_secrets(content: str) -> GuardrailResult: + pattern = r"sk-[a-zA-Z0-9]{40,}" + if re.search(pattern, content): + return GuardrailResult(passed=False, message="API key redacted.", + fixed_output=re.sub(pattern, "sk-***REDACTED***", content)) + return GuardrailResult(passed=True) + +@tool(guardrails=[Guardrail(redact_secrets, position="output", on_fail="fix")]) +def fetch_config(service: str) -> str: + return '{"api_key": "sk-abc123def456ghi789jkl012mno345pqr678stu901"}' +# The tool result has the API key redacted before the LLM sees it +``` + +--- + +## 5. Background & rationale + +*Condensed from an industry review of OpenAI Agents SDK, AG2 (AutoGen), LangGraph/LangChain, CrewAI, Guardrails AI, and NVIDIA NeMo Guardrails, and a gap analysis of our own implementation.* + +### Why guardrails matter for agents, not just LLMs + +For a single LLM call, guardrails are useful; for **agents** they are essential, because autonomy amplifies risk: + +- Agents make multi-step decisions without human oversight, and each tool call is an **action** (email, DB write, API call), not just text. +- A single bad decision cascades through tool chains; a 25-turn agent has far more surface area than one prompt/response. +- The **Swiss-cheese model** applies: no single guardrail catches everything, so effective safety means defense in depth across multiple checkpoints. + +| Surface | LLM risk | Agent risk (amplified) | +|---------|----------|------------------------| +| Prompt injection | LLM follows injected instructions | Agent executes injected tool calls | +| Data exfiltration | LLM mentions sensitive data | Agent sends sensitive data via tools | +| Hallucination | Wrong text | Wrong actions from hallucinated reasoning | +| Loop exploitation | N/A | Infinite tool-call loop, burning tokens | + +Guardrails span six concern layers — Safety (toxic content), Security (injection, exfiltration), Compliance (PII, HIPAA/GDPR), Quality (hallucination, format), Policy (brand/tone), and Cost (token/loop guards). + +### Failure-mode patterns across the industry + +The industry has converged on five patterns; our `on_fail` modes implement four of them directly, and route-to-agent is expressible via multi-agent strategies: + +- **Tripwire** (OpenAI) — raise and halt → our `raise`. +- **Retry with feedback** (Orkes/CrewAI) — append feedback, re-run → our `retry`. +- **Route/redirect** (AG2) — hand off to a safety agent. +- **Fix/modify** (Guardrails AI) — auto-correct and continue → our `fix`. +- **Human escalation** — pause for review → our `human`, backed by Conductor's HumanTask. + +### How the industry does it — SDK comparison + +| Aspect | OpenAI | AG2 | LangGraph | CrewAI | Guardrails AI | NeMo | +|--------|--------|-----|-----------|--------|---------------|------| +| Architecture | Parallel/blocking modes | Event-driven actors | Middleware hooks | Task-level | Composable validators | Flow DSL (Colang) | +| Input | Yes | Yes | Before hooks | Limited | Yes | Yes | +| Output | Yes | Yes | After hooks | Yes | Yes | Yes | +| Tool | Yes | Limited | Wrap hooks | Tool-call hooks | Limited | Execution rails | +| Failure mode | Tripwire only | Message routing | Raise/modify | Retry/error | exception/fix/retry/custom | Event blocking | +| Unique feature | Parallel mode | Agent routing | 5 lifecycle hooks | Hallucination guard | Validator hub (100+) | Colang DSL | + +Takeaways: OpenAI's parallel-vs-blocking execution is a genuine latency innovation but offers only tripwire; LangGraph's five lifecycle hooks are the most flexible but unopinionated; Guardrails AI has the best composability but isn't agent-aware; CrewAI's hallucination guardrail is a useful domain-specific type; NeMo's Colang is the most expressive but adds a language to learn. Most SDKs cover only input/output (checkpoints 1 and 5) — the agent-critical checkpoints 3 (post-model) and 4 (tool) are where only OpenAI and LangGraph have meaningful coverage. + +### Our differentiator and the gap analysis that drove this design + +Our key advantage is **server-side durable execution via Conductor**. Two capabilities follow that no other SDK has: + +- **Durable, assignable, auditable human-in-the-loop escalation** (`on_fail="human"`) via HumanTask, with assignment, form templates, and timeout policies — surviving process restarts. +- **Loop-internal retry** that costs one DoWhile iteration instead of a full re-execution. + +The original implementation ran guardrails **client-side in Python**, which contradicted that advantage: checks were skipped if the client crashed, invisible in the UI, re-submitted the entire execution on retry, and were unavailable to `start()`/`stream()`. A `compile_guardrail_tasks()` method existed but was never wired in. This design closes those gaps in phases: + +1. **Core server-side guardrails** — wire compilation into the DoWhile loop; support `retry` (SetVariable + continue) and `raise` (TerminateTask); configurable `max_retries`; remove client-side output logic. +2. **New failure modes** — `human` (HumanTask), `fix` (corrected output), and `LLMGuardrail` as a server-side `LlmChatComplete`. +3. **Tool guardrails** — `@tool(guardrails=[...])`, pre/post compilation, DynamicFork integration. +4. **Advanced** — parallel guardrails via ForkTask, composable `&`/`|` operators, and built-in types (`PIIGuardrail`, `ToxicityGuardrail`, `PromptInjectionGuardrail`, `HallucinationGuardrail`), plus pass/fail and retry-cost metrics surfaced in the Conductor UI. + +The resulting recommended architecture keeps **input guardrails client-side** (one-shot, no durability benefit) and compiles **output and tool guardrails into the workflow** (durable, visible, efficient retry, human escalation) — see the loop diagrams in §3. diff --git a/design/sdk-design.md b/design/sdk-design.md new file mode 100644 index 000000000..11f40093b --- /dev/null +++ b/design/sdk-design.md @@ -0,0 +1,632 @@ +# SDK Design + +**Status:** Consolidated 2026-06-26 + +**Scope.** This is the canonical guide to authoring a Agentspan SDK in any language. It defines the contract every SDK must satisfy — the public API surface (Agent, tools, guardrails, strategies, memory, handoffs, termination, results, streaming), the `AgentConfig` JSON wire format, worker registration, the control-plane REST/SSE API, skills, and framework bridges — plus the ~89-feature parity matrix, per-language idiom guides, and acceptance testing. It is authoritative for *what* an SDK must do; it links to siblings ([api-design.md](api-design.md), [agentspan-design.md](agentspan-design.md), [guardrails-design.md](guardrails-design.md), [tool-execution-and-credentials-design.md](tool-execution-and-credentials-design.md), [framework-integration.md](framework-integration.md)) for the wire/platform detail, and to per-language docs for *how* to do it idiomatically. + +--- + +## 1. Scope & Philosophy + +### Everything is an Agent + +A single `Agent` wraps an LLM + tools. An agent with sub-agents *is* a multi-agent system. There is one type to learn. Simple or complex, every agent is an instance of the same class; orchestration is selected by a `strategy` over its `agents` list. + +### Reference implementation + translation guide + +We use **Approach 2: a reference implementation plus translation guides.** The **Python SDK is the spec** — it is the executable definition of correct behavior. The **Java SDK (`sdk/java`)** is the reference for record/POJO-shaped languages. Every other SDK (TypeScript, Go, Kotlin, C#, Ruby) must reproduce *behavior* parity, not API shape: port the **model**, be idiomatic to the language. + +Each SDK's job is identical: + +1. **Define** agents, tools, guardrails as language-native constructs. +2. **Serialize** to the `AgentConfig` JSON the server expects (§3). +3. **Register** tool/guardrail/callback workers the server dispatches to (§3.6). +4. **Execute** via the control-plane REST API — start, deploy, compile, status, respond (§3.7). +5. **Stream** via SSE for real-time events (§2 / §3.8). +6. **Resolve** credentials via execution tokens at runtime (see [tool-execution-and-credentials-design.md](tool-execution-and-credentials-design.md)). + +``` +┌─────────────────────────────────────────────────┐ +│ SDK (any language) │ +│ Agent Definition → Serialization → AgentConfig │ +│ Worker Poll Loop → Tool Execution → Results │ +│ SSE Client → Event Stream → AgentStream │ +│ Credential Fetcher → Execution Token → Secrets │ +└──────────────────────┬──────────────────────────┘ + │ REST + SSE (JSON) +┌──────────────────────▼──────────────────────────┐ +│ Agentspan Server (Java) │ +│ Compiler → Conductor WorkflowDef │ +│ Executor → Conductor Workflow Engine │ +│ StreamRegistry → SSE Events │ +│ CredentialService → AES-256-GCM Store │ +└─────────────────────────────────────────────────┘ +``` + +**Correctness criterion:** equivalent agent definitions must produce **identical `AgentConfig` JSON** across SDKs. That is the primary thing the acceptance test (§5) checks. + +### Build on the Conductor SDK + +Agentspan runs on Conductor. Every SDK extends the equivalent Conductor SDK (`https://github.com/conductor-oss/{lang}-sdk` — java, go, python, csharp, javascript, rust, ruby, …) rather than rolling its own transport. + +- Do **not** implement custom HTTP transport. Use Conductor's `ApiClient` for all remote calls — it owns token management, auth, timeouts, and config. +- Do **not** redefine connection properties already in the Conductor SDK config. +- Namespace: `org.conductoross.conductor.ai` (or the language equivalent). + +Separation of concerns (as in Java): + +- The **Conductor client** (`ApiClient`) owns server URL + auth. +- An **SDK config** object owns *only* worker-runner tuning (poll interval, thread count). It carries no connection details. +- `AgentRuntime` takes both and wires them together. + +### Authentication & Configuration + +OSS deployments need no auth. Orkes deployments use an API key (preferred) or legacy key/secret, passed through the Conductor `ApiClient`. + +| Mode | Headers | Use case | +|------|---------|----------| +| API Key (preferred) | `Authorization: Bearer ` | Production | +| Legacy Key/Secret | `X-Auth-Key`, `X-Auth-Secret` | Backward compat | + +Because the SDK builds on Conductor's `ApiClient`, the `CONDUCTOR_SERVER_URL` / `CONDUCTOR_AUTH_KEY` / `CONDUCTOR_AUTH_SECRET` variables are honored transitively — do not re-implement them. The `AGENTSPAN_*` variables are the SDK-level overrides read before constructing the client. + +| Environment Variable | Default | Description | +|---------------------|---------|-------------| +| `AGENTSPAN_SERVER_URL` | `http://localhost:6767/api` | Server API URL | +| `AGENTSPAN_API_KEY` | — | Bearer token / API key | +| `AGENTSPAN_AUTH_KEY` / `AGENTSPAN_AUTH_SECRET` | — | Legacy auth | +| `AGENTSPAN_WORKER_POLL_INTERVAL` | `100` | Worker poll interval (ms) | +| `AGENTSPAN_WORKER_THREADS` | `1` | Threads per worker | +| `AGENTSPAN_LLM_RETRY_COUNT` | `3` | LLM call retry count | +| `AGENTSPAN_AUTO_START_WORKERS` | `true` | Auto-start worker processes | +| `AGENTSPAN_AUTO_START_SERVER` | `true` | Auto-start local server | +| `AGENTSPAN_DAEMON_WORKERS` | `true` | Kill workers on exit | +| `AGENTSPAN_STREAMING_ENABLED` | `true` | Enable SSE streaming | +| `AGENTSPAN_SECRET_STRICT_MODE` | `false` | No env-var fallback for credentials | +| `AGENTSPAN_INTEGRATIONS_AUTO_REGISTER` | `false` | Auto-register LLM integrations | +| `AGENTSPAN_LOG_LEVEL` | `INFO` | Logging level | + +**URL normalization:** strip a trailing `/` and any `/api` suffix, then append `/api`. + +--- + +## 2. The SDK Contract + +Every SDK must expose the following public surface. Names follow the target language's conventions (`snake_case` in Python/Ruby, `camelCase` in JS/Java/Kotlin, `PascalCase` in C#), but the **semantics must be identical**. This section is the conceptual model; §3 is the wire format it serializes to. + +### 2.1 Agent + +The single orchestration primitive — an immutable, declarative config built with a fluent builder (or constructor / data class as idiomatic). `name` is required and must match `^[a-zA-Z_][a-zA-Z0-9_-]*$`. `maxTurns` defaults to 25. + +```java +Agent agent = Agent.builder() + .name("assistant") + .model("openai/gpt-4o") // "provider/model"; omit for external agents + .instructions("You are helpful.") + .maxTurns(10) + .build(); +``` + +| Field | Type | Default | Notes | +|-------|------|---------|-------| +| `name` | string | required | Unique agent name | +| `model` | string | null | `provider/model`; **omit ⇒ external** (references a deployed workflow) | +| `instructions` | string \| callable \| PromptTemplate | null | System prompt; **callable is re-evaluated at serialize time** | +| `tools` | Tool[] | [] | Tools available to this agent | +| `agents` | Agent[] | [] | Sub-agents (multi-agent) | +| `strategy` | Strategy enum | null | Orchestration; emitted only when `agents` non-empty | +| `router` | Agent \| callable | null | Router for `router` strategy | +| `outputType` | class/schema | null | Structured output type | +| `guardrails` | Guardrail[] | [] | Input/output validators | +| `memory` | ConversationMemory | null | Conversation history | +| `maxTurns` | int | 25 | Max LLM call turns | +| `maxTokens` / `temperature` | int / float | null | LLM params | +| `timeoutSeconds` | int | 0 | Execution timeout (0 = none) | +| `external` | bool | false | Runs elsewhere | +| `stopWhen` / `termination` / `gate` | — | null | Stop conditions (§2.6) | +| `handoffs` / `allowedTransitions` | — | [] / null | Handoff triggers + reachability (§2.7) | +| `introduction` / `metadata` | string / map | null | Self-intro, arbitrary metadata | +| `callbacks` | CallbackHandler[] | [] | Lifecycle hooks (§2.9) | +| `enablePlanning` | bool | false | Plan-first preamble (ADK feature) | +| `includeContents` | string | null | `"default"` full parent context, `"none"` fresh | +| `thinkingBudgetTokens` | int | null | Extended thinking budget | +| `requiredTools` | string[] | null | Tools the LLM must use | +| `codeExecutionConfig` / `cliConfig` | — | null | Sandbox / CLI allowlist | +| `credentials` | (string \| CredentialFile)[] | null | Agent-level credentials | + +**Sequential sugar:** `a >> b` (Python/Kotlin/C#/Ruby operator; `a.then(b)` in Java; `.pipe()` in TS) returns a new `SEQUENTIAL` agent. Chaining **flattens**: `a >> b >> c` → `Agent(name="a_b_c", strategy=SEQUENTIAL, agents=[a,b,c])`, never nested. + +**`@agent` / `@AgentDef` annotation (alternative declarative path):** define an agent from an annotated method/function. Attributes mirror the constructor (`name`, `model`, `instructions`, `tools`, `guardrails`, `agents`, `strategy`, `maxTurns`, `maxTokens`, `temperature`, `credentials`, `contextWindowBudget`). Resolve with `Agent.fromInstance(obj[, name])`. Return type controls behavior: `void` (attrs only), `String` (dynamic instructions), `PromptTemplate`, `Agent.Builder` (decorate then build), or `Agent` (full factory). `@Tool` / `@GuardrailDef` methods on the same object attach to the agents. + +### 2.2 Strategies + +Multi-agent orchestration selected by `strategy` over `agents`: + +`HANDOFF` (default), `SEQUENTIAL`, `PARALLEL`, `ROUTER`, `ROUND_ROBIN`, `RANDOM`, `SWARM`, `MANUAL`, `PLAN_EXECUTE`. + +Server-side compilation: handoff/swarm/manual → `SWITCH`-driven loops, sequential → chained sub-workflows, parallel → `FORK_JOIN`, router → `SWITCH`. `PLAN_EXECUTE` uses named `planner` (required) + `fallback` (optional) slots instead of positional `agents` (§3.5). + +Some strategies expect locally-registered workers by name (in the Python reference). Note: **for non-Python SDKs, the server handles several of these internally** — verify which by running the feature's example *without* a worker (see §15 lessons). Worker name patterns: + +- **SWARM** — `{src}_transfer_to_{dst}`, `{name}_check_transfer`, `{name}_handoff_check`. (Transfer tools `transfer_to_{agent}` are **auto-generated by the server** — do not add them manually.) +- **MANUAL** — `{name}_process_selection`. + +### 2.3 Tools + +#### Custom (local) tools + +Two ways to define a local tool: + +1. **Annotation/decorator** — mark a method `@tool` / `@Tool(name, description, …)`; discover via reflection (`ToolRegistry.fromInstance(obj)` → `List`). +2. **Builder** — construct a `ToolDef`/`ToolConfig` directly. + +The SDK extracts function name, docstring, and parameter schema (type hints), generates JSON Schema, registers a Conductor SIMPLE task, and starts a worker (§3.6). A `ToolDef` carries: `name`, `description`, in/out `schema`, local `func`, `toolType` (default `worker`), `approvalRequired` (HITL gate), `credentials`, `timeoutSeconds`, retry policy, `maxCalls`, `guardrails`, `agentRef`, `stateful`, `isolated` (credential isolation, default true). + +```python +@tool +def get_weather(city: str) -> str: + """Get current weather for a city.""" + return f"72F and sunny in {city}" +``` + +**ToolContext (dependency injection):** when a tool declares a `ToolContext` parameter, the SDK injects `session_id`, `execution_id`, `agent_name`, `metadata`, `dependencies`, `state`. The server passes `__agentspan_ctx__` in task input; the SDK extracts/populates it and **strips it before calling the user function**. State mutations are captured back into the result under `_state_updates` (§3.10). + +**External / by-reference tools:** a tool with no local function — the SDK emits only the task name; a remote worker (possibly another language/machine) picks it up. This is the core mechanism for distributed agent systems. Every SDK must support defining tools by reference (name + schema only). + +#### Built-in / server-side tools + +Provide factories/builders for each; all produce the same tool model. These execute **on the server — no local worker** (except `agent_tool`, which depends on the sub-agent). + +| Tool | Constructor shape | toolType | +|---|---|---| +| HTTP | `httpTool(name, description, url, method, headers, …, credentials)` | `http` | +| API (OpenAPI/Swagger/Postman auto-discovery) | `apiTool(url, name, …, maxTools=64, credentials)` | `api` | +| MCP | `mcpTool(serverUrl, name, …, toolNames, maxTools=64, credentials)` | `mcp` | +| Agent-as-tool | `agentTool(agent[, description])` | `agent_tool` | +| Human (HITL) | `humanTool(name, description[, inputSchema])` | `human` | +| Media (image/audio/video) | `imageTool(name, desc, provider, model[, schema])` (+audio/video) | `generate_*` | +| PDF | `pdfTool([name, description, inputSchema, defaults])` | `generate_pdf` | +| RAG | `searchTool(…)` / `indexTool(…)` | `rag_search` / `rag_index` | + +**Critical:** media (`generate_*`) and RAG tools are **server-side only** — never execute them as worker tasks. HTTP headers may reference credentials with `${NAME}` syntax, resolved server-side at execution time; all placeholders must be declared in `credentials`. + +### 2.4 Guardrails + +Input/output validation attached to an agent (or a tool). All produce a `GuardrailDef`/`GuardrailConfig` with `position` (`INPUT`/`OUTPUT`), `onFail` (`RETRY`/`RAISE`/`FIX`/`HUMAN`), `maxRetries`, and a `guardrailType`. See [guardrails-design.md](guardrails-design.md) for the full compilation model. + +| Type | Execution | Constructor | +|---|---|---| +| Custom | SDK worker (`{agent}_output_guardrail` / `{guardrail.name}`) | `Guardrail.of(name, fn)` / `@guardrail` — `fn: String → GuardrailResult` | +| Regex | Server-side INLINE (JS) | `RegexGuardrail.builder().patterns(…).mode("block"\|"allow")` | +| LLM | Server-side LLM call | `LLMGuardrail.builder().model(…).policy(…)` | +| External | Remote worker (no local worker) | `Guardrail.external(name)` | + +`GuardrailResult`: `passed` (bool), `message` (failure reason), `fixedOutput` (for `onFail=fix`). **OnFail semantics:** `RETRY` re-runs the LLM with feedback (DO_WHILE loop); `RAISE` fails the execution; `FIX` uses `fixedOutput`; `HUMAN` pauses for review (HUMAN task). The default `onFail` is now **`raise` uniformly across all four SDKs**. `human`+`input` is rejected at construction time in **all four SDKs** (an input guardrail runs client-side and cannot pause an execution). Guardrails attach at **two levels** — `agent.guardrails` and `tool.guardrails`; the runtime must register workers for both. + +### 2.5 Results & Streaming + +#### Execution surface (AgentRuntime) + +`AutoCloseable` — shut down workers and release the HTTP pool on close. Provide both sync and async variants of each. `run` = `start` then wait. Workers register inside `start` so they bind to the correct queue. + +| Method | Purpose | Returns | +|---|---|---| +| `run(agent, prompt)` | Execute synchronously | `AgentResult` | +| `start(agent, prompt)` | Fire-and-forget | `AgentHandle` | +| `stream(agent, prompt)` | Execute and stream events | `AgentStream` | +| `plan(agent)` | Compile to a workflow def without executing (dry run) | `ExecutionPlan` | +| `deploy(agents…)` | Compile + register (CI/CD); no workers, no execution | `DeploymentInfo` | +| `deploy(agent, schedules)` | Deploy + reconcile cron schedules (§2.8) | — | +| `serve(agents…)` | Register workers and poll until interrupted | — | +| `resume(executionId, agent)` | Re-attach to a running execution, re-register workers | — | +| `schedules()` | Cron-schedule lifecycle accessor | — | +| `configure(config)` / `shutdown()` | Pre-configure / tear down the singleton runtime | — | + +The runtime operates on a lazily-initialized **singleton** or an explicit instance, supports language-appropriate resource management (Python `with`, Java try-with-resources, Go `defer`, C# `using`, Ruby `ensure`), and auto-starts workers/local server when configured. + +#### AgentResult + +`output` (always a dict — normalized; raw or typed via a class), `status` (`COMPLETED`/`FAILED`/`TERMINATED`/`TIMED_OUT`), `finishReason` (`STOP`/`LENGTH`/`TOOL_CALLS`/`ERROR`/`CANCELLED`/`TIMEOUT`/`GUARDRAIL`/`REJECTED`), `messages`, `toolCalls`, `tokenUsage` (prompt/completion/total), `error`, `events`, `subResults` (per-agent, parallel), `correlationId`. Convenience: `isSuccess`, `isFailed`, `isRejected`, `printResult`. Token usage and tool calls are enriched from the completed workflow via the Conductor workflow client. + +**Result normalization** (`output` always a dict): dict → as-is; string-on-success → `{"result": s}`; null-on-success → `{"result": null}`; string-on-failure → `{"error": s, "status": "FAILED"}`; null-on-failure → `{"error": "Unknown error", "status": "FAILED"}`. + +#### AgentHandle / AgentStatus + +`AgentHandle` (from `start`): `getStatus`, `waitForResult(timeout, poll)`, `isWaiting`, `waitUntilWaiting(timeout)`, `approve`/`reject`/`respond`/`send`, `pause`/`resume`/`cancel`, `stream`. Every method has sync + async variants. `AgentStatus`: `executionId`, `isComplete`/`isRunning`/`isWaiting`, `output`, `status`, `reason`, `currentTask`, `messages`, `pendingTool`. + +#### Streaming & HITL + +`stream` returns an iterable `AgentStream` of typed `AgentEvent` plus HITL controls. After iteration: `events` (all captured), `result` (built from events), `getResult()` (drain + return). + +`EventType` enum (every SDK): `THINKING, TOOL_CALL, TOOL_RESULT, HANDOFF, WAITING, MESSAGE, ERROR, DONE, GUARDRAIL_PASS, GUARDRAIL_FAIL`. Server-only types (`context_condensed`, `subagent_start`, `subagent_stop`) are **not** in the enum — pass them through as raw events. Before exposing args, **strip internal keys** `_agent_state`, `method`. + +**HITL:** on pause the agent emits `WAITING` carrying the pending tool (`taskRefName`, name, params, optional response/UI schema). Respond via `approve()` / `approve(comment)` / `reject(reason)` / `respond(map)`. **Route to the right execution:** under HANDOFF/SEQUENTIAL/PARALLEL the HUMAN task lives in a *sub-execution* — pass the `WAITING` event to the approve/reject call so it targets that event's `executionId`, not the root. After approving a sub-execution, poll workflow status via `waitForResult` rather than blocking on the original stream. See [api-design.md](api-design.md) and `2026-03-20-hitl-endpoint-design.md` for endpoint detail. + +**SSE client requirements:** parse the wire format (event/id/data), handle heartbeat comments (`:` lines), reconnect with `Last-Event-ID`, detect SSE unavailability (only heartbeats for 15s → fall back to polling `GET /{id}/status`), yield parsed events. + +### 2.6 Termination, Stop & Gate + +**Termination conditions** are composable with `and`/`or` (operator overloading or builder), each implementing both `toJSON()` (wire) and `shouldTerminate(context)` (worker evaluation → `{shouldTerminate, reason}`): + +- `MaxMessageTermination.of(n)` +- `TextMentionTermination.of(text[, caseSensitive])` +- `StopMessageTermination.of(text)` +- `TokenUsageTermination.ofTotal/ofPrompt/ofCompletion(n)` + +```java +MaxMessageTermination.of(10).or(TextMentionTermination.of("DONE")) +``` + +**Gate** stops a sequential pipeline when an agent's output contains a sentinel: `new TextGate(text[, caseSensitive])`, attached via `.gate(...)`. Compiled to an INLINE (text) or SIMPLE (worker) task that returns `{"decision": "continue"|"stop"}`. **stop_when** is a callable stop condition registered as `{agent}_stop_when`. + +### 2.7 Handoffs + +SWARM transfer triggers, each naming a target agent: + +- `OnTextMention.of(text, target)` — output contains text. +- `OnToolResult.of(tool, target[, resultContains])` — after a tool runs. +- `OnCondition(target, predicate)` — local predicate worker (`{agent}_handoff_{target}`). + +Restrict reachability with `allowedTransitions` (source → allowed targets), enforced server-side. + +### 2.8 Memory + +**ConversationMemory** — session history: `addUser/Assistant/SystemMessage`, `addToolCall`, `addToolResult`, `toChatMessages`, `clear`. With `maxMessages` set, trim oldest but always preserve system messages. Serializes as `{"messages": [...], "maxMessages": N}`. + +**SemanticMemory** — cross-session recall (SDK-side, all four SDKs): `add`, `search(query, topK)`, `delete`, `clear`, `listAll`. This is a **client-side keyword (Jaccard-overlap) store** — it is **not** a vector DB / embedding feature, and is **not serialized to the wire**. Pluggable `MemoryStore`; SDK must ship at least `InMemoryStore` (keyword-overlap similarity). + +### 2.9 Callbacks + +Lifecycle hooks registered on the agent, run as local workers. Either single functions (`beforeModelCallback`, `afterModelCallback`, `beforeAgentCallback`, `afterAgentCallback`) or a composable `CallbackHandler` overriding any of `onAgentStart/End`, `onModelStart/End`, `onToolStart/End`. Returning a non-empty map short-circuits/overrides at that position; multiple handlers run in order. + +Wire positions are `before_agent`, `after_agent`, `before_model`, `after_model`, `before_tool`, `after_tool` (**not** the method names). Serialized as `{"position": "", "taskName": "{agent}_"}`; the worker bridges server input (`{messages, llm_result}`) to the handler's typed signature, supplying `agentName` from the registration closure. + +### 2.10 Code Execution & Schedules + +**Code execution:** `CodeExecutionConfig` (`enabled`, `allowedLanguages`, `allowedCommands`, `timeout`) and `cliConfig` (CLI allowlist). Executor implementations: `LocalCodeExecutor`, `DockerCodeExecutor`, `JupyterCodeExecutor`, `ServerlessCodeExecutor`; `as_tool()` converts an executor to a tool. See [tool-execution-and-credentials-design.md](tool-execution-and-credentials-design.md). + +**Schedules:** declarative cron via `deploy(agent, schedules)`: + +```java +Schedule.builder().name("weekday-9am").cron("0 0 9 * * MON-FRI") + .timezone("America/Los_Angeles").input(Map.of("channel", "#eng")).build() +``` + +Tri-state reconcile: `null` = leave untouched, empty list = purge, non-empty = upsert + prune others. Lifecycle via `runtime.schedules()`: `save`, `get`, `list`, `pause`, `resume`, `delete`, `runNow`, `previewNext(cron, n)`, `reconcile`. See [sentinel-agents.md](sentinel-agents.md). + +### 2.11 Exceptions + +`AgentspanError` (base), `AgentAPIError` (server error), `AgentNotFoundError`, `ConfigurationError` (invalid config). Credential errors: `CredentialNotFoundError`, `CredentialAuthError`, `CredentialRateLimitError` (120 calls/min), `CredentialServiceError`. + +### 2.12 Feature matrix (summary) + +The full contract is **~89 features**, each traceable: concept → Python reference module → wire-format key → server handler → kitchen-sink stage. There is no single authoritative per-row matrix in the repo; the grouped summary below is the working inventory. + +**SDK parity status.** All four SDKs (Python, TypeScript, Java, C#) are now at **essentially complete parity** — the recent cross-SDK fixes closed the former Java/C# gaps, leaving only one honest open item (the language-driven provider/framework asymmetries, below). The fixes: + +- **SemanticMemory** — now in **all four SDKs** (Java added). Reminder: it is a client-side keyword (Jaccard-overlap) store, not a vector DB; not serialized to the wire (§2.8). +- **ConversationMemory** — now in **all four SDKs** (C# added); wire shape `{messages, maxMessages}`. +- **`api_tool`** (OpenAPI/Swagger/Postman discovery) — now in **all four SDKs** (Java added; C# also gained the `${NAME}` credential-placeholder validation). +- **External guardrail** (`Guardrail.external`) — now in **all four SDKs** (C# added). +- **Tier-1 credential accessor** — now in **all four SDKs** (Python `get_secret`, TS `getCredential`, Java `ToolContext.getCredential`, C# `ToolContext.GetCredential` / `Secrets.Get`). +- **Default guardrail `onFail`** is now **`raise` uniformly** across all four; `human`+`input` is rejected at construction in all four. + +- **Code-execution executors** — now **full across all four SDKs**: Local / Docker / Jupyter / Serverless ship in Python, TS, Java, and C#. The only difference is the Jupyter *mechanism* (Python in-process `jupyter_client`; TS `jupyter run` CLI; Java/C# Jupyter Kernel Gateway over HTTP); the executor itself is present everywhere. See [tool-execution-and-credentials-design.md](tool-execution-and-credentials-design.md) §2.10. +- **Optional config-field gaps — closed.** C# now emits `synthesize`, `prefillTools`, `cliConfig` (with `workingDir`), `reasoningEffort`, `contextWindowBudget`, and `maskedFields`; TS now emits `reasoningEffort`, `contextWindowBudget`, and `maskedFields`. All four SDKs now emit the full optional-field set. +- **Server `maskedFields` — now applied.** All four SDKs emit `maskedFields`, and the server now wires it into the compiled `WorkflowDef` (in `AgentCompiler.compile()`'s shared post-processing step, so it covers every compile shape; recursively-compiled sub-agents carry their own masked fields). The named input/output fields are redacted in execution history/UI. (Previously the field was accepted but dropped.) + +The one remaining honest open item: + +- **Provider/framework asymmetries (language-support driven):** the `claude-code` model + `ClaudeCode` config are Python/TS only; the Claude Agent SDK framework is Python-only; Vercel AI SDK is TS-only; OCG is Python-only. + +Treat the matrix below as the shared reference set; the only remaining delta is the open item above. + +| Group | Features (count) | Examples | +|---|---|---| +| Agent core | Agent def, `>>` chaining, structured output, introductions, metadata | #1, 30, 38, 39 | +| Strategies (9) | handoff, sequential, parallel, router, round_robin, random, swarm, manual, plan_execute | #2–9, +plan_execute | +| Tools (12 documented + internal `pull_workflow_messages`) | worker, http, api, mcp, agent_tool, human, generate_image/audio/video/pdf, rag_search/index | #10–16, 89 | +| Tool features | approval (HITL), ToolContext, credentials, tool-guardrails, external | #17–21 | +| Guardrails | regex, llm, custom, external × onFail retry/raise/fix/human | #22–29 | +| Memory | ConversationMemory, SemanticMemory | #31, 32 | +| Control flow | termination (composable), handoffs (3), allowed transitions, gate, stop_when, required_tools | #33–37, 71, 77, 70 | +| HITL | approve, reject, feedback | #40–42 | +| Streaming / exec | SSE, async stream, polling fallback, run/async, start, deploy, serve, plan | #43–51 | +| Credentials (7 modes) | isolated, in-process, CLI, HTTP header, MCP, framework, external | #52–57 | +| Code exec (4) | local, docker, jupyter, serverless | #58–61 | +| Advanced agent | callbacks, PromptTemplate, token tracking, thinking, include_contents, planner, CLI config, context condensation | #62–73 | +| SDK utilities | scatter_gather, agent discovery, OTel tracing | #74–76 | +| Testing (6) | mock_run, expect, assertions, record/replay, strategy validators, eval runner | #78–83 | +| Validation (4) | runner, judge, native execution, HTML report | #84–87 | +| Distributed | external agent | #88 | + +A new SDK is **feature-complete** when all ~89 are implemented (parity across the four shipped SDKs is now essentially complete — see the parity-status note above; the only standing caveat is the language-driven provider/framework asymmetries, not an SDK feature gap), the kitchen sink produces identical `AgentConfig` JSON and executes end-to-end, both sync and async APIs work, the validation report generates, and all Python examples are ported (§5). + +--- + +## 3. Serialization, Workers & Control Plane + +The SDK's core job is to serialize the agent tree into the `AgentConfig` JSON the server compiles into a Conductor `WorkflowDef`. **Producing identical JSON for equivalent definitions is the primary correctness criterion.** See `agent-schema.json` / `agent-schema.md` for the formal wire contract and `agent-structure.md` for the field → JSON-key mapping; this section synthesizes the rules. + +### 3.1 Top-level AgentConfig + +```json +{ + "name": "agent_name", + "model": "provider/model_name", + "strategy": "handoff|sequential|parallel|router|round_robin|random|swarm|manual|plan_execute", + "maxTurns": 25, + "timeoutSeconds": 300, + "external": false, + "instructions": "string | { prompt_template } | null", + "tools": [ ToolConfig... ], + "agents": [ AgentConfig... ], + "router": "AgentConfig | { taskName }", + "outputType": { "schema": {...}, "className": "MyModel" }, + "guardrails": [ GuardrailConfig... ], + "memory": { "messages": [...], "maxMessages": 50 }, + "maxTokens": 4096, "temperature": 0.7, + "stopWhen": { "taskName": "agent_name_stop_when" }, + "termination": TerminationConfig, + "handoffs": [ HandoffConfig... ], + "allowedTransitions": { "agent_a": ["agent_b"] }, + "introduction": "...", "metadata": { "key": "value" }, + "enablePlanning": true, + "planner": AgentConfig, "fallback": AgentConfig, "fallbackMaxTurns": 5, + "callbacks": [ { "position": "before_agent", "taskName": "agent_name_before_agent" } ], + "includeContents": "default|none", + "thinkingConfig": { "enabled": true, "budgetTokens": 1024 }, + "requiredTools": ["tool_a"], + "gate": GateConfig, + "codeExecution": { "enabled": true, "allowedLanguages": ["python"], "allowedCommands": ["python3"], "timeout": 30 }, + "cliConfig": { "enabled": true, "allowedCommands": ["git","gh"], "timeout": 30, "allowShell": false }, + "credentials": ["GITHUB_TOKEN"] +} +``` + +**Rules:** all keys are **camelCase**; omit `null`-valued keys; `agents` is recursive; `strategy` is emitted only when `agents` is non-empty (or PLAN_EXECUTE slots exist); `synthesize` is emitted only when `false`. Dynamic instructions resolve at serialize time. + +### 3.2 ToolConfig + +```json +{ + "name": "tool_name", + "description": "...", + "inputSchema": { "type": "object", "properties": {...}, "required": [...] }, + "toolType": "worker|http|api|mcp|agent_tool|human|generate_image|generate_audio|generate_video|generate_pdf|rag_search|rag_index", + "outputSchema": {...}, + "approvalRequired": true, + "timeoutSeconds": 0, + "config": { "url": "...", "method": "GET", "headers": {"Authorization": "Bearer ${API_KEY}"}, "credentials": ["API_KEY"] }, + "guardrails": [ GuardrailConfig... ] +} +``` + +**Execution model** (which side runs it, whether an SDK worker is needed): + +| toolType | Conductor task | SDK worker? | +|----------|---------------|-------------| +| `worker` | SIMPLE | **Yes** (or none ⇒ external/remote) | +| `http` / `api` | HTTP (`api` via `LIST_API_TOOLS` discovery) | No | +| `mcp` | CALL_MCP_TOOL | No | +| `agent_tool` | SUB_WORKFLOW | Depends on sub-agent | +| `human` | HUMAN | No | +| `generate_image/audio/video/pdf` | GENERATE_* | No (server-only) | +| `rag_search` / `rag_index` | LLM_SEARCH_INDEX / LLM_INDEX_TEXT | No (server-only) | + +### 3.3 GuardrailConfig + +```json +{ "name": "...", "position": "input|output", "onFail": "retry|raise|fix|human", "maxRetries": 3, + "guardrailType": "regex|llm|custom|external", + "patterns": ["\\b\\d{3}-\\d{2}-\\d{4}\\b"], "mode": "block|allow", "message": "...", + "model": "openai/gpt-4o", "policy": "...", "maxTokens": 100, "taskName": "guardrail_worker_name" } +``` + +`regex` → server INLINE JS (patterns/mode/message); `llm` → server LLM_CHAT_COMPLETE (model/policy/maxTokens); `custom` → SDK SIMPLE worker (taskName); `external` → remote SIMPLE (taskName, no local worker). + +### 3.4 Other config shapes + +- **TerminationConfig** (composable): `{"type":"text_mention","text":"DONE","caseSensitive":false}`, `stop_message`, `max_message`, `token_usage`, plus `{"type":"and|or","conditions":[…]}`. +- **HandoffConfig**: `on_tool_result` (toolName, resultContains), `on_text_mention` (text), `on_condition` (taskName). +- **PromptTemplate instructions**: `{"type":"prompt_template","name":"...","variables":{...},"version":1}`. +- **GateConfig**: `{"type":"text_contains","text":"APPROVED","caseSensitive":true}` or `{"taskName":"agent_name_gate"}`. +- **OutputType**: `{"schema":{…JSON Schema…},"className":"ArticleScore"}`. + +### 3.5 PLAN_EXECUTE — typed plan builders + `Ref` + +`PLAN_EXECUTE` (a.k.a. PAC/PAE) splits a task into a **planner** agent that emits a JSON DAG of operations and a server-compiled deterministic Conductor sub-workflow. Every SDK exposing it must provide: a `plan_execute` strategy value; `planner` (required) + `fallback` (optional) sub-agent slots (full `AgentConfig`, not booleans); typed builders `Plan`/`Step`/`Op`/`Generate`/`Validation`/`Action`; a `Ref(stepId)` helper; and a `run(agent, prompt, plan=…)` overload forwarding the plan as `static_plan`. + +Plan wire shape (must match byte-for-byte across SDKs for round-tripping): + +```json +{ + "steps": [ + { "id": "", "depends_on": [""], "parallel": false, + "operations": [ + { "tool": "", "args": { } }, + { "tool": "", "generate": { "instructions": "...", "output_schema": "...", "max_tokens": 4096, "context": "..." } } + ] } + ], + "validation": [ { "tool": "", "args": {...}, "success_condition": "$.passed === true" } ], + "on_success": [ ... ], "on_failure": [ ... ] +} +``` + +`Ref("step_id")` wires the whole output of an upstream step into a downstream arg; the serializer walks every plan-value tree (`Op.args`, `Generate.context`, `Validation.args`, `Action.args`) and replaces `Ref` with `{"$ref": "step_id"}`. **Validation rules (hard errors):** self-refs; refs to a non-existent step; refs to a step not in `depends_on`. `Op` takes literal `args` **or** a `Generate` (per-op LLM call). `Context.text(...)` / `Context.url(...)` supply planner reference material (URLs fetched per run; support `${CRED_NAME}`). + +**`static_plan`** (skip the planner LLM): forward the supplied plan as top-level `static_plan` on `POST /api/agent/start`. The server reads `workflow.input.static_plan` as highest-priority Case-0 and discards the planner's output. Use for tests, replays, and externally-planned pipelines. + +### 3.6 Workers + +Local tool functions, callbacks, guardrails, and termination/gate/stop conditions are registered as Conductor workers that poll for tasks. Walk the agent tree (sub-agents, router, agent-tools) and register every local handler. + +**How a tool becomes a worker:** generate a task definition → register a worker that receives JSON input, extracts `__agentspan_ctx__`, resolves credentials, deserializes args (coercing types, §3.9), calls the user function, serializes the return → start a poll loop reporting success/failure to Conductor. + +**Worker configuration:** poll interval 100ms; threads 1; daemon true; task-def `timeoutSeconds` **MUST be 0** (agent-level `timeoutSeconds` controls duration — a hardcoded task timeout prematurely kills long agents); `responseTimeoutSeconds` 3600 (Conductor minimum is 1s); retry count 2, delay 2s, LINEAR_BACKOFF. + +**System worker names** (server expects these exactly; collected recursively through nested/`agent_tool` agents): + +| Worker | Name pattern | Created when | +|---|---|---| +| Tool | `{tool.name}` | `@tool` function | +| Tool-level guardrail | `{guardrail.name}` | tool has guardrails | +| Output guardrail wrapper | `{agent}_output_guardrail` | agent has custom guardrails | +| stop_when / termination / gate | `{agent}_stop_when` / `_termination` / `_gate` | the field is set/callable | +| check_transfer | `{agent}_check_transfer` | agent has tools AND sub-agents | +| router_fn | `{agent}_router_fn` | ROUTER + callable router | +| handoff_check | `{agent}_handoff_check` | non-empty `handoffs` | +| process_selection | `{agent}_process_selection` | MANUAL | +| Callback | `{agent}_{position}` | callback handler for that position | + +**Stateful agents** get a per-execution domain (a `runId` UUID) used as `taskToDomain`; register their workers under that domain so concurrent runs don't dequeue each other's tasks. An agent is stateful if `stateful=true`, any tool is stateful, or any descendant is. See [stateful-agents.md](stateful-agents.md). + +**External workers** (by reference): emit the task name, register no local worker, trust a remote worker to pick it up. + +**Circuit breaker:** disable a tool after 10 consecutive failures (per tool name, module-level, persists across workflows); reset on any success or via `reset_circuit_breaker(name)`. When open, throw immediately. + +### 3.7 Control-plane API + +All calls go through the Conductor `ApiClient`; map transport errors to typed SDK exceptions (not-found vs. generic). Base URL `{server_url}/agent`. Full endpoint detail in [api-design.md](api-design.md); platform context in [agentspan-design.md](agentspan-design.md). + +| Method | Endpoint | Notes | +|---|---|---| +| compile | `POST /api/agent/compile` | returns `workflowDef`, no execution | +| deploy | `POST /api/agent/deploy` | compile + register; returns `registeredName` + `workflowDef`, **no** `executionId` | +| start | `POST /api/agent/start` | returns `{executionId, registeredName}` | +| status | `GET /api/agent/{executionId}/status` | poll | +| respond (HITL) | `POST /api/agent/{executionId}/respond` | `{approved}` / `{approved,reason}` / `{message}` | +| stream | `GET /api/agent/stream/{executionId}` | SSE; `Last-Event-ID` reconnect | +| events (framework push) | `POST /api/agent/{executionId}/events` | workers push intermediate events | +| list / search / detail | `GET /api/agent/list`, `/executions`, `/execution/{id}` | | +| delete | `DELETE /api/agent/{name}` | | + +**Start payload** carries the compiled `agentConfig` (or `framework`+`rawConfig`), `prompt`, and optional fields. Presence rules: `sessionId` **always present** (empty string if unset); `media` **always present** (empty array); `idempotencyKey` only if provided; `timeoutSeconds`/`credentials`/`static_plan` only if provided. + +```json +{ "agentConfig": {...}, "prompt": "...", "sessionId": "", "media": [], + "idempotencyKey": "optional", "timeoutSeconds": 300, "credentials": ["CRED_A"] } +``` + +**Idempotency:** `idempotencyKey` → Conductor `correlationId`. Server searches RUNNING/COMPLETED (not FAILED) workflows with the same agent name + correlationId; returns the existing `executionId` if found, else creates a new execution. Failed workflows are **not** deduplicated. `correlationId` is also auto-generated by the SDK as a UUID per call for client-side tracing. + +### 3.8 SSE wire format + +``` +event: → AgentEvent.type +id: → reconnection cursor +data: → AgentEvent fields (blank line ends the event) +: → heartbeat (ignore; sent every 15s) +``` + +Reconnect with `Last-Event-ID`; the server replays from a 200-event / 5-min buffer. **Framework event push** to `POST /agent/{id}/events` supports exactly 6 types — `thinking`, `tool_call`, `tool_result`, `context_condensed`, `subagent_start`, `subagent_stop` — unknown types are silently dropped. + +### 3.9 Type coercion (worker dispatch) + +Coerce tool inputs from Conductor's type system to the target language, applied **in order**, all failures **silent** (return original, never throw): (1) null/unknown → unchanged; (2) unwrap `Optional` and recurse; (3) already-matching → unchanged; (4) string→list/dict via `JSON.parse` (fallback to string); (5) dict/list→string via `JSON.stringify` (AI_MODEL args arrive parsed; tools wanting JSON strings must re-serialize); (6) string→int/float/bool (`"true"/"1"/"yes"`→true, `"false"/"0"/"no"`→false); (7) fallback unchanged. + +### 3.10 Other server contracts SDKs must honor + +- **ToolContext.state capture:** append non-empty post-execution `state` to the result under `_state_updates` (merged into a dict result, or wrapped as `{"result": , "_state_updates": {...}}`). The server persists and strips it. +- **Execution token extraction:** primary `task.input_data.__agentspan_ctx__.execution_token`, fallback `task.workflow_input.…`. Strip `__agentspan_ctx__` before calling the tool. See [tool-execution-and-credentials-design.md](tool-execution-and-credentials-design.md) and `secret-injection-contract.md`. +- **required_tools** wraps the agent loop in an outer DO_WHILE (≤3 outer iterations) — can triple worst-case execution time. +- **Class-instance normalization:** if an SDK type has a `toGuardrailDef()`/`toJSON()`/`toWireFormat()` method, the serializer must call it before reading properties (duck-typed). + +### 3.11 Sync + async dual model + +Every execution API has sync and async variants; the internal implementation should be async-native with blocking sync wrappers. + +| Language | Async primitive | Sync wrapper | +|---|---|---| +| Python | `asyncio` | `asyncio.run()` in thread | +| TypeScript | `Promise` | inherently async | +| Go | goroutines + channels | blocking default | +| Java | `CompletableFuture` / virtual threads | `.get()`/`.join()` | +| Kotlin | `suspend` / coroutines | `runBlocking {}` | +| C# | `Task` | `.GetAwaiter().GetResult()` | +| Ruby | `Async` / `Fiber` | blocking default | + +--- + +## 4. Skills (agentskills.io directories as agents) + +Skills make an [agentskills.io](https://agentskills.io/specification)-compatible directory a **first-class `Agent`** — composable, durable, observable. `skill("./dg")` works out of the box: convention-based discovery, no manifest. Because `skill()` returns `Agent`, skills mix freely with regular and framework agents (`>>`, `agent_tool()`, strategy teams, deploy, serve, stream). + +**Thin SDK, thick server.** All parsing/normalization lives server-side in a `SkillNormalizer` (alongside the framework normalizers); the SDK just reads the directory, packages contents, registers script + `read_skill_file` workers (which must run user-side), and sends `{"framework": "skill", "rawConfig": {...}}` to the server. This keeps every SDK's footprint ~260 LOC. + +A skill directory: + +``` +skill-name/ +├── SKILL.md # Required: YAML frontmatter + markdown body +├── *-agent.md # Optional: each becomes a sub-agent +├── scripts/ # Optional: each executable → a named worker tool +├── references/ examples/ assets/ # Optional: read on demand via read_skill_file +``` + +**Discovery (convention-based):** `SKILL.md` frontmatter → metadata, body → orchestrator instructions; `*-agent.md` → sub-agents (filename minus `-agent.md` = name); `scripts/*` → named worker tools; `references/examples/assets/*` and other root files → paths listed, served via `read_skill_file` (an `enum`-constrained worker so the LLM can only read files that exist). Cross-skill references are matched against the search path (sibling dirs → `./.agents/skills/` → `~/.agents/skills/` → explicit `searchPath`) and packaged recursively (with cycle detection). + +**SDK surface:** `skill(path, model[, agentModels, params, searchPath])` and `loadSkills(dir, model)` for a tree. Sub-agents inherit the orchestrator's model unless overridden per-agent. Script contents and resource file contents are **not** sent to the server — scripts run as local workers; resources are read on demand. The server's `SkillNormalizer` builds the orchestrator `AgentConfig`, wraps sub-agents and cross-skill refs as `agent_tool` (→ SUB_WORKFLOW), and emits script/`read_skill_file` tools as `worker` (→ SIMPLE). Worker task names are prefixed with the skill name (`dg__read_skill_file`) to avoid collisions when composing skills. No changes to `AgentCompiler`/`ToolCompiler`/`MultiAgentCompiler` — the normalizer produces the structure they already handle. + +Each script call and file read is a distinct named Conductor task with full I/O, timing, retry, and crash recovery (the execution DAG resumes from the last completed task). A registry CLI (`agentspan skill register/list/get/pull/delete`, plus `run/load/serve`) stores immutable skill packages server-side. + +This section is the consolidated skills design (normalizer steps, execution traces, registry API, edge cases). + +--- + +## 5. Per-Language Guides, Implementations & Testing + +### Framework bridges + +Adapt native framework objects into the `Agent` model and send them via the `framework` + `rawConfig` path so the server's matching normalizer handles them. The runtime's `run/start/stream/deploy/serve/plan/resume` accept the raw native object and coerce it (detect by fully-qualified type name so the core never hard-references an optional dependency). **There is no passthrough** — every framework agent is compiled to a full AgentConfig → Conductor workflow with individual tasks per tool/LLM-call/sub-agent (durable, observable). Framework packages are optional dev/peer dependencies. + +Supported bridges: OpenAI Agents SDK, Google ADK (`BaseAgent`/`LlmAgent`), LangChain / LangGraph, Vercel AI SDK (TS). OpenAI and Google ADK expose model/tools/instructions as public properties (zero user changes); JS frameworks that hide them in closures (Vercel AI `generateText`, LangGraph `createReactAgent`, LangChain `AgentExecutor`) use **drop-in import wrappers** — one import change captures model/tools at creation time. Detection order must check native `Agent` first, then framework markers. Full extraction rules per framework: [framework-integration.md](framework-integration.md) (and `langchain-integration.md`). + +### Per-language idiom guides + +> **Shipped vs. guide-only.** Only **four SDKs ship today**: Python, TypeScript, Java, and C# (`sdk/python`, `sdk/typescript`, `sdk/java`, `sdk/csharp`). The **Go, Kotlin, and Ruby** docs are *translation guides only* — idiom references for a future port; there is no published Go/Kotlin/Ruby SDK, so their coordinates/namespaces in those guides are illustrative, not authoritative. + +Each language doc covers project setup, type-system mapping, the decorator/annotation pattern, async model, worker + SSE implementation, error handling, the testing framework, and a kitchen-sink translation. Reference type/pattern mappings: + +| Python | TS | Go | Java | Kotlin | C# | Ruby | +|--------|-----|-----|------|--------|-----|------| +| `dataclass` | interface/class | struct | record/POJO | data class | record | Struct/Data | +| `>>` | `.pipe()` | `Pipeline()` | `.then()` | `then` infix | `>>` overload | `>>` | +| `&`/`\|` | `.and()`/`.or()` | `And()`/`Or()` | `.and()`/`.or()` | `and`/`or` infix | `&`/`\|` | `&`/`\|` | +| `@tool` | `@Tool()`/`tool()` | `Tool()` option | `@Tool` | `tool {}` DSL | `[Tool]` | `tool` method | + +Guides: [java](sdk-design/languages/java.md) · [typescript](sdk-design/languages/typescript.md) · [csharp](sdk-design/languages/csharp.md) · [go](sdk-design/languages/go.md) · [kotlin](sdk-design/languages/kotlin.md) · [ruby](sdk-design/languages/ruby.md). + +### Concrete implementation references + +All four shipped SDKs have detailed reference-implementation write-ups (source layout, serializer, worker manager, SSE client, language-specific gotchas): [python](sdk-design/languages/python-implementation.md) (the reference SDK), [typescript](sdk-design/languages/typescript-implementation.md), [java](sdk-design/languages/java-implementation.md), and [csharp](sdk-design/languages/csharp-implementation.md). The TypeScript audit surfaced the recurring risks every new SDK should check: + +1. **Worker-registration parity is the #1 risk** — every `taskName` the serializer emits must have a registered worker. After writing the serializer, grep all `taskName` references and verify each has a matching registration (termination, custom guardrail, stop_when, callbacks, gate, router_fn were all initially missed). +2. **Normalize class instances** before serializing (call `toGuardrailDef()` etc.). +3. **Bridge callback worker args** to typed handler signatures (supply `agentName` from the closure). +4. **Termination needs `shouldTerminate()`**, not just `toJSON()`. +5. **Some Python "SDK-side" workers are server-side for other SDKs** (check_transfer, handoff_check, swarm transfer, manual selection) — verify by running the example without the worker; don't add conflicting ones. +6. **Register tool-level guardrails as well as agent-level.** + +### Acceptance testing + +The **kitchen sink** is the single acceptance test — one mega-workflow (9 stages: intake/router, parallel research, sequential writing, guardrails, HITL, multi-strategy translation/discussion, handoff publishing, analytics/media/RAG, all execution modes) exercising every feature plus all cross-cutting concerns (7 credential modes, CLI config, code execution, thinking, include_contents, planner, metadata, context condensation). A new SDK **passes** when it: produces identical `AgentConfig` JSON for the same tree; workers execute all tool/guardrail/callback tasks; SSE yields the same event sequence; HITL completes; the final `AgentResult` matches; all assertions pass; and the LLM judge scores ≥ threshold on the quality rubrics. Spec + rubrics: [sdk-design/kitchen-sink.md](sdk-design/kitchen-sink.md). + +Each SDK ships a **testing framework** mirroring Python: `mock_run()` (no server), an `expect()` fluent API (`expect(result).completed().outputContains("article")`), `assert_*` helpers (`assertToolUsed`, `assertGuardrailPassed`), `record()`/`replay()`, strategy validators, and an LLM-judge eval runner. Per CLAUDE.md, **do not use an LLM for validation except when judging output quality/evals**; structural and behavioral assertions must be deterministic. + +A **validation framework** (concurrent runner, TOML config, example groups, LLM judge, HTML report, resume/retry) runs every ported example against multiple models and — validation-only, never a runtime dependency — compares Agentspan-compiled vs. native-framework execution for semantic equivalence. Designs: see the [validation methodology overview](validation/README.md) and the per-SDK docs — [python](validation/python-validation.md), [typescript](validation/typescript-validation.md), [java](validation/java-validation.md), [csharp](validation/csharp-validation.md). + +**Example parity:** every SDK ports **all** Python examples — ~97 native + framework examples (LangGraph 44, LangChain 25, OpenAI 10, ADK 35; Vercel AI 10 for TS) — using the same numbering, translated to idiomatic patterns. **Hard rule:** framework examples must import and use the **real** native SDK (never mocks); if a package can't be installed, omit the example entirely and file a tracking issue — a missing example is honest, a mock is misleading. + +### Implementation order + +Configuration → HTTP client → Agent + Tool types → serialization → worker system → runtime (run/start/deploy) → SSE streaming → credentials → guardrails → memory → termination + handoffs → code execution → extended types → callbacks → framework integration → testing framework → validation framework → kitchen sink → examples. Audit each new SDK with the 3-pass methodology: (1) feature coverage / missing worker registrations, (2) edge cases / signature + normalization gaps, (3) end-to-end trace of 2–3 examples through the full pipeline. + +### Reference docs (wire/platform detail) + +- `agent-schema.md` / `agent-schema.json` — formal wire contract +- `agent-structure.md` — Agent field → JSON-key mapping and serialization rules +- `agent-client-api.md` — control-plane client (compile/deploy/start/status/respond) +- `agent-runtime-api.md` — runtime, streaming, and HITL semantics +- [api-design.md](api-design.md), [agentspan-design.md](agentspan-design.md) — REST/SSE and platform +- [guardrails-design.md](guardrails-design.md), [tool-execution-and-credentials-design.md](tool-execution-and-credentials-design.md), [framework-integration.md](framework-integration.md) diff --git a/docs/sdk-design/kitchen-sink.md b/design/sdk-design/kitchen-sink.md similarity index 99% rename from docs/sdk-design/kitchen-sink.md rename to design/sdk-design/kitchen-sink.md index 66b4b5b19..deec7a0aa 100644 --- a/docs/sdk-design/kitchen-sink.md +++ b/design/sdk-design/kitchen-sink.md @@ -2,7 +2,7 @@ ## Overview -A single mega-workflow that processes an article request through a complete publishing pipeline, exercising every Agentspan SDK feature (89 features per the traceability matrix in `2026-03-23-multi-language-sdk-design.md` Section 11). +A single mega-workflow that processes an article request through a complete publishing pipeline, exercising every Agentspan SDK feature (89 features per the traceability matrix in `design/sdk-design.md` Section 11). **Reference implementation:** `sdk/python/examples/kitchen_sink.py` diff --git a/design/sdk-design/languages/csharp-implementation.md b/design/sdk-design/languages/csharp-implementation.md new file mode 100644 index 000000000..933fea27e --- /dev/null +++ b/design/sdk-design/languages/csharp-implementation.md @@ -0,0 +1,285 @@ +# C# SDK — Reference Implementation + +**Status:** Created 2026-06-26 + +**Scope:** This document describes the .NET implementation of the Agentspan SDK as a *reference implementation* of the cross-language SDK contract. It is written from the actual source under `sdk/csharp/src/`. It covers the package layout, the compilation/serialization model, the runtime lifecycle, worker dispatch internals, the streaming/SSE client, guardrails and credentials, the framework adapters (OpenAI / Google ADK / Semantic Kernel), the C#-specific design choices, and the test layout. The SDK contract itself lives in [`../../sdk-design.md`](../../sdk-design.md); the server-side compilation model in [`../../agentspan-design.md`](../../agentspan-design.md); the HTTP control-plane in [`../../api-design.md`](../../api-design.md); framework normalizers in [`../../framework-integration.md`](../../framework-integration.md); and the secret/worker contract in [`../../tool-execution-and-credentials-design.md`](../../tool-execution-and-credentials-design.md). For idiomatic usage and the public surface, see the sibling idiom guide [`csharp.md`](csharp.md). Python is the canonical reference SDK; see [`python-implementation.md`](python-implementation.md) for the structure this document mirrors. + +--- + +## 1. Overview + +The C# SDK lets a developer declare an `Agent` (an LLM + tools, or a multi-agent system) and execute it as a durable Conductor workflow. The SDK never interprets the agent locally: it serializes the agent tree to JSON, POSTs it to the server's `/api/agent/*` endpoints, and the server-side Java compiler turns it into a Conductor `WorkflowDef`. The SDK's runtime responsibility is narrow but essential — register and poll **local tool workers**, and poll/stream execution state. + +Distribution: + +| Package (NuGet `PackageId`) | Assembly / namespace | Purpose | +|---|---|---| +| `conductor-agent-sdk` | `Conductor.AI` | Core SDK — `Agent`, tools, runtime, client, workers | +| `conductor-agent-sdk-openai` | `Conductor.AI.OpenAI` | OpenAI Agents SDK shape adapter | +| `conductor-agent-sdk-google-adk` | `Conductor.AI.GoogleADK` | Google ADK shape adapter | +| `conductor-agent-sdk-semantic-kernel` | `Conductor.AI.SemanticKernel` | Microsoft Semantic Kernel plugin bridge | + +- **Target framework:** `net10.0` (all projects). `Nullable` and `ImplicitUsings` enabled; `LangVersion=latest`. +- **Core dependencies:** `conductor-csharp` 1.1.4 (brings `Conductor.Client.Configuration`, `TaskResourceApi`, Newtonsoft.Json, and `Microsoft.Extensions.Logging.Abstractions` transitively) and `Newtonsoft.Json` 13.0.3 pinned. `System.Text.Json` and `System.Threading.Channels` are in-box on .NET 10. +- **Design principles** (shared with the reference SDK): everything is an `Agent`; server-first execution; compile-don't-interpret; zero-config for simple cases; Conductor-native mapping. + +--- + +## 2. Package & source layout + +All under `sdk/csharp/src/`. + +### Core — `Conductor.AI/` + +| File | Responsibility | +|---|---| +| `Agent.cs` | The `Agent` primitive, `Strategy` enum, `AgentBuilder` fluent builder, `Agent.ScatterGather`, and the `>>` sequential-pipeline operator. | +| `AgentConfigSerializer.cs` | `internal static` — serializes the `Agent` tree to the server wire JSON. The heart of compile-don't-interpret. | +| `AgentRuntime.cs` | Primary entry point. Owns worker orchestration + control-plane client. `IAsyncDisposable`. | +| `AgentClient.cs` | Control-plane HTTP client for `/agent/*` (compile/deploy/start/status/respond/stream) and `/workflow/*`. | +| `AgentAuth.cs` | `AgentAuthHandler` — `DelegatingHandler` that mints/caches a JWT and attaches `X-Authorization`. | +| `WorkerManager.cs` | `WorkerPollLoop` (per-task-type poller) + `WorkerManager` (registers all workers from an agent tree). | +| `Tool.cs` | `ToolAttribute`, `ToolDef`, `ToolContext` (now exposes the tier-1 `GetCredential(name)` accessor), reflection-based `ToolRegistry`, and tool factories (`HttpTools`, `McpTools`, `RagTools`, `MediaTools`, `HumanTool`, `WaitForMessageTool`, `ApiTools` — OpenAPI/Swagger/Postman discovery, now with `${NAME}` credential-placeholder validation, `CliTool`, `AgentTool`, `ToolDefFactory`). | +| `Result.cs` | `AgentResult`, `AgentStatus`, `AgentEvent`, `AgentHandle`, value records (`TokenUsage`, `DeploymentInfo`, …), and the `EventType`/`Status`/`FinishReason`/`OnFail`/`Position` enums. | +| `CredentialInjection.cs` | Process-wide-lock env-var injection for tier-2 credential passthrough. | +| `Guardrail.cs`, `Handoff.cs`, `Termination.cs`, `Gate.cs`, `Callback.cs`, `Skill.cs`, `Plans.cs`, `ConversationMemory.cs`, `SemanticMemory.cs`, `LocalCodeExecutor.cs`, `DockerCodeExecutor.cs`, `JupyterCodeExecutor.cs`, `ServerlessCodeExecutor.cs`, `Tracing.cs`, `CredentialInjection.cs`, `Exceptions.cs`, `AgentDef.cs`, `GPTAssistantAgent.cs` | Feature modules: guardrail defs (custom / `RegexGuardrail` / `LLMGuardrail` / `Guardrail.External`), SWARM handoff triggers, termination conditions, sequential gate, lifecycle callback handlers, skills, deterministic plans (PLAN_EXECUTE), `ConversationMemory` (wire shape `{messages, maxMessages}`) and `SemanticMemory` (client-side keyword/Jaccard store, not serialized), the full code-executor family — `LocalCodeExecutor` (subprocess + temp file), `DockerCodeExecutor` (container), `JupyterCodeExecutor` (stateful kernel via a Jupyter Kernel Gateway over HTTP), `ServerlessCodeExecutor` (HTTP endpoint) — tracing, exception hierarchy, declarative `[AgentDef]` discovery. | +| `Scheduling/` | `Schedule`, `Schedules` (cron lifecycle: save/list/pause/resume/delete/runNow/preview/reconcile), `ScheduleException`. | + +### Adapters + +Each adapter is a thin project that references `Conductor.AI` and produces an `Agent` with a `Framework` tag set: + +- `Conductor.AI.OpenAI/OpenAIAgent.cs` — builder producing `Framework = "openai"`. +- `Conductor.AI.GoogleADK/GoogleADKAgent.cs` — builder producing `Framework = "google_adk"`. +- `Conductor.AI.SemanticKernel/SemanticKernelAgent.cs` — references `Microsoft.SemanticKernel` 1.76.0; produces a *plain* `Agent` (no framework tag) whose tools wrap `[KernelFunction]` methods. + +--- + +## 3. Compilation & serialization + +`Agent` objects are **never executed locally**. `AgentConfigSerializer` (in `AgentConfigSerializer.cs`) walks the agent tree and emits the JSON the server consumes; the server's Java `AgentCompiler` produces a Conductor `WorkflowDef`. See [`../../agentspan-design.md`](../../agentspan-design.md) for the server compilation model and [`../../api-design.md`](../../api-design.md) for the endpoints. + +``` +Agent ──AgentConfigSerializer.Serialize()──► JSON payload + │ + POST /api/agent/start (run/start) ▼ + POST /api/agent/compile (plan, dry-run) Server AgentCompiler + POST /api/agent/deploy (deploy, no exec) ──► Conductor WorkflowDef +``` + +### Two entry shapes + +`Serialize(agent, prompt, sessionId, media)` produces the **start** payload, and `SerializeAgent(agent)` produces the bare **agentConfig** used by `deploy`/`compile`. There are two wire envelopes: + +1. **Default envelope** — `{ agentConfig, prompt, sessionId, media }`. Used for plain agents. +2. **Framework envelope** — when `agent.Framework` is `"openai"`, `"google_adk"`, or `"skill"`, the start payload becomes `{ framework, rawConfig, prompt[, sessionId] }`, routed server-side to the matching normalizer (`OpenAINormalizer`, `GoogleADKNormalizer`). For `deploy`/`compile`, `AgentClient.FrameworkAwarePayload` inspects a `_framework` marker and wraps as `{ framework, rawConfig }`, else `{ agentConfig }`. + +### What `SerializeAgent` emits (selected, verify against source) + +- Scalar config: `model`, resolved `instructions` (via `Agent.ResolveInstructions()`, which evaluates `InstructionsFn` at serialize time — a prompt template is now nested **under `instructions`**, not emitted as a top-level `promptTemplate`), `maxTurns` (**defaults to 25 and is always emitted** so the server no longer silently applies its own 100 default), `maxTokens`, `temperature`, `timeoutSeconds`, `includeContents`, `introduction`, `external`, `enablePlanning`. +- **Newly-emitted optional config fields** (these close the former C# optional-field gap, bringing it to parity with the other SDKs): `synthesize` (emitted only when `false`), `prefillTools`, `cliConfig` (CLI allowlist, including `workingDir`), `reasoningEffort`, `contextWindowBudget`, and `maskedFields`. (`maskedFields` is emitted and the server now applies it to the compiled `WorkflowDef` — see [`../../sdk-design.md`](../../sdk-design.md) §2.12.) +- `thinkingConfig` — extended thinking is now serialized as a **nested object** `{enabled, budgetTokens}` (the earlier flat `thinkingBudgetTokens` key was a wire bug and has been removed), matching the other SDKs. +- PLAN_EXECUTE slots: `planner`, `fallback`, `fallbackMaxTurns`, and `plannerContext` (the latter *throws* `InvalidOperationException` at serialize time if set on a non-`PlanExecute` strategy — the last line of defence for that guard). +- `codeExecution` block when `LocalCodeExecution`/`CodeExecution`/`AllowedLanguages`/`AllowedCommands` is set, plus an injected `{agent.Name}_execute_code` worker tool so the LLM sees a callable function. +- `outputType` — `{ schema, className }` where the schema is produced by `System.Text.Json.Schema.JsonSchemaExporter.GetJsonSchemaAsNode(...)` over the CLR `Type`. +- `tools` (each via `SerializeTool`), `guardrails`, `agents` (recursive), `strategy` (via `StrategyToWire`, e.g. `RoundRobin → "round_robin"`, `PlanExecute → "plan_execute"`), `router`, `termination`, `allowedTransitions`, `metadata`, `handoffs`, `gate`, and lifecycle `callbacks` (one `{position, taskName}` per active hook, deduped). + +### Tool serialization (`SerializeTool`) + +Each `ToolDef` emits `{ name, description, inputSchema, toolType }`. `toolType` defaults to `"worker"` (or `"external"` when `External`). Notable rules verified in source: + +- Stateful routing: `stateful=true` is emitted when the agent or the tool is stateful, but only for `worker`/`external` tool types. +- Retry tuning is only emitted when it diverges from defaults (`retryCount != 2`, `retryDelaySeconds != 2`, `retryPolicy != "linear_backoff"`). +- **Credentials always land inside `config.credentials`** (never top-level) because the server's `AgentService.extractDeclaredCredentials` reads `tool.getConfig().get("credentials")`. The serializer merges credentials into the config object for every tool type. +- `agent_tool` embeds the child agent under `config.agentConfig` (and, for skill children, `config.workerNames`). + +--- + +## 4. Runtime lifecycle — `AgentRuntime` + `AgentClient` + +The SDK has two execution surfaces. **`AgentRuntime` is the primary entry point** — it owns local tool-worker orchestration *and* a backing `AgentClient`. **`AgentClient` is control-plane only**: its `RunAsync`/`StartAsync` compile + start + poll but do **not** register or poll local tool workers. Use the client directly only for LLM-only agents, remote tools (HTTP/MCP), or pre-deployed workflows; any agent with local `[Tool]` functions must run through `AgentRuntime`. + +`AgentRuntime` is `IAsyncDisposable`/`IDisposable` and is intended to be created with `await using`. + +### Configuration + +The constructor reads options or environment: + +| Setting | Env var | Default | +|---|---|---| +| Server URL | `AGENTSPAN_SERVER_URL` | `http://localhost:6767/api` | +| Auth key | `AGENTSPAN_AUTH_KEY` | (none → OSS anonymous) | +| Auth secret | `AGENTSPAN_AUTH_SECRET` | (none) | +| Worker poll interval (ms) | `AGENTSPAN_WORKER_POLL_INTERVAL` | 100 (min 1) | +| Worker thread count | `AGENTSPAN_WORKER_THREADS` | 1 (min 1) | + +It builds a `conductor-csharp` `Configuration` for worker polling; when key+secret are present it attaches `OrkesAuthenticationSettings` (JWT exchange). Connection/auth for the control plane is owned by the `AgentClient` via `AgentAuthHandler`. + +### Async API + sync wrappers + +The async methods are the source of truth; synchronous wrappers call `.GetAwaiter().GetResult()`: + +- `RunAsync(agent, prompt, …)` → `StartInternalAsync` → `handle.WaitAsync` → stop workers → `AgentResult`. +- `StartAsync(...)` → `AgentHandle` (for streaming / HITL). +- `RunByNameAsync` / `StartByNameAsync` — execute a pre-deployed workflow by name (no agentConfig payload; posts to `/workflow`). +- `StreamAsync(...)` — start then yield `AgentEvent`s, then stop workers. +- `DeployAsync(params Agent[])` and `DeployAsync(agent, schedules)` — CI/CD: compile + register without executing, optionally reconciling cron schedules. +- `ServeAsync(agent, ct)` — register local workers and block on `Task.Delay(Timeout.Infinite, ct)` until cancelled (the workflow must already be deployed). +- `PlanAsync(agent)` — dry-run compile (`POST /agent/compile`), returns the raw `JsonNode` WorkflowDef. +- `ResumeAsync(executionId, agent)` — re-attach across process restarts (see §5). +- HITL/WMQ: `GetStatusAsync`, `RespondAsync`, `ApproveAsync`/`RejectAsync` (both root and event-targeted), `SendMessageAsync` (Workflow Message Queue). +- `Schedules` — cron lifecycle, delegated to the client. + +### `StartInternalAsync` (the core path) + +1. If the agent (or any sub-agent / tool / router) is stateful (`HasStatefulTools`), generate a fresh per-execution domain `runId = Guid.NewGuid().ToString("N")`. This mirrors the Python runtime's `_has_stateful_tools` + `run_id = uuid.uuid4()`. +2. Lazily create the `WorkerManager`, `RegisterAgentTools(agent, runId)`, and `Start()`. +3. Serialize the start payload; attach `runId` and (for PLAN_EXECUTE) `static_plan`. +4. `POST /agent/start`; wrap the returned executionId in an `AgentHandle` (carrying the runId for domain-routed polling). + +After `RunAsync` completes, workers are disposed; `StartAsync`/`StreamAsync` leave them running for the caller's session. + +### Result polling — `AgentHandle.WaitAsync` + +`AgentHandle` polls `GET /agent/{id}/status` every **500 ms** until `COMPLETED`/`FAILED`/`TERMINATED`/`TIMED_OUT`, then fetches the full execution record (`GET /agent/execution/{id}`) for `tokenUsage` and `finishReason`, and builds an `AgentResult`. `finishReason` strings map to the `FinishReason` enum (e.g. `TOOL_CALL`/`TOOL_CALLS → ToolCalls`). + +--- + +## 5. Worker & dispatch internals + +Unlike Python (which has a single universal `dispatch_worker`), the C# SDK registers **one poll loop per task type** and lets the server route tool calls. Tools execute as Conductor worker tasks polled locally. + +### `WorkerManager.RegisterAgentTools(agent, domain?)` + +Recursively walks the agent tree and registers: + +- **Tool workers** — for every `ToolDef` whose `Handler` is non-null (`RegisterTools`). Remote/server-side tools (HTTP, MCP, RAG, media, human, WMQ) have no handler and are skipped — the server executes them. +- **Guardrail workers** — local guardrail functions, wrapped to emit the `{passed, message, on_fail, fixed_output, guardrail_name, should_continue}` contract (`RegisterGuardrails`); also per-tool guardrails. +- **Callback workers** — `before/after_model` (bespoke signatures) and the generic kwargs-based `before/after_agent`, `before/after_tool`, plus any `CallbackHandler` overrides (`RegisterCallbacks`). +- **Local code execution worker** — `{agent.Name}_execute_code`, which delegates to `LocalCodeExecutor` (subprocess + temp file, interpreter table, timeout). The configured executor may instead be `DockerCodeExecutor`, `JupyterCodeExecutor`, or `ServerlessCodeExecutor`; all four ship in C#, matching the other SDKs. +- **Strategy workers** — SWARM transfer + `_check_transfer` + `_handoff_check` workers; MANUAL `_process_selection`; skill workers. + +### `WorkerPollLoop` + +Each loop spawns `_threadCount` concurrent `PollLoopAsync` tasks (so a slow handler doesn't stall siblings of the same type), each driven by a `PeriodicTimer` at the poll interval. It uses the `conductor-csharp` `TaskResourceApi.PollAsync(taskName, workerid: Environment.MachineName, domain)`. On a task: + +1. `ConvertInputData` bridges Newtonsoft → `System.Text.Json` `JsonElement`. +2. `ExtractToolContext` pulls `__agentspan_ctx__` (a `ToolContext`) and `_agent_state`; internal keys are stripped from the handler-visible args. +3. If the tool declares credentials, resolve them (`AgentClient.ResolveCredentialsAsync`) and run the handler inside `CredentialInjection.InjectViaEnvAsync` (§7). +4. Wrap primitives as `{ result: … }`; merge shared-state updates as `_state_updates`. +5. Report via `TaskResult` (Newtonsoft dict): `COMPLETED`, or `FAILED_WITH_TERMINAL_ERROR` for `TerminalToolException` and credential failures (configuration errors are non-retryable), or `FAILED` otherwise. + +### Reflection-based tool definition — `ToolRegistry.FromInstance` + +Scans an object's public methods for `[Tool]`. For each it builds a `ToolDef` with a name (`Attr.Name` or `ToSnakeCase(methodName)`), a JSON Schema inferred from parameters (`BuildInputSchema`), and a handler that coerces JSON args to CLR parameter types (`CoerceArg`, incl. string→int/bool/double coercion), injects `ToolContext` if a parameter matches, invokes the method, and unwraps `Task`/`Task`. External tools are skipped (no local handler). + +--- + +## 6. Streaming / SSE client + +`AgentClient.StreamEventsAsync(executionId, ct)` opens `GET /agent/stream/{id}` with `Accept: text/event-stream` and `HttpCompletionOption.ResponseHeadersRead`, then parses the SSE wire format line-by-line: + +- `:`-prefixed heartbeats are skipped. +- `event:` / `id:` / `data:` lines accumulate; a blank line flushes an event block. +- `ParseEvent` maps the `event:` name + JSON `data` to a typed `AgentEvent`. + +Event mapping (verified in `ParseEvent`): + +| SSE `event:` | `AgentEvent.Type` | Notable fields | +|---|---|---| +| `thinking` | `Thinking` | `Content` | +| `tool_call` | `ToolCall` | `ToolName` | +| `tool_result` | `ToolResult` | `ToolName` | +| `guardrail_pass` / `guardrail_fail` | `GuardrailPass` / `GuardrailFail` | `GuardrailName`, `Content` | +| `waiting` | `Waiting` | (HITL pause) | +| `handoff` | `Handoff` | `Target` | +| `done` | `Done` | `Status` (finishReason), `Content` | +| `error` | `Error` | `Content` | + +Iteration ends on a `done` event (`yield break`). The async iterator surfaces through `AgentHandle.StreamAsync` and `AgentRuntime.StreamAsync`. Events carry the emitting `ExecutionId`, which the event-targeted HITL helpers use so a sub-execution's HUMAN task is answered on its own execution rather than the root. + +--- + +## 7. Guardrails & credentials (SDK-side) + +### Guardrails + +`GuardrailDef` (in `Guardrail.cs`) carries `Name`, `Position` (`Input`/`Output`), `OnFail` (`Retry`/`Raise`/`Fix`/`Human`), `MaxRetries`, and an optional local `Handler`. The serializer emits `{ name, position, onFail, maxRetries, guardrailType: "custom", taskName }` (Conductor task name = guardrail name). Local guardrails register as workers; external guardrails (no handler) are referenced by name and run remotely. The worker wrapper enforces escalation: an `OnFail.Retry` that has reached `MaxRetries` (or `OnFail.Fix` with no `FixedOutput`) downgrades to `Raise`. The full guardrail compilation model is in [`../../guardrails-design.md`](../../guardrails-design.md). + +### Credentials & secret injection + +The SDK never hard-codes secrets. Declared credential names travel inside `config.credentials` (§3). At dispatch time the worker resolves them and injects them for exactly one invocation: + +- **Resolution** — `AgentClient.ResolveCredentialsAsync(executionToken, names)` POSTs `{ token, names }` to `/workers/secrets`. The error contract is strict and matches Python's `WorkerCredentialFetcher`: empty names → empty dict (no HTTP); missing token → `CredentialNotFoundException`; 401 → `CredentialAuthException`; 429 → `CredentialRateLimitException`; 5xx/network → `CredentialServiceException`; a 200 missing any requested name → `CredentialNotFoundException`. It never silently returns empty values. +- **Injection (tier 1, preferred)** — resolved secrets are exposed to handler code via the tier-1 accessor `ToolContext.GetCredential(name)` (and the static `Secrets.Get(name)`), backed by a per-invocation context the worker populates before dispatch and clears afterward — secrets never enter task I/O. This is the no-lock, fully-concurrent path; pass keys explicitly to model clients. (The accessor was previously missing on C#; the SDK is now at parity with Python `get_secret` / TS `getCredential` / Java `ToolContext.getCredential`.) +- **Injection (tier 2, fallback)** — `CredentialInjection.InjectViaEnvAsync` holds a single **process-wide `SemaphoreSlim`** across mutation + invocation + restoration, so concurrent framework workers serialize instead of clobbering shared process-env. It is strictly serial within one process; scale by adding worker processes. See [`../../tool-execution-and-credentials-design.md`](../../tool-execution-and-credentials-design.md). + +### Control-plane auth — `AgentAuthHandler` + +`AgentClient` wraps a `DelegatingHandler` that attaches the control-plane auth header to every `/agent/*` request, mirroring the Python/TS SDKs: + +- No credentials → no header (OSS anonymous). +- Explicit key, no secret → the key is treated as a ready token. +- Key + secret → `POST {server}/token` mints a JWT, cached until ~30 s before its decoded `exp` (`DecodeJwtExp`), and sent as `X-Authorization`. The token mint uses a *separate* `HttpClient` so minting never recurses through the auth handler, and a `SemaphoreSlim` guards concurrent refresh. + +The e2e/test assembly is granted `InternalsVisibleTo("AgentspanE2eTests")` specifically so it can exercise `AgentAuthHandler`. + +--- + +## 8. Framework integration + +Adapters convert a framework-shaped declaration into a Agentspan `Agent`; the server normalizers (`OpenAINormalizer`, `GoogleADKNormalizer`) consume the resulting wire shape. See [`../../framework-integration.md`](../../framework-integration.md). + +- **OpenAI** (`Conductor.AI.OpenAI`) — `OpenAIAgent.Builder()` / `.From(...)` builds an `Agent` with `Framework = "openai"`. `Handoffs(...)` → `FrameworkConfig["handoffs"]`; `OutputType(name)` → `output_type`. Tools come from `[Tool]`-annotated objects via `ToolRegistry.FromInstance`. Models without a provider prefix are auto-prefixed `openai/` server-side. +- **Google ADK** (`Conductor.AI.GoogleADK`) — `GoogleADKAgent.Builder()` builds `Framework = "google_adk"`. Wire differences from OpenAI: `instruction` (singular, mapped by the serializer), `sub_agents` not `handoffs`, and bare models prefixed `google_gemini/` server-side. +- **Semantic Kernel** (`Conductor.AI.SemanticKernel`) — `SemanticKernelAgent.From(name, model, instructions, plugins…)` produces a *plain* `Agent` (no framework tag). It extracts tools from `KernelPlugin` instances or `[KernelFunction]` methods (`KernelPluginFactory.CreateFromObject`), builds schemas from `KernelFunctionMetadata`, and the tool handler invokes the `KernelFunction` against a bare `Kernel`. Because the result is a plain Agentspan agent, these tools run as ordinary Conductor worker tools. + +### Framework tool wire shape + +For `openai`/`google_adk`, `SerializeFrameworkAgent` emits tools as `{ _worker_ref, description, parameters }` (the normalizers drop the default `{name, inputSchema, toolType}` shape). Agent-as-tool emits `{ _type: "AgentTool", name, description, agent }` so the normalizer compiles a SUB_WORKFLOW. + +--- + +## 9. C#-specific design choices + +- **Async-first, sync wrappers.** Every network operation is `async`/`await` with `CancellationToken` support; the streaming API is `IAsyncEnumerable` with `[EnumeratorCancellation]`. Synchronous overloads (`Run`, `Start`, `Deploy`, `GetStatus`, …) exist for scripts and delegate to the async path via `.GetAwaiter().GetResult()`. +- **Records for data, classes for behaviour.** DTOs (`AgentResult`, `AgentStatus`, `AgentEvent`, `ToolContext`, `TokenUsage`, `GuardrailResult`, …) are `record`s with `init`-only members and `with`-expression updates (e.g. `ToolContext with { State = … }`). Mutable behavioural types (`Agent`, `AgentRuntime`, `AgentClient`) are classes. +- **Nullable reference types** are enabled across all projects; optional config uses `T?`/`Nullable`, and "absent" is consistently distinguished from "default" at serialization. +- **Two JSON stacks, bridged deliberately.** The SDK uses `System.Text.Json` (`AgentspanJson.Options`: camelCase, ignore-null, snake_case enum converter) for its own wire format and JSON-Schema export, while `conductor-csharp` uses Newtonsoft.Json for task I/O. `WorkerPollLoop` bridges the two (`ConvertInputData` / `ToNewtonsoftDict`). +- **Dependency on `conductor-csharp`.** Worker polling reuses the official client's `Configuration`, `TaskResourceApi`, `TaskResult`, and (for Orkes) `OrkesAuthenticationSettings` rather than reimplementing the task protocol. The agent control plane (`/agent/*`) is bespoke (`AgentClient`) because those endpoints are Agentspan-specific. +- **Operator + factory ergonomics.** `a >> b` builds a sequential pipeline (`Strategy.Sequential`), extending an existing pipeline in place; `Agent.ScatterGather(...)` and the tool factories (`HttpTools`, `McpTools`, `MediaTools`, `RagTools`, `ApiTools`, `CliTool`, `HumanTool`, `WaitForMessageTool`, `AgentTool`) provide one-call construction. +- **Schema from CLR types.** Structured output uses `System.Text.Json.Schema.JsonSchemaExporter` over the `OutputType` CLR `Type`, so the schema is generated, not hand-written. + +--- + +## 10. Testing + +> Full validation & e2e design: [csharp-validation.md](../../validation/csharp-validation.md) + +Tests live under `sdk/csharp/tests/`, split by concern (xUnit `[Fact]`/`[Theory]`): + +| Project | Scope | Approx. test count | +|---|---|---| +| `Agentspan.OpenAI.Tests` | OpenAI adapter + `CliTool` (`OpenAIAgentTests`, `CliToolTests`) | ~13 | +| `Agentspan.GoogleADK.Tests` | Google ADK adapter (`GoogleADKAgentTests`) | ~5 | +| `Agentspan.SemanticKernel.Tests` | Semantic Kernel bridge (`SemanticKernelAgentTests`) | ~5 | +| `AgentspanE2eTests` | End-to-end against a live server — ~20 suites (`Suite1_BasicValidation` … `Suite19_AuthHeader`, plus `Plans_*`, `ScheduleTests`, `CredentialInjectionConcurrentTest`) | ~62 | + +> **Counts are approximate** — derived from `[Fact]`/`[Theory]` occurrences at doc-creation time and will drift; treat the suite list as the durable signal. + +**Assembly name `AgentspanE2eTests` is intentionally kept** even though the namespaces were renamed `Agentspan → Conductor.AI`: the core `Conductor.AI.csproj` grants `InternalsVisibleTo("AgentspanE2eTests")` so the e2e suite can reach internal types (notably `AgentAuthHandler`). Renaming the assembly would break that grant. + +The E2E suites cover basic validation, tool calling, guardrails, termination, strategies, callbacks, credentials, coding/code-execution agents, MCP/HTTP/CLI/PDF/media tools, skills, stateful-domain routing, PLAN_EXECUTE refs, schedules, SDK parity, the `AgentClient` control plane, and the auth header. Per the project testing convention, e2e validation avoids using an LLM to judge output except where the test exists specifically to evaluate quality. + +--- + +## Cross-references + +- [`../../sdk-design.md`](../../sdk-design.md) — the cross-language SDK contract this implements. +- [`../../agentspan-design.md`](../../agentspan-design.md) — server-side compilation model. +- [`../../api-design.md`](../../api-design.md) — the `/agent/*` HTTP control plane. +- [`../../framework-integration.md`](../../framework-integration.md) — framework normalizers. +- [`../../tool-execution-and-credentials-design.md`](../../tool-execution-and-credentials-design.md) — worker + secret contract. +- [`csharp.md`](csharp.md) — C# idiom guide / public surface. +- [`python-implementation.md`](python-implementation.md) — the reference SDK implementation. + diff --git a/docs/sdk-design/csharp.md b/design/sdk-design/languages/csharp.md similarity index 99% rename from docs/sdk-design/csharp.md rename to design/sdk-design/languages/csharp.md index 4877b4bec..aba2d9fc4 100644 --- a/docs/sdk-design/csharp.md +++ b/design/sdk-design/languages/csharp.md @@ -2,8 +2,9 @@ **Date:** 2026-03-23 **Status:** Draft -**Base Spec:** `docs/sdk-design/2026-03-23-multi-language-sdk-design.md` +**Base Spec:** `design/sdk-design.md` **Reference Implementation:** `sdk/python/examples/kitchen_sink.py` +**As-built internals:** [csharp-implementation.md](csharp-implementation.md) --- diff --git a/docs/sdk-design/go.md b/design/sdk-design/languages/go.md similarity index 99% rename from docs/sdk-design/go.md rename to design/sdk-design/languages/go.md index f3d6014f3..8164249b4 100644 --- a/docs/sdk-design/go.md +++ b/design/sdk-design/languages/go.md @@ -1,7 +1,7 @@ # Go SDK Translation Guide **Date:** 2026-03-23 -**Base Spec:** `docs/sdk-design/2026-03-23-multi-language-sdk-design.md` +**Base Spec:** `design/sdk-design.md` **Python Reference:** `sdk/python/examples/kitchen_sink.py` --- @@ -1385,7 +1385,7 @@ group = "SMOKE_TEST" timeout = 300 [judge] -model = "openai/gpt-4o-mini" +model = "anthropic/claude-sonnet-4-6" max_output_chars = 3000 max_tokens = 300 ``` @@ -1603,7 +1603,7 @@ piiGuardrail := agentspan.NewRegexGuardrail("pii_blocker", ) biasGuardrail := agentspan.NewLLMGuardrail("bias_detector", - "openai/gpt-4o-mini", + "anthropic/claude-sonnet-4-6", "Check for biased language or stereotypes.", agentspan.WithPosition(agentspan.PositionOutput), agentspan.WithOnFail(agentspan.OnFailFix), diff --git a/design/sdk-design/languages/java-implementation.md b/design/sdk-design/languages/java-implementation.md new file mode 100644 index 000000000..a6fa81ffe --- /dev/null +++ b/design/sdk-design/languages/java-implementation.md @@ -0,0 +1,265 @@ +# Java SDK — Reference Implementation + +**Status:** Created 2026-06-26 + +**Scope:** This document describes the internal architecture of the **Java SDK** as a *reference implementation* of the Agentspan SDK contract — written from the actual source under `sdk/java/`. It mirrors the structure of the Python reference doc ([`python-implementation.md`](python-implementation.md)) and covers package layout, the compile-and-execute model, the runtime/worker internals, streaming, guardrails, credentials, framework bridges, the Spring Boot adapter, and Java-specific design choices. For the language-agnostic contract see [`../../sdk-design.md`](../../sdk-design.md); for the server-side compilation model see [`../../agentspan-design.md`](../../agentspan-design.md); for the wire endpoints see [`../../api-design.md`](../../api-design.md); for tools and secrets see [`../../tool-execution-and-credentials-design.md`](../../tool-execution-and-credentials-design.md); for frameworks see [`../../framework-integration.md`](../../framework-integration.md). The companion idiom/translation guide is [`java.md`](java.md). + +--- + +## 1. Overview + +The Java SDK lets you declare `Agent` objects in Java and run them as durable Conductor workflows. Like every Agentspan SDK, it follows the **compile-don't-interpret** model: the SDK serializes an `Agent` tree to an `AgentConfig` JSON payload and POSTs it to the server, which compiles it into a Conductor workflow definition. The SDK then runs the agent's tool functions as Conductor *workers* and polls/streams the execution. The SDK never interprets the agent loop locally — that lives on the server. + +**Maven coordinates** (`sdk/java/build.gradle`, group `org.conductoross.conductor`): + +| Artifact | Module | Purpose | +|---|---|---| +| `org.conductoross.conductor:conductor-agent-sdk` | `sdk/java` (root) | Core SDK | +| `org.conductoross.conductor:conductor-agent-sdk-spring` | `sdk/java/spring` | Spring Boot auto-configuration | + +- **Namespace:** `org.conductoross.conductor.ai` +- **Java toolchain:** Java **21** (`JavaLanguageVersion.of(21)`); compiled with `-parameters` so reflective tool-parameter names survive. +- **Key dependencies:** the official Conductor Java client `org.conductoross:conductor-client:5.0.1` (`api` scope — its `ApiClient`/`ConductorClient` are part of the SDK's public surface), Jackson 2.17 (`api`), SLF4J. LangChain4j (1.0.0), Google ADK (1.3.0), and LangGraph4j (1.6.0-beta5) are **`compileOnly`** — bridges link only at runtime when a user passes a native object. + +> Design note: connection details (server URL, auth key/secret) are *not* SDK config — they live entirely on the Conductor `ApiClient`. The SDK's own `AgentConfig` carries only worker-runner tuning. This differs from the Python SDK's monolithic `AgentConfig`. + +--- + +## 2. Package & source layout + +Root package `org.conductoross.conductor.ai` (99 main source files): + +| Package | Contents | +|---|---| +| *(root)* | `Agent`, `Agent.Builder`, `AgentRuntime`, `AgentConfig`, `CallbackHandler` | +| `tools` | Server-side tool factories: `HttpTool`, `ApiTool` (OpenAPI/Swagger/Postman discovery, `${NAME}` credential validation), `McpTool`, `AgentTool`, `HumanTool`, `MediaTools`, `PdfTool`, `RagTools`, `WaitForMessageTool` | +| `annotations` | `@Tool`, `@AgentDef`, `@GuardrailDef` (declarative POJO style) | +| `guardrail` | `Guardrail`, `LLMGuardrail`, `RegexGuardrail` (server-side built-ins) | +| `model` | DTOs/value types: `AgentResult`, `AgentHandle`, `AgentStream`, `AgentEvent`, `ToolDef`, `GuardrailDef`, `GuardrailResult`, `ToolContext`, `ConversationMemory`, `SemanticMemory` (client-side keyword/Jaccard-overlap store; not serialized to the wire), `PromptTemplate`, `TokenUsage`, `CompileResponse`, `DeploymentInfo`, `PendingToolCall`, `PrefillToolCall`, `CredentialFile` | +| `enums` | `Strategy`, `EventType`, `AgentStatus`, `OnFail`, `Position`, `Framework` | +| `internal` | The plumbing: `AgentConfigSerializer`, `AgentClient`, `AgentRequest`, `WorkerManager`, `ToolRegistry`, `AgentRegistry`, `SseClient`, `JsonMapper`, `WorkerCredentialFetcher`, `CredentialContext`, response DTOs (`StartResponse`, `CompileResponse`, `AgentStatusResponse`), `RespondBody`, `PendingTool` | +| `handoff` | `Handoff`, `OnTextMention`, `OnToolResult`, `OnCondition` (SWARM triggers) | +| `termination` | `TerminationCondition` + `MaxMessageTermination`, `StopMessageTermination`, `TextMentionTermination`, `TokenUsageTermination`, `AndTermination`, `OrTermination`, `TerminationResult` | +| `plans` | `Plan` and supporting types (`Step`, `Action`, `Op`, `Ref`, `Generate`, `Validation`, `Context`, `PlanValues`) — deterministic PLAN_EXECUTE | +| `schedule` | `Schedule`, `Schedules`, `ScheduleInfo` (cron lifecycle). `Schedules.runNowAndWait(name)` now returns an `AgentResult` (uniform with the other SDKs), not a Conductor `Workflow`. | +| `skill` | `Skill`, `SkillLoadError` (Agent Skills: `SKILL.md` directories) | +| `gate` | `TextGate` (sequential-pipeline sentinel gate) | +| `execution` | `CliCommandExecutor`, `CliConfig`, `CodeExecutor`, the full executor family — `LocalCodeExecutor` (subprocess + temp file), `DockerCodeExecutor` (container), `JupyterCodeExecutor` (stateful kernel via a Jupyter Kernel Gateway over HTTP), `ServerlessCodeExecutor` (HTTP endpoint) — and `ExecutionResult` (local CLI / code execution) | +| `frameworks` | `AdkBridge`, `LangChainBridge`, `LangChain4jAgent`, `OpenAIAgent` | +| `openai` | `GPTAssistantAgent` (OpenAI Assistants API wrapper) | +| `exceptions` | `AgentspanException`, `AgentAPIException`, `AgentNotFoundException`, and credential errors (`CredentialNotFound/Auth/RateLimit/Service`) | + +The `spring/` module (3 main classes) holds the Spring Boot adapter (§9). + +--- + +## 3. Compilation & serialization + +An `Agent` is an immutable value object built with a fluent `Agent.Builder` (§10). It is never compiled locally — the SDK serializes it and the server compiles it. + +### Serialization (`internal/AgentConfigSerializer`) + +`AgentConfigSerializer.serialize(Agent)` walks the agent tree and produces a **camelCase `Map`** matching the server's `AgentConfig` DTO. Notable behaviors verified in source: + +- **Native vs framework path.** A `framework` of `"skill"`, `"openai"`, or `"google_adk"` takes a dedicated branch emitting the raw framework config (e.g. `_framework`, OpenAI's `instructions` vs ADK's singular `instruction`, the `_worker_ref` tool shape framework normalizers expect). All other agents take the native path. +- **Dynamic instructions** are supplier-backed (`Supplier`) and resolved exactly once per serialization, so callable instructions re-evaluate on every run submission (matching Python). +- **Strategy** is emitted only when `agents` is non-empty *or* a PLAN_EXECUTE named slot (`planner`/`fallback`) is set — otherwise the server would default to `handoff` and reject the named slots with HTTP 400. +- **Injected tools.** Enabling `localCodeExecution` injects a `{name}_execute_code` worker tool; a `CliConfig` injects a `{name}_run_command` worker tool — both mirror the Python SDK's `_attach_*` helpers. +- **Credentials** declared on a tool are nested under `config.credentials` so the server includes them in the execution token's `declared_names`. +- `enablePlanning` (plan-first preamble) is deliberately a separate boolean key from the `planner` slot, because the server reused the `planner` JSON key for the PLAN_EXECUTE sub-agent. + +The serializer is also exposed as a Jackson `JsonSerializer` (`AgentConfigSerializer.AsJson`) so `Agent`-typed fields serialize correctly without callers pre-converting to a `Map`. + +### Request shape (`internal/AgentRequest`) + +`compile`, `deploy`, and `start` share one server DTO. `AgentRequest` carries the `Agent`, an optional `Framework`, and execution fields (`prompt`, `sessionId`, `runId`, `staticPlan`, `media`, `context`, `idempotencyKey`, `credentials`, `timeoutSeconds`). Its custom `Serializer` writes **mutually-exclusive** keys: + +- Native agent → `"agentConfig": {…}` +- Framework agent → `"framework": "", "rawConfig": {…}` + +A deterministic `Plan` is forwarded as `"static_plan"` (server reads it as the highest-priority Case-0 plan, skipping the planner LLM). See [`../../agentspan-design.md`](../../agentspan-design.md) for the server-side compilation pipeline (single/tools/multi-agent/hybrid dispatch). + +--- + +## 4. Runtime lifecycle — `AgentRuntime` + `AgentClient` + +`AgentRuntime` (`AutoCloseable`) is the entry point. Unlike Python's module-level singleton, the Java SDK uses **explicit `AgentRuntime` instances** (try-with-resources friendly). See the API reference at `sdk/java/docs/agent-runtime-api.md` and `sdk/java/docs/agent-client-api.md`. + +### Construction + +The runtime owns **one** native Conductor `ApiClient` (server URL + auth), shared by every typed client: `AgentClient` (control plane), `WorkflowClient` (token/tool enrichment), `WorkerManager`, `SseClient`, and the lazy `Schedules`. Factory helpers build the client: + +```java +AgentRuntime.clientFromEnv(); // AGENTSPAN_SERVER_URL / _AUTH_KEY / _AUTH_SECRET +AgentRuntime.client(url); // unauthenticated +AgentRuntime.client(url, key, secret); // native key/secret → token +``` + +The `/api` base path is appended automatically; explicit connect/read/write timeouts (10s/30s/30s) bound a slow server. + +### Operations + +`run`, `start`, `stream` (each with async `…Async` variants), plus `plan`, `deploy`, `serve`, `resume`, and the `schedules()` accessor. `run` is `start` + `waitForResult`; `stream` is `start` + an SSE connection. + +``` +startAsync(agent, prompt, plan) + | + +-- runId = hasStatefulTools(agent) ? uuid : null # per-execution domain + +-- prepareWorkers(agent, runId) # register local workers + +-- workerManager.startAll() # build/rebuild task runner + +-- agentClient.startAgent(AgentRequest…) # POST /api/agent/start + +-- return AgentHandle(executionId, agentClient, workflowClient) +``` + +`runAsync` intentionally does **not** pre-register workers before `startAsync` — for stateful agents, registration must happen under the per-execution domain (`runId`), or the worker would poll the default queue while the server enqueues under `runId` (a real bug that was fixed and regression-tested). + +### Control plane (`internal/AgentClient`) + +Strictly five endpoints, all routed through the shared `ConductorClient` (native HTTP + auth + serialization; no hand-rolled HTTP): + +| Method | Endpoint | +|---|---| +| `compileAgent` | `POST /api/agent/compile` | +| `deployAgent` | `POST /api/agent/deploy` | +| `startAgent` | `POST /api/agent/start` | +| `getAgentStatus` | `GET /api/agent/{executionId}/status` | +| `respond` | `POST /api/agent/{executionId}/respond` | + +Conductor's `ConductorClientException` is mapped to the SDK's typed `AgentNotFoundException` (404) / `AgentAPIException`. Standard Conductor endpoints (`/api/workflow/*`, `/api/tasks`, `/api/scheduler/*`) use the Conductor SDK's own typed clients. + +### Result & handle (`model/AgentHandle`, `model/AgentResult`) + +`AgentHandle.waitForResult()` polls `getAgentStatus` (2s interval, 10-min default timeout) until a terminal status (`COMPLETED`/`FAILED`/`TERMINATED`/`TIMED_OUT`), escalating log level after 3 consecutive poll errors and giving up after 10. On completion it walks the workflow tasks once via `WorkflowClient` to aggregate `TokenUsage` (from `LLM_CHAT_COMPLETE` tasks) and `toolCalls` (tasks whose ref name starts with `call_`), since the server doesn't aggregate these on the status response. HITL methods: `approve()/approve(comment)/reject(reason)/respond(Map)/send`, plus `isWaiting()` / `waitUntilWaiting(timeoutMs)`. + +--- + +## 5. Worker & dispatch internals + +### `internal/WorkerManager` + +Rather than a hand-rolled poll loop, the SDK drives workers with the official Conductor client's `TaskRunnerConfigurer` + `Worker`, which provides battle-tested polling, backoff, managed threads, and — crucially — **automatic lease extension (heartbeat)** (every worker returns `leaseExtendEnabled() == true`), so a handler that blocks for minutes keeps its lease instead of being reclaimed. + +- **Incremental registration vs fixed runner.** Agentspan registers workers per-run (sometimes under a per-execution domain), but `TaskRunnerConfigurer` is built from a fixed worker set. The bridge: `startAll()` (re)builds the configurer only when a *new* task type appeared since the last build (`workerSetChanged`). Re-registering an existing task only swaps the handler (looked up live in `Worker.execute`) — no rebuild — *unless* its domain changed (which is baked into `taskToDomain` at build time and forces a rebuild). +- **Task-def sizing.** Each new task registers a `TaskDef` whose `responseTimeoutSeconds` = `effectiveTaskTimeout(handlerTimeout)` (floor 300s, plus 60s slack) so the server's patience never drifts below the handler's blocking timeout. +- **Thread count** = `max(config.workerThreadCount, 1 × workerCount)` — at least one thread per worker type so a blocking handler can't starve others. +- **Output mapping:** a handler returning a `Map` becomes `outputData` directly; any other value is wrapped as `{"result": value}`. + +### What `prepareWorkers` registers (`AgentRuntime.prepareWorkers`) + +Walking the agent tree, the runtime registers local Java handlers for: + +- **`@Tool` worker tools** (`toolType == "worker"`) — with declared credentials and timeout. +- **`agent_tool`** child agents (recursively). +- **Callbacks** — legacy `before/after_model` functions and `CallbackHandler` lists (chained per position: `before/after_agent/model/tool`). +- **Combined output guardrail** worker (`{name}_output_guardrail`) — runs all custom guardrail functions, returns `passed/on_fail/fixed_output/should_continue`. +- **Termination** worker (`{name}_termination`) — evaluates the composable `TerminationCondition`. +- **Local code execution** (`{name}_execute_code`) and **CLI** (`{name}_run_command`) workers. +- **SWARM** workers (`{src}_transfer_to_{peer}`, `{name}_check_transfer`, `{name}_handoff_check`) and **MANUAL** `{name}_process_selection`. +- **Skill** workers for `framework == "skill"` agents. + +### How `@Tool` becomes a worker (`internal/ToolRegistry`) + +`ToolRegistry.fromInstance(Object)` reflects over `@Tool`-annotated public methods. For each it: reads annotation metadata (name/description/credentials/retry); generates a JSON Schema from the method parameters (`-parameters`-retained names, `typeToJsonSchema`); wraps the method in a `Function` that coerces inputData → args, injects `ToolContext` if declared, and returns the result; and builds a `ToolDef` with **`toolType = "worker"`**. The server compiles each into a Conductor `SIMPLE` task; the SDK's `WorkerManager` polls and runs the handler. See [`../../tool-execution-and-credentials-design.md`](../../tool-execution-and-credentials-design.md). + +**Tool types** (the `toolType` wire strings): `worker`, `http`, `api`, `mcp`, `agent_tool`, `human`, `generate_image`/`generate_audio`/`generate_video`/`generate_pdf`, `rag_search`/`rag_index`, `pull_workflow_messages`. Only `worker` (and injected code/CLI workers) execute locally; the rest are server-side task types. + +--- + +## 6. Streaming / SSE client + +`stream(agent, prompt)` starts the agent, then opens `GET /api/agent/stream/{executionId}` via `internal/SseClient`. The request is built with `ApiClient.buildCall` so it rides the shared OkHttp client and the token-refresh auth interceptor — exactly like every other client (no separate HTTP stack). + +- A daemon thread reads the response body line-by-line, parsing standard SSE framing (`event:` / `data:` / `id:` / `:` comments), buffering multi-line `data:`, and on a blank line dispatches the accumulated event. +- Parsed events become `AgentEvent` (via `AgentEvent.fromMap`) and are placed on a `LinkedBlockingQueue`; consumers call `nextEvent()` (blocking) or iterate `AgentStream` (`Iterable` + `AutoCloseable`). +- `[DONE]` or an event of type `done` enqueues a `DONE_SENTINEL` ending the stream. + +`AgentStream` also supports event-targeted HITL (`approve(event)`/`reject(event, reason)`) and a `waitForResult` fallback that aggregates from captured events. Event types (`enums/EventType`): `THINKING`, `TOOL_CALL`, `TOOL_RESULT`, `HANDOFF`, `WAITING`, `MESSAGE`, `ERROR`, `DONE`, `GUARDRAIL_PASS`, `GUARDRAIL_FAIL`. + +--- + +## 7. Guardrails & credentials (SDK-side) + +### Guardrails + +Four `guardrailType` values: + +- **`custom`** — a local `Function` (from `@GuardrailDef` or `Guardrail.of(...)`). Compiled into a single combined `{name}_output_guardrail` worker. +- **`external`** — references an existing Conductor worker by name (no local function). +- **`llm`** — `LLMGuardrail`: server-side LLM policy evaluation. +- **`regex`** — `RegexGuardrail`: server-side pattern match (block/allow mode). + +`GuardrailResult` is `pass()`, `fail(message)`, or `fix(fixedOutput)`. `OnFail` (`RETRY`/`RAISE`/`FIX`/`HUMAN`) drives the server's post-guardrail switch; `Position` is `INPUT` or `OUTPUT` (default). The combined worker enforces `maxRetries` (downgrading `retry`→`raise` once exhausted, and `fix`→`raise` when no fixed output is present), matching Python. + +### Credentials + +Tools/agents declare credential names as `List` (e.g. `@Tool(credentials = {...})`, `HttpTool.credentials(...)`, `Agent.Builder.credentials(...)`), serialized so the server's execution token carries the `declared_names`. At runtime, before invoking a handler with declared secrets, `WorkerManager` pulls the execution token from `inputData["__agentspan_ctx__"]["execution_token"]` and calls `internal/WorkerCredentialFetcher` → `POST /api/workers/secrets` (`{token, names}`) → `Map`. Resolution failures are **terminal** (`FAILED_WITH_TERMINAL_ERROR`) so Conductor doesn't burn retries on a config problem. Resolved secrets are placed in a **`ThreadLocal`** (`internal/CredentialContext`) for the handler's duration and cleared in a `finally` — they never enter task I/O, and tool code reads them via `ToolContext.getCredential(name)`. Typed errors: `CredentialNotFound/Auth/RateLimit/Service`. + +--- + +## 8. Framework integration + +Native framework objects are accepted via `Object`-typed drop-in overloads (`run/start/stream/deploy/serve/plan/resume`) and coerced in `AgentRuntime.coerceAgent`. The core class **never references a `compileOnly` framework type in a signature** — detection is by fully-qualified name walking the type hierarchy (`isInstanceOf`), so the SDK compiles and runs without those frameworks on the classpath; the JVM loads framework classes only when a user actually passes one. See [`../../framework-integration.md`](../../framework-integration.md). + +| Framework wire value | Bridge / builder | Notes | +|---|---|---| +| `openai` | `frameworks/OpenAIAgent` (builder), `openai/GPTAssistantAgent` | OpenAI Agents SDK shape; handoffs → `frameworkConfig.handoffs` | +| `google_adk` | `frameworks/AdkBridge` | Native ADK `BaseAgent` (Llm/Sequential/Parallel/Loop) → `frameworkConfig` for `GoogleADKNormalizer` | +| `langchain` | `frameworks/LangChainBridge` + `LangChain4jAgent` | LangChain4j `ChatModel` + `@Tool` POJOs; model → `provider/model` string | +| `langgraph` | `LangChainBridge` (via `AgentExecutor.Builder`) | Recovers `chatModel`/`systemMessage` reflectively; validates `.build()` before shipping | +| `skill` | `skill/Skill` | `SKILL.md` directories; registers skill workers | +| `vercel_ai`, `claude_agent_sdk` | (enum-only wire values) | Routed to server normalizers | + +`Framework` (`enums/Framework`) maps each wire value 1:1 to a server-side normalizer; `Framework.of(String)` resolves the agent's framework string and selects the native (`agentConfig`) vs framework (`framework`+`rawConfig`) request path. + +--- + +## 9. Spring Boot adapter (`-spring` module) + +`spring/` adds Spring Boot auto-configuration (`org.conductoross.conductor:conductor-agent-sdk-spring`). The `META-INF/spring/…AutoConfiguration.imports` file registers `AgentAutoConfiguration`, which wires three `@ConditionalOnMissingBean` beans: + +- **`AgentConfig`** — from `agentspan.*` properties (`AgentProperties`: worker poll interval, thread count). +- **`AgentRuntime`** — built from the injected `ApiClient` + `AgentConfig`. +- **`AgentCatalog`** — discovers agents from the `ApplicationContext`. + +Crucially, the `ApiClient` (server URL + auth) is **not** created here — it comes from the Conductor client's own `OrkesConductorClientAutoConfiguration` (pulled in via `conductor-client-spring`), configured under the `conductor.*` namespace (`conductor.root-uri`, `conductor.security.client.key-id/secret`). `@AutoConfiguration(after = OrkesConductorClientAutoConfiguration.class)` orders this correctly. All beans are conditional, so users can override any of them. + +--- + +## 10. Java-specific design choices + +- **Immutable value object + fluent `Builder`.** `Agent` is final-field immutable with a `~50`-setter `Agent.Builder`; varargs convenience overloads (`tools(…)`, `agents(…)`, `guardrails(…)`, `handoffs(…)`) accumulate. Validation (`build()`): name matches `^[a-zA-Z_][a-zA-Z0-9_-]*$`, `maxTurns >= 1`, `plannerContext` only with `PLAN_EXECUTE`. +- **Dual declarative styles.** Imperative builders *and* annotation-driven POJOs (`@AgentDef` methods resolved by `Agent.fromInstance` / `AgentRegistry`; `@Tool` and `@GuardrailDef` methods reflected by `ToolRegistry`). `@AgentDef` methods may return `String`, `PromptTemplate`, `Agent.Builder`, or `Agent`. + + > The idiom guide [`java.md`](java.md) frames the SDK for *both* Java 16+ (records, sealed types) and Java 8+ (POJOs). The shipped reference SDK targets the **Java 21** toolchain; user tool/output POJOs can be records or classes — `AgentConfigSerializer.generateJsonSchema` reflects declared fields either way, and `-parameters` preserves tool-parameter names. + +- **Supplier-backed dynamic instructions** — `instructions(Supplier)` re-evaluates per serialization (the closest Java analogue to Python's callable instructions). +- **Explicit runtime instances over a global singleton** — `AgentRuntime implements AutoCloseable`; `close()`/`shutdown()` releases the OkHttp dispatcher + connection pool (which otherwise leak idle threads across test suites). +- **Separation of transport and tuning** — `ApiClient` owns connectivity; `AgentConfig` owns only worker-runner tuning. This is a deliberate divergence from Python's combined config. +- **Lean control-plane client** — `AgentClient` is scoped to exactly five proprietary endpoints; everything standard reuses the Conductor SDK's typed clients, riding one shared auth/HTTP stack. +- **No hard framework coupling** — `compileOnly` framework deps + FQN-based coercion keep the core dependency-free at runtime. +- **Async via `CompletableFuture`** — every sync op has a `…Async` variant; `run = start + waitForResult`. + +--- + +## 11. Testing + +> Full validation & e2e design: [java-validation.md](../../validation/java-validation.md) + +Test layout under `sdk/java/`: + +| Location | Files | Approx. `@Test` | +|---|---|---| +| `src/test/java` (unit) | 24 test classes | ~241 | +| `e2e/` (integration, live server) | 26 files | ~162 | +| `spring/src/test/java` | 2 (`AgentAutoConfigurationTest`, `AgentCatalogTest`) | — | + +- **Unit tests** are organized by package (`internal`, `model`, `tools`, `termination`, `plans`, `handoff`, `exceptions`, `execution`, `schedule`, `frameworks`) and run with JUnit 5 — no server required. +- **E2E suites** (`e2e/SuiteNN*.java`, e.g. `Suite8Guardrails`, `Suite14StatefulDomain`, `Suite18ToolTypes`, `Suite4McpTools`) are tagged `@Tag("e2e")` and **excluded by default**; run with `-Pe2e` (which also sets `maxParallelForks = 3`). They require a live Conductor server. Per project convention, e2e validation avoids using an LLM to judge correctness except where the test is specifically about output/eval quality. +- **Tooling:** Gradle (`java-library`), JaCoCo coverage (`jacocoTestReport`), Spotless with `palantirJavaFormat`. + +--- + +## Uncertainties / flagged in-doc + +- **`vercel_ai` / `claude_agent_sdk`** appear as `Framework` enum wire values but have no dedicated Java bridge class in `frameworks/`/`openai/` — documented as enum-only/server-normalizer routing rather than a first-class SDK builder. Verify against the server normalizer set if a Java-side bridge is expected. +- **Test `@Test` counts** are derived from raw `grep` of `@Test` occurrences (241 unit / 162 e2e) and class-file counts; treat as approximate, not authoritative per-method tallies. diff --git a/docs/sdk-design/java.md b/design/sdk-design/languages/java.md similarity index 99% rename from docs/sdk-design/java.md rename to design/sdk-design/languages/java.md index 73d5cf4ef..f96496c6c 100644 --- a/docs/sdk-design/java.md +++ b/design/sdk-design/languages/java.md @@ -2,8 +2,9 @@ **Date:** 2026-03-23 **Status:** Draft -**Base Spec:** `docs/sdk-design/2026-03-23-multi-language-sdk-design.md` +**Base Spec:** `design/sdk-design.md` **Reference Implementation:** `sdk/python/examples/kitchen_sink.py` +**As-built internals:** [java-implementation.md](java-implementation.md) This guide covers implementing the Agentspan SDK in Java with full feature parity against the Python reference. It addresses **two** target audiences simultaneously: projects on Java 16+ (records, sealed interfaces, pattern matching) and projects constrained to Java 8+ (POJOs, Lombok optional). Every section shows both styles side-by-side. @@ -14,8 +15,8 @@ This guide covers implementing the Agentspan SDK in Java with full feature parit ### 1.1 Module Coordinates ``` -groupId: dev.agentspan -artifactId: agentspan-sdk +groupId: org.conductoross.conductor +artifactId: conductor-agent-sdk version: 0.1.0 ``` @@ -23,8 +24,8 @@ version: 0.1.0 ```xml - dev.agentspan - agentspan-sdk + org.conductoross.conductor + conductor-agent-sdk 0.1.0 @@ -545,7 +546,7 @@ Guardrail piiGuardrail = RegexGuardrail.builder() Guardrail biasGuardrail = LLMGuardrail.builder() .name("bias_detector") - .model("openai/gpt-4o-mini") + .model("anthropic/claude-sonnet-4-6") .policy("Check for biased language or stereotypes. If found, provide corrected version.") .position(Position.OUTPUT) .onFail(OnFail.FIX) diff --git a/docs/sdk-design/kotlin.md b/design/sdk-design/languages/kotlin.md similarity index 99% rename from docs/sdk-design/kotlin.md rename to design/sdk-design/languages/kotlin.md index f76a87a24..70e94eb8f 100644 --- a/docs/sdk-design/kotlin.md +++ b/design/sdk-design/languages/kotlin.md @@ -2,7 +2,7 @@ **Date:** 2026-03-23 **Status:** Draft -**Reference:** `docs/sdk-design/2026-03-23-multi-language-sdk-design.md` (base spec) +**Reference:** `design/sdk-design.md` (base spec) **Kitchen Sink Reference:** `sdk/python/examples/kitchen_sink.py` --- @@ -959,7 +959,7 @@ val reviewAgent = agent("safety_reviewer") { } // #28 on_fail=FIX llm("bias_detector") { - model = "openai/gpt-4o-mini" + model = "anthropic/claude-sonnet-4-6" policy = "Check for biased language or stereotypes. If found, provide corrected version." position = Position.OUTPUT onFail = OnFail.FIX diff --git a/design/sdk-design/languages/python-implementation.md b/design/sdk-design/languages/python-implementation.md new file mode 100644 index 000000000..b72a2852e --- /dev/null +++ b/design/sdk-design/languages/python-implementation.md @@ -0,0 +1,289 @@ +# Python SDK — Reference Implementation + +**Status:** Refreshed 2026-06-26 + +**Scope:** This document describes how the Python SDK implements the cross-language SDK contract. Python is the *reference implementation*: it is the first SDK to land every feature, and the other language SDKs (TypeScript, C#, Java) are translated from its behavior. When this doc and another SDK disagree, Python is correct and the other SDK has drifted. The shared, language-neutral contract lives in [`../../sdk-design.md`](../../sdk-design.md); the platform-level compilation/execution model lives in [`../../agentspan-design.md`](../../agentspan-design.md). This file documents only the Python-observed internals and gotchas — it does not re-derive the model. + +Cross-links: contract [`../../sdk-design.md`](../../sdk-design.md) · platform [`../../agentspan-design.md`](../../agentspan-design.md) · API [`../../api-design.md`](../../api-design.md) · frameworks [`../../framework-integration.md`](../../framework-integration.md) · tools/credentials [`../../tool-execution-and-credentials-design.md`](../../tool-execution-and-credentials-design.md). + +--- + +## 1. Overview + +The Python SDK lets you define `Agent` objects in Python and run them as durable Conductor workflows. The SDK serializes an agent tree to JSON, the **server-side Java compiler** turns that into a Conductor `WorkflowDef`, and a long-lived worker process executes your tool functions as distributed Conductor tasks. Each agent compiles to one workflow; user-facing runs are called "executions". + +Reference-implementation design principles (these hold across all SDKs): + +1. **Everything is an Agent.** One primitive for single agents, multi-agent teams, and nested hierarchies — no separate Team/Network/Swarm classes. +2. **Server-first execution.** Tools execute as distributed Conductor tasks, not in-process. The agent survives process crashes; human approvals can take days. +3. **Compile, don't interpret.** Agent definitions compile to static workflow JSON. Behavior is inspectable, versioned, reproducible. +4. **Zero config for simple cases.** `Agent + tool + run` works in ~5 lines; advanced features (memory, guardrails, streaming) layer on without changing the core API. +5. **Conductor-native.** Every SDK concept maps directly to a Conductor primitive. + +**Naming & distribution:** + +| Aspect | Value | +|---|---| +| PyPI package | `conductor-agent-sdk` | +| Import namespace | `conductor.ai.agents` (public surface), with internals under `conductor.ai.agents.runtime`, `conductor.ai.models` | +| Supported Python | 3.10–3.13 (`requires-python = ">=3.10,<3.14"`) | +| Logger root | `conductor.ai.agents.*` | +| CLI binary | `agentspan` (console script → `conductor.ai.cli:main`) | +| Env vars | `AGENTSPAN_*` | +| OTel tracer name | `conductor.ai.agents` | + +> Runtime contracts that are intentionally still `agentspan` (these are wire/operator-facing and must match the server): the `AGENTSPAN_*` environment variables, the `agentspan` CLI binary name, and the `__agentspan_ctx__` task-input context key. Logger names and the import namespace follow `conductor.ai`. + +--- + +## 2. Package & source layout + +Source root: `sdk/python/src/conductor/ai/` + +``` +conductor/ai/ +├── __init__.py +├── __main__.py # mirrors the `agentspan` console script +├── agents/ # the SDK proper — public API in __init__.py +│ ├── agent.py # Agent, AgentDef, Strategy, @agent, scatter_gather +│ ├── run.py # module-level run/start/stream/deploy/resume/serve + singleton +│ ├── result.py # AgentResult, AgentStatus, AgentEvent, EventType, AgentHandle, streams +│ ├── config_serializer.py # Agent tree -> AgentConfig JSON (the wire DTO) +│ ├── guardrail.py # @guardrail, Guardrail, OnFail, Position, GuardrailResult +│ ├── handoff.py / termination.py / memory.py / semantic_memory.py / plans.py +│ ├── tool.py # @tool, http_tool, mcp_tool, @worker_task, ToolContext +│ ├── skill.py / claude_code.py / cli_config.py / code_execution_config.py / code_executor.py +│ ├── openai_compat.py / langchain.py / ocg.py / gate.py / callback.py / tracing.py / exceptions.py +│ ├── _internal/ # model_parser, provider_registry, schema_utils, token_utils +│ ├── frameworks/ # langchain, langgraph, claude_agent_sdk adapters + serializer +│ ├── runtime/ # the execution engine (see §4) +│ │ ├── runtime.py # AgentRuntime — compile/prepare/execute/extract/stream +│ │ ├── _dispatch.py # dispatch worker internals (see §5) +│ │ ├── worker_manager.py # long-lived worker process lifecycle +│ │ ├── tool_registry.py # worker registration + global registries +│ │ ├── config.py # AgentConfig.from_env() — AGENTSPAN_* loader +│ │ ├── http_client.py # AgentClient (REST + SSE) +│ │ ├── server.py # locate/install the `agentspan` CLI binary +│ │ ├── _liveness.py / discovery.py / mcp_discovery.py / secret_injection.py +│ │ └── credentials/ # accessor (get_secret), fetcher, types +│ ├── schedule/ # recurring-execution API/client +│ └── testing/ # pytest plugin, assertions, mocks, eval runner, recording +├── cli/ # deploy.py + discover.py — invoked BY the native CLI binary +└── models/ # multi-model LLM management (providers, routing, monitoring) +``` + +The public API is re-exported from `conductor.ai.agents.__init__`; users import from there (e.g. `from conductor.ai.agents import Agent, tool, run`). + +--- + +## 3. Compilation pipeline + +Compilation is **always server-side**. The SDK never builds workflow JSON locally; it serializes the agent and POSTs it to the Java compiler. For the full compilation/execution model — strategy dispatch, the compiled workflow shapes, durability semantics — see [`../../agentspan-design.md`](../../agentspan-design.md). What follows is only the Python-observed surface. + +``` +User Code Python SDK Java Server +========== ========== =========== +Agent( AgentConfigSerializer.serialize() + name, model, | -> AgentConfig JSON dict + tools, agents, v + ... POST {server_url}/agent/compile -> AgentCompiler.compile() +) {"agentConfig": {...}} dispatches by shape: + | agents & !tools -> MultiAgentCompiler + ServerCompiledWorkflow <--JSON-- agents & tools -> compileHybrid + (wraps WorkflowDef) workflowDef tools -> compileWithTools + | no tools -> compileSimple + tool_registry.register_tool_workers() + v + @worker_task functions registered, worker process started +``` + +- Serializer: `config_serializer.py` → `AgentConfigSerializer.serialize(agent)` returns a dict matching the Java `AgentConfig` DTO. Callables (tools, guardrails, `stop_when`, router, handoffs) are registered as workers locally and sent as **task-name references**, never code. +- Compile call: `AgentRuntime._compile_via_server()` POSTs `{"agentConfig": config_json}` to `{server_url}/agent/compile` (30s timeout), reads `workflowDef` from the response, and wraps it in `ServerCompiledWorkflow`. There is an async twin `_compile_via_server_async()` using `AgentClient`. Results are cached per `agent.name`. +- "Local vs server compile" survives only as *terminology in streaming detection and guardrail registration* (the SDK registers both an individual-worker form and a combined-worker form so either compile path on the server resolves the task names). The compile itself is server-only. + +**Python-observed compiled shapes** (illustrative; canonical definitions in the platform doc): + +*Single agent with tools — DoWhile loop:* +``` +[SetVariable: init messages] + -> [DoWhile] + [LlmChatComplete] reads ${workflow.variables.messages}, json_output=True + [dispatch_worker] routes tool calls, updates messages + [SetVariable] messages = ${dispatch.output.messages} + [stop_when_worker] (optional, if agent.stop_when set) + condition: $.loop.iteration < max_turns + && $.dispatch.continue_loop == true + [&& $.stop_when.should_continue == true] + -> Output: ${dispatch.output.result} +``` +Key detail: in DoWhile conditions, task refs map to outputData with **no `.output` wrapper** — `$.dispatch.continue_loop`, not `$.dispatch.output.continue_loop`. + +*Single agent, no tools:* one `[LlmChatComplete]` over `[system_prompt, user_prompt]` → `${llm.output.result}`. + +Multi-agent strategies (handoff/router → `SwitchTask` + inline sub-workflows; sequential → chained `SubWorkflow`; parallel → `Fork`/`Join`; hybrid → DoWhile with `transfer_to_{name}` tools feeding a `SwitchTask`) follow the platform doc — the SDK only chooses the strategy via the serialized config; the server emits the structure. + +--- + +## 4. Runtime lifecycle + +`run()`, `start()`, `stream()`, `run_async()`, `deploy()`, `resume()`, `serve()` (in `run.py`) share a module-level singleton `AgentRuntime`, created lazily on first use and torn down via `atexit`: + +```python +# run.py +_default_runtime = None # created on first use, thread-safe +atexit.register(_shutdown_default_runtime) + +def run(agent, prompt, *, runtime=None, **kwargs): + rt = runtime or _get_default_runtime() + return rt.run(agent, prompt, **kwargs) +``` + +This avoids spinning up new Conductor clients and worker processes per call. Workers start once and run until process exit (not stopped after each call). + +`AgentRuntime.run()` flow: + +``` +run(agent, prompt) + -> input guardrails (checked here, before execution; on_fail="raise" -> ValueError) + -> _compile_agent(agent) # cached per agent.name -> ServerCompiledWorkflow + -> _prepare(agent) + _register_workers(agent) # tool_registry.register_tool_workers() + WorkerManager.start() # long-lived; restarts if new tools registered + -> execute workflow_input={prompt, session_id, __agentspan_ctx__} + -> _extract_output(workflow_run, agent) + parse structured output (Pydantic/dataclass) if output_type set + extract handoff result from nested dict + -> AgentResult +``` + +`WorkerManager` is the single long-lived worker host (poll interval from `AGENTSPAN_WORKER_POLL_INTERVAL`, threads from `AGENTSPAN_WORKER_THREADS`). Configuration is loaded by `AgentConfig.from_env()` (`runtime/config.py`): + +``` +AGENTSPAN_* env vars -> AgentConfig (dataclass via from_env()) -> AgentRuntime +``` + +| Field | Env var | Default | +|---|---|---| +| server_url | `AGENTSPAN_SERVER_URL` | `http://localhost:6767/api` | +| api_key | `AGENTSPAN_API_KEY` | — | +| auth_key / auth_secret | `AGENTSPAN_AUTH_KEY` / `AGENTSPAN_AUTH_SECRET` | — | +| log_level | `AGENTSPAN_LOG_LEVEL` | `INFO` | +| llm_retry_count | `AGENTSPAN_LLM_RETRY_COUNT` | 3 | +| worker_poll_interval_ms | `AGENTSPAN_WORKER_POLL_INTERVAL` | 100 | +| worker_thread_count | `AGENTSPAN_WORKER_THREADS` | 1 | +| auto_start_workers | `AGENTSPAN_AUTO_START_WORKERS` | True | +| auto_start_server | `AGENTSPAN_AUTO_START_SERVER` | True | +| daemon_workers | `AGENTSPAN_DAEMON_WORKERS` | True | +| auto_register_integrations | `AGENTSPAN_INTEGRATIONS_AUTO_REGISTER` | False | +| streaming_enabled | `AGENTSPAN_STREAMING_ENABLED` | True | +| secret_strict_mode | `AGENTSPAN_SECRET_STRICT_MODE` | False | +| liveness check | `AGENTSPAN_LIVENESS_ENABLED` | (on) | + +The config is also serialized into the agent config JSON (timeout, retry count, etc.) so the server compiles matching task definitions. + +--- + +## 5. Dispatch worker internals + +`runtime/_dispatch.py` hosts the universal tool-execution router (`dispatch_worker`) — a single Conductor worker task shared by all agents that processes each LLM response. + +``` +LLM response + -> parse (fuzzy: strip markdown fences, extract JSON, normalize keys) + -> is it a tool call (type == "function")? + no -> final answer: continue_loop=False, result=text + yes -> circuit breaker (3 consecutive failures for this tool?) + tripped -> error message, continue_loop=True + ok -> approval_required? + yes -> needs_approval=True, continue_loop=False (HITL pause) + no -> execute tool function + inject ToolContext if declared + coerce args to annotations; validate result JSON-serializable + append result to messages; continue_loop=True +``` + +**Critical Python-specific gotchas** (these are *load-bearing* — other SDKs must replicate the equivalent): + +- **`object` type annotations.** `llm_response` and `messages` parameters are typed `object`, not `dict`/`list`. Conductor's worker framework calls `convert_from_dict_or_list()` on non-simple types, and bare `list`/`dict` crash with `IndexError` inside `typing.get_args()`. `object` short-circuits that. +- **No `from __future__ import annotations`.** `_dispatch.py` deliberately omits it (see the module docstring) because the worker framework needs **real type objects**, not stringized annotations, for runtime parameter resolution. (By contrast `config_serializer.py` *does* use it — it never feeds the worker framework.) +- **Module-level global registries.** Tool functions, per-tool error counts, and approval flags live in module-level dicts in `tool_registry.py` (`_tool_registry`, `_tool_error_counts`, `_tool_approval_flags`). The dispatch worker is registered once per task name and shared across all agents, so per-agent state cannot live on the function — it must be keyed in globals. +- **Framework callables.** Tools marked `_agentspan_framework_callable` (LangChain/LangGraph/OpenAI/Claude adapters) get kwargs/results normalized (`SimpleNamespace` for `ctx`/`context`/`agent`; recursive dataclass/`model_dump`/`__dict__` flattening) before/after invocation. +- **Result serialization guard.** `_validate_serializable()` rejects non-JSON tool returns with `ToolSerializationError` and an actionable message. + +--- + +## 6. Streaming + +`stream()` / `stream_async()` poll the workflow with `include_tasks=True`, track seen task IDs, and emit typed `AgentEvent`s for new/changed tasks. (The SDK also supports SSE via `AgentClient`, falling back to polling when SSE is unavailable.) + +`EventType` (from `result.py`): `thinking`, `tool_call`, `tool_result`, `handoff`, `waiting`, `message`, `error`, `done`, `guardrail_pass`, `guardrail_fail`. + +| Task observed | Condition | Event(s) | +|---|---|---| +| `LLM_CHAT_COMPLETE` | new task | `THINKING` | +| dispatch task (local-compile form) | COMPLETED, `function` in output | `TOOL_CALL` + `TOOL_RESULT` | +| `call_*` tool task (server-compile form) | COMPLETED, non-system task type | `TOOL_CALL` + `TOOL_RESULT` (args stripped of `__agentspan_ctx__`) | +| guardrail task | COMPLETED, `passed` present | `GUARDRAIL_PASS` / `GUARDRAIL_FAIL` | +| `SUB_WORKFLOW` | new task | `HANDOFF` | +| workflow | PAUSED | `WAITING` (carries HITL resume target) | +| workflow | FAILED | `ERROR` | +| workflow | COMPLETED | `DONE` | + +Poll cadence: 0.5s normally; backs off to 2s while waiting on a human (HITL) task. + +--- + +## 7. Guardrails & credentials (SDK-side) + +**Guardrails** run in two places: + +1. **Input guardrails** — checked in `AgentRuntime.run()` *before* workflow execution. `on_fail="raise"` raises `ValueError`. This is a runtime check, not compiled. +2. **Output guardrails** — compiled into the DoWhile loop as durable tasks. Each custom guardrail is registered both as an individual worker (server-compile path, keyed by `guardrail.name`) and bundled into a combined worker (local-compile path). After each guardrail task a `SwitchTask` routes on the result: retry (append feedback + continue), raise (terminate), fix (use corrected output), or human (HumanTask escalation). + +API surface (`guardrail.py`): the `@guardrail` decorator, `OnFail`/`Position` enums (both `str` subclasses, so plain `"retry"`/`"output"` stay backward-compatible), and `Guardrail`/`GuardrailDef`: + +```python +from conductor.ai.agents import guardrail, Guardrail, GuardrailResult, OnFail, Position + +@guardrail +def no_pii(content: str) -> GuardrailResult: ... + +agent = Agent(guardrails=[Guardrail(no_pii, position=Position.OUTPUT, on_fail=OnFail.RETRY)]) +# External guardrail — worker runs in another service, referenced by name only: +agent = Agent(guardrails=[Guardrail(name="compliance_checker", on_fail=OnFail.RETRY)]) +``` + +**Credentials** (`runtime/credentials/`): tool workers call `get_secret(name)` (accessor) which uses `WorkerCredentialFetcher` to fetch secrets from the server, authorized by the `__agentspan_ctx__.execution_token` extracted from the Conductor task input. Errors are typed: `CredentialNotFoundError`, `CredentialAuthError`, `CredentialRateLimitError`, `CredentialServiceError`. See [`../../tool-execution-and-credentials-design.md`](../../tool-execution-and-credentials-design.md) for the full model. + +--- + +## 8. Language-specific design choices / gotchas + +- **No Pydantic dependency in the SDK core.** Models and config use `dataclasses`; Pydantic is touched only when an external framework (e.g. OpenAI structured output) requires it. `output_type` parsing in `_extract_output` handles Pydantic *or* dataclass. +- **`uv`, not `pip`,** for all package management (see `sdk/python/CLAUDE.md`). +- **CLI is a native binary, not Python.** The `agentspan` console script (`cli/__init__.py:main`) locates/downloads a platform-specific native binary (`server.py` / `_ensure_binary()`, honoring `AGENTSPAN_FORCE_DOWNLOAD`) and execs it. The Python modules under `cli/` (`deploy.py`, `discover.py`) are *invoked by* that native CLI — `discover.py` scans `.py` files for module-level `Agent` instances; `deploy.py` deploys them. +- **`from __future__ import annotations` is conditional**, file by file — required to be *absent* in `_dispatch.py` (worker type resolution), fine elsewhere. +- **Tracer name is `conductor.ai.agents`** (OTel), distinct from the `agentspan` wire/operator surface — tracing only activates if `opentelemetry-api` is installed. +- **OpenAI/Claude/LangChain/LangGraph compatibility** is provided via adapters (`openai_compat.py`, `frameworks/*`) that convert foreign agent/tool objects into Agentspan `Agent` objects; the runtime marks the resulting callables `_agentspan_framework_callable` so the dispatch worker normalizes their I/O. See [`../../framework-integration.md`](../../framework-integration.md). + +--- + +## 9. Testing + +> Full validation & e2e design: [python-validation.md](../../validation/python-validation.md) + +Layout under `sdk/python/tests/` (`testpaths = ["tests"]`, 120s per-test timeout): + +| Location | Scope | +|---|---| +| `tests/unit/` | **1701 unit tests** — no server required (agent, tool, compiler, dispatch, runtime, guardrail, memory, result, schedule, skill, framework adapters, examples, etc.) | +| `tests/integration/` | ~130 tests against a live Conductor server | +| `tests/cli/` | ~8 CLI-binary tests | +| `tests/` (root) | `test_kitchen_sink.py` and harnesses (`_worker_harness.py`, `count_workers.py`) | +| `tests/fixtures/`, `tests/compilation_diffs/` | shared fixtures (skills) and compiled-workflow golden diffs | + +Representative unit files: `test_agent.py`, `test_tool.py`/`test_dispatch.py`/`test_dispatch_advanced.py`, `test_compiler.py`/`test_config_serializer.py`/`test_runtime_server_compile.py`, `test_runtime.py`/`test_run.py`, `test_guardrail.py`, `test_memory.py`/`test_result.py`, `test_mcp_discovery.py`, `test_schedule.py`, `test_skill.py`, the `test_langchain_*`/`test_langgraph_*`/`test_claude_agent_sdk_worker.py` framework suites, and `test_example_*` for shipped examples. + +A pytest plugin (`conductor.ai.agents.testing.pytest_plugin`, entry point `agentspan-testing`) plus `testing/` helpers (assertions, mocks, eval runner, recording, semantic/LLM-judge) support agent-level testing. Custom markers: `integration`, `e2e`, `sse`, `agent_correctness`, `semantic`. + +**CI/CD** (`.github/workflows/ci.yml`): unit tests on Python 3.10–3.13, lint with `ruff`, type-check with `mypy`. diff --git a/docs/sdk-design/ruby.md b/design/sdk-design/languages/ruby.md similarity index 99% rename from docs/sdk-design/ruby.md rename to design/sdk-design/languages/ruby.md index 651caf73e..02b6416ed 100644 --- a/docs/sdk-design/ruby.md +++ b/design/sdk-design/languages/ruby.md @@ -1,7 +1,7 @@ # Ruby SDK Translation Guide **Date:** 2026-03-23 -**Base spec:** `docs/sdk-design/2026-03-23-multi-language-sdk-design.md` +**Base spec:** `design/sdk-design.md` **Reference implementation:** `sdk/python/examples/kitchen_sink.py` **Target:** Ruby 3.2+ @@ -1784,7 +1784,7 @@ pii_guardrail = Agentspan::RegexGuardrail.new( # LLM guardrail (server-side, on_fail=FIX) bias_guardrail = Agentspan::LLMGuardrail.new( name: "bias_detector", - model: "openai/gpt-4o-mini", + model: "anthropic/claude-sonnet-4-6", policy: "Check for biased language or stereotypes. If found, provide corrected version.", position: Agentspan::Position::OUTPUT, on_fail: Agentspan::OnFail::FIX, diff --git a/design/sdk-design/languages/typescript-implementation.md b/design/sdk-design/languages/typescript-implementation.md new file mode 100644 index 000000000..8ab49d6d1 --- /dev/null +++ b/design/sdk-design/languages/typescript-implementation.md @@ -0,0 +1,340 @@ +# TypeScript SDK — Reference Implementation + +**Status:** Refreshed 2026-06-26 + +**Scope:** This document describes the TypeScript SDK *as built* — the shipped code under `sdk/typescript/`, published to npm as **`@conductor-oss/conductor-agent-sdk`**. It is a reference for how the SDK is structured and how it behaves at runtime, not a plan for future work. It is present-tense and maps directly to source files. For the cross-language contract and feature set, see the shared design docs ([`../../sdk-design.md`](../../sdk-design.md), [`../../agentspan-design.md`](../../agentspan-design.md), [`../../api-design.md`](../../api-design.md), [`../../framework-integration.md`](../../framework-integration.md), [`../../tool-execution-and-credentials-design.md`](../../tool-execution-and-credentials-design.md)). For language idioms and ergonomics, see the sibling guide [`typescript.md`](./typescript.md). For the validation harness, see [`../../validation/typescript-validation.md`](../../validation/typescript-validation.md). + +--- + +## 1. Overview + +The TypeScript SDK lets you define agents, tools, guardrails, memory, and multi-agent strategies in TypeScript, compile them to the Agentspan wire format (`AgentConfig` JSON), and run them on Agentspan' durable Conductor-backed runtime. It also runs native framework agents — Vercel AI SDK, LangGraph.js, LangChain.js, OpenAI Agents, Google ADK — on the same runtime via auto-detection and drop-in wrappers. + +### 1.1 Design choices (as built) + +| Aspect | Decision | +|--------|----------| +| Language | TypeScript-first (`.ts` source, compiled to ESM + CJS) | +| Package | `@conductor-oss/conductor-agent-sdk` | +| Runtime | Node.js 18+ (native `fetch`, `AbortController`, `ReadableStream`, `crypto.randomUUID`) | +| Schema | **Superset** — accepts both Zod schemas and raw JSON Schema, auto-detecting format; Zod is an optional peer | +| Framework integration | Auto-detecting runtime (`detectFramework`) plus drop-in `./vercel-ai`, `./langgraph`, `./langchain` wrappers | +| Conductor client | Worker polling runs on `@io-orkes/conductor-javascript`'s `TaskManager`; all `/agent/*` control-plane calls use a raw `fetch` client (`AgentClient`) that mints/refreshes an Orkes JWT | +| Build | `tsup` → ESM + CJS dual output with `.d.ts` declarations, multiple entry points | +| Test runner | Vitest (unit + e2e suites) | +| API style | Options-object pattern; composition via `.and()`/`.or()`/`.pipe()` methods (no operator overloading) | + +> **Worker transport note:** Earlier drafts proposed dropping `@io-orkes/conductor-javascript` and polling with raw `fetch`. The shipped implementation keeps the Conductor JS client for *task polling* (`WorkerManager` wraps its `TaskManager`), because it provides lease extension, concurrency control, and retry handling for free. The Agentspan-specific middleware (ToolContext extraction, credential injection, state capture, circuit breaker, error mapping) runs inside each worker's `execute()` callback. The control plane (`/agent/start`, `/agent/compile`, status, respond, schedules) uses the SDK's own `fetch`-based `AgentClient`. + +### 1.2 Runtime contracts (intentional, kept stable) + +- **Environment variables** are `AGENTSPAN_*` (see §9). +- **Credential routing context** key in task input is `__agentspan_ctx__`. + +--- + +## 2. Package & source layout + +### 2.1 Dependencies + +| Dependency | Type | Purpose | +|-----------|------|---------| +| `@io-orkes/conductor-javascript` | runtime | Worker task polling (`TaskManager`, `createConductorClient`) | +| `dotenv` | runtime | `.env` loading on import | +| `zod` | peer (optional) | Schema validation + type inference | +| `zod-to-json-schema` | peer (optional) | Convert Zod → JSON Schema at serialization time | +| `ai` | peer (optional) | Vercel AI SDK passthrough/wrapper | +| `@langchain/core` | peer (optional) | LangChain.js passthrough/wrapper | +| `@langchain/langgraph` | peer (optional) | LangGraph.js passthrough/wrapper | + +All framework peers and `zod`/`zod-to-json-schema` are marked optional in `peerDependenciesMeta`; the SDK works without any of them installed (detection and serialization use duck-typing). + +### 2.2 package.json highlights + +```jsonc +{ + "name": "@conductor-oss/conductor-agent-sdk", + "type": "module", + "main": "./dist/index.cjs", + "module": "./dist/index.js", + "types": "./dist/index.d.ts", + "exports": { + ".": { "import": "./dist/index.js", "require": "./dist/index.cjs" }, + "./testing": { "import": "./dist/testing/index.js", "require": "./dist/testing/index.cjs" }, + "./vercel-ai": { "import": "./dist/wrappers/ai.js", "require": "./dist/wrappers/ai.cjs" }, + "./langgraph": { "import": "./dist/wrappers/langgraph.js","require": "./dist/wrappers/langgraph.cjs" }, + "./langchain": { "import": "./dist/wrappers/langchain.js","require": "./dist/wrappers/langchain.cjs" } + }, + "engines": { "node": ">=18.0.0" } +} +``` + +### 2.3 Source map (`sdk/typescript/src/`) + +``` +src/ + index.ts # Public re-exports (single barrel) + agent.ts # Agent, PromptTemplate, scatterGather, agent(), @AgentDec, agentsFrom + tool.ts # tool(), server-side tool ctors, @Tool, toolsFrom, waitForMessageTool + guardrail.ts # guardrail(), RegexGuardrail, LLMGuardrail, @Guardrail + termination.ts # TerminationCondition + TextMention/StopMessage/MaxMessage/TokenUsageCondition, And/Or + handoff.ts # OnToolResult, OnTextMention, OnCondition, TextGate, gate() + memory.ts # ConversationMemory, SemanticMemory, InMemoryStore + credentials.ts # getCredential, resolveCredentials, execution-token + credential-context plumbing + callback.ts # CallbackHandler (6 lifecycle positions) + worker-name helpers + code-execution.ts # CodeExecutor + Local/Docker/Jupyter/Serverless + CommandValidator + cli-config.ts # makeCliTool() — run_command CLI tool + claude-code.ts # ClaudeCode model wrapper, PermissionMode, resolveClaudeCodeModel + skill.ts # skill(), loadSkills(), createSkillWorkers (Agent Skills from SKILL.md) + plans.ts # Plan/Step/Op/Generate/Validation/Action/Ref/Context (PLAN_EXECUTE builders) + schedule.ts # Schedule, ScheduleClient (cron schedules for deployed agents) + schedules-api.ts # module-function facade over ScheduleClient (singleton) + ext.ts # GPTAssistantAgent + discovery.ts # discoverAgents(path) + tracing.ts # isTracingEnabled() (OTel env probe) + types.ts # Shared interfaces/enums, AgentResult helpers, event/output normalizers + errors.ts # AgentspanError hierarchy + config.ts # AgentConfig — env loading + URL normalization + serializer.ts # AgentConfigSerializer — Agent tree → AgentConfig JSON + worker.ts # WorkerManager (wraps Conductor TaskManager) + coercion/circuit-breaker/state-capture + runtime.ts # AgentRuntime — run/start/stream/deploy/plan/serve/shutdown + system-worker registration + agent-client.ts # AgentClient — /agent/* control-plane client, Orkes JWT auth, ClientHandle + workflow-client.ts # WorkflowClient — read-only Conductor execution + token-usage aggregation + stream.ts # AgentStream — SSE client (AsyncIterable) + HITL + polling fallback + frameworks/ + detect.ts # detectFramework() — duck-typing + serializer.ts # serializeFrameworkAgent() — generic deep object → rawConfig + WorkerInfo[] + langgraph-serializer.ts # serializeLangGraph() — CompiledStateGraph extraction + langchain-serializer.ts # serializeLangChain() — AgentExecutor extraction + wrappers/ + ai.ts # ./vercel-ai — drop-in generateText/streamText + langgraph.ts # ./langgraph — drop-in createReactAgent + langchain.ts # ./langchain — drop-in createAgentExecutor / runnable metadata + testing/ + index.ts, mock.ts, expect.ts, assertions.ts, eval.ts, strategy.ts, recording.ts +cli-bin/ + deploy.ts, discover.ts, shared.ts # Node scripts (deploy/discover agents from a directory) +``` + +### 2.4 Build entry points (`tsup.config.ts`) + +```ts +entry: ['src/index.ts', 'src/testing/index.ts', + 'src/wrappers/ai.ts', 'src/wrappers/langgraph.ts', 'src/wrappers/langchain.ts'], +format: ['esm', 'cjs'], dts: true, splitting: true, sourcemap: true, target: 'node18' +``` + +`npm run build` runs `tsup` then `scripts/verify-dist.mjs` to assert the dual-format output is well-formed. + +--- + +## 3. Compilation & serialization + +`AgentConfigSerializer` (`serializer.ts`) recursively converts an `Agent` tree into the wire format from [`../../agentspan-design.md`](../../agentspan-design.md). `serialize(agent, prompt?, opts?)` returns the full `POST /api/agent/start` payload; the same serializer feeds `POST /api/agent/compile` (via `plan()`) and `POST /api/agent/deploy`. + +Key rules (as built): + +- All keys are **camelCase**; `null`/`undefined` values are omitted. +- `agents` holds nested `AgentConfig` objects; `strategy` is only emitted when `agents` is non-empty. +- Zod schemas (tool `inputSchema`/`outputSchema`, agent `outputType`) are converted via `zod-to-json-schema` at serialization time; raw JSON Schema passes through unchanged. +- `instructions` may be a string, a `PromptTemplate`, or a function (functions register as system workers). +- Guardrails, handoffs, termination conditions, gates, callbacks, `codeExecution`, `cliConfig`, and `credentials` each have a dedicated serialization path. +- Skill agents (`_framework: "skill"`) bypass `AgentConfig` and serialize to a framework `rawConfig` (see §7); the runtime pre-deploys them before serialization and replaces the `agent_tool` entry with a `workflowName` reference. +- PLAN_EXECUTE agents emit `planner`/`plannerContext` (and a static plan via `static_plan` when `RunOptions.plan` is supplied — see §6.7). + +`frameworks/serializer.ts` (`serializeFrameworkAgent`) is a framework-agnostic deep walker that turns an arbitrary agent object into `[rawConfig, WorkerInfo[]]`: primitives/enums pass through, callables become `{_worker_ref, description, parameters}` stubs, tool objects are extracted, and circular references are caught. + +--- + +## 4. Runtime lifecycle & worker system + +### 4.1 AgentRuntime (`runtime.ts`) + +```ts +class AgentRuntime { + constructor(options?: AgentConfigOptions); + run(agent, prompt, options?): Promise; // start, register workers, drain SSE, enrich, return + start(agent, prompt, options?): Promise; // fire-and-forget; handle with wait/respond/pause/... + stream(agent, prompt, options?): Promise; // start() then handle.stream() + deploy(agent, { schedules? }?): Promise;// compile + register; optional schedule reconcile + plan(agent): Promise; // POST /agent/compile (dry run) + serve(...agents): Promise; // register workers, poll, block until SIGINT/SIGTERM + shutdown(): Promise; // stop polling + get workflows(): WorkflowClient; // read-only execution client + schedulesClient(): ScheduleClient; +} +``` + +The `agent` argument is `Agent | object`. The runtime calls `detectFramework(agent)` first; framework objects take the passthrough path (`_runFramework`/`_startFramework`), native `Agent` instances take the standard path. Module-level singleton functions (`configure`, `run`, `start`, `stream`, `deploy`, `plan`, `serve`, `shutdown`) delegate to a lazily-created `AgentRuntime`. + +**`run()` flow (native agent):** +1. Generate a correlation id (`crypto.randomUUID`). +2. Pre-deploy any nested skill agents (`_preDeployNestedSkills`). +3. If any tool/sub-agent is stateful, mint a `runId` (worker domain) for isolation. +4. Serialize the agent + prompt → payload; attach `timeoutSeconds`/`credentials`/`context`/`runId`/`static_plan` from options. +5. Register tool workers and skill workers (in the run's domain). +6. `POST /agent/start`; read `executionId` and optional `requiredWorkers`. +7. Register **system** workers (only those in `requiredWorkers`, or all on older servers); start polling. +8. Open an `AgentStream`, drain events, build `AgentResult`, then **enrich** it from `GET /agent/execution/{id}` (tool calls, messages, recursively-aggregated token usage, and a non-junk output fallback). +9. `finally` stop polling. + +### 4.2 System workers registered by the runtime + +`_registerSystemWorkers` walks the agent tree and registers Conductor SIMPLE workers for any feature backed by user code. Naming conventions (collected recursively): + +| Worker | Task name | +|--------|-----------| +| Tool | `{tool.name}` | +| Tool / agent guardrail (custom) | `{guardrail.taskName}` | +| Termination condition | `{agentName}_termination` | +| `stopWhen` | `{agentName}_stop_when` | +| Callback (per position) | `{agentName}_{position}` (`before_agent`…`after_tool`) | +| Gate (callable) | `{agentName}_gate` → `{ decision: "continue" \| "stop" }` | +| Router function | `{agentName}_router_fn` → `{ selected_agent }` | +| Swarm transfer (no-op / blocked-error) | `{source}_transfer_to_{target}` | +| Check transfer | `{agentName}_check_transfer` → `{ is_transfer, transfer_to }` | +| Handoff check (swarm) | `{agentName}_handoff_check` → `{ active_agent, handoff }` | +| Manual selection | `{agentName}_process_selection` → `{ selected }` | + +### 4.3 WorkerManager (`worker.ts`) + +`WorkerManager` is a thin lifecycle wrapper over Conductor's `TaskManager`. `addWorker(taskName, handler, credentials?, domain?)` queues a worker; `startPolling()` builds a `createConductorClient` (overriding `CONDUCTOR_SERVER_URL`, injecting auth headers per request via a `headersProvider` that resolves the Orkes JWT) and starts the `TaskManager`. `(taskName, domain)` pairs are distinct workers, enabling stateful-domain isolation. Each wrapped worker's `execute()` runs the full middleware chain: + +1. **Circuit breaker** — 10 consecutive failures opens the breaker (`NonRetryableException` thereafter); any success resets it; `resetCircuitBreaker(name)` / `resetAllCircuitBreakers()` are exported. +2. **ToolContext extraction** — reads `__agentspan_ctx__`, builds a `ToolContext` with a mutable copy of `state`, snapshots state. +3. **Key stripping** — removes `_agent_state`, `method`, `__agentspan_ctx__` from input before the handler sees it. +4. **Credential resolution** — if the worker declares credentials, extracts the execution token and resolves values up-front; injection happens atomically inside `injectSecretsForInvocation` (process-wide lock). +5. **State capture** — diffs `ToolContext.state` before/after and appends `_state_updates` to the result (merged into objects, or wrapped as `{ result, _state_updates }` for primitives). +6. **Output shaping** — non-object results are wrapped as `{ result }` (Conductor requires object `outputData`). +7. **Error mapping** — `TerminalToolError` → `NonRetryableException`; other errors propagate (retryable). + +**Type coercion** (`coerceValue`) converts Conductor-typed inputs (string⇄number/boolean, string⇄object/array via `JSON.parse`/`stringify`); all failures are silent and return the original value. + +--- + +## 5. Streaming / SSE client (`stream.ts`) + +`AgentStream implements AsyncIterable` over `GET /agent/stream/{executionId}`, consumed with `for await...of`. + +- **Transport:** native `fetch` + `ReadableStream` (so custom `Authorization`/`X-Authorization` headers work — `EventSource` can't send those). Parses `event:`/`id:`/`data:` fields; `:`-prefixed lines are heartbeats. +- **Timeout:** no real event within 15s → fall through to polling. +- **Reconnection:** on drop, up to 5 retries with linear backoff (`1s * attempt`), resuming with `Last-Event-ID`. +- **Polling fallback:** poll `GET /agent/{id}/status` every 500ms until terminal, emitting a synthetic terminal event. +- **HITL:** `respond(output)`, `approve(output?)`, `reject(reason?)`, `send(message)`. +- **Result:** `getResult()` drains the stream and builds an `AgentResult`; internal event keys are stripped via `stripInternalEventKeys`. + +`AgentEvent.type` is `EventType | string` so server-only event types (`context_condensed`, `subagent_start`, `subagent_stop`) pass through to users untouched. + +--- + +## 6. Type system, tools, guardrails, memory, credentials, callbacks, code execution + +### 6.1 Type system + +Enums are string unions (`Strategy`, `EventType`, `Status`, `FinishReason`, `OnFail`, `Position`, `ToolType`, `GuardrailType`, `FrameworkId`). Core data shapes (`types.ts`): `TokenUsage`, `ToolContext` (with mutable `state`), `GuardrailResult`, `AgentEvent`, `AgentResult`, `AgentStatus`, `DeploymentInfo`, `RunOptions`, `ToolDef`. Helpers `createAgentResult`, `normalizeOutput`, `stripInternalEventKeys` enforce the result/event invariants. `AgentResult.output` is always normalized to a `Record` (strings wrapped as `{ result }`, COMPLETED null → `{ result: null }`, FAILED null → `{ error }`). + +### 6.2 Tools (superset) + +`tool(fn, options)` returns a callable `ToolFunction` carrying a hidden `_toolDef`. `inputSchema`/`outputSchema` accept Zod or JSON Schema. `normalizeToolInput` accepts (a) Agentspan `ToolDef`s, (b) Vercel AI SDK `tool()` objects (Zod `inputSchema` + `execute`), and (c) raw `{name, description, inputSchema}` objects — so all three coexist in one `tools` array. `external: true` emits schema only (no local worker). Server-side constructors (no local worker): `httpTool`, `apiTool`, `mcpTool`, `agentTool`, `humanTool`, `imageTool`, `audioTool`, `videoTool`, `pdfTool`, `searchTool`, `indexTool`, `waitForMessageTool`. Class-method form: `@Tool` decorator + `toolsFrom(instance)`. + +### 6.3 Guardrails (`guardrail.ts`) + +`guardrail(fn, {name, position?, onFail?, maxRetries?})` registers a custom SIMPLE worker. `RegexGuardrail` (server-side inline JS, block/allow) and `LLMGuardrail` (server-side `LLM_CHAT_COMPLETE`) need no worker. `guardrail.external` and the `@Guardrail` decorator + `guardrailsFrom` are also provided. `position` defaults to `output`; `onFail` ∈ `retry|raise|fix|human`. + +### 6.4 Memory (`memory.ts`) + +`ConversationMemory` (session history, optional `maxMessages` windowing that preserves system messages, serialized as `{messages, maxMessages}`). `SemanticMemory` over a pluggable `MemoryStore`; `InMemoryStore` ships a keyword-overlap similarity with no external deps. + +### 6.5 Credentials (`credentials.ts`) + +`getCredential(name)`, `resolveCredentials(serverUrl, headers, token, names)`, `extractExecutionToken`, and the `setCredentialContext`/`runWithCredentialContext`/`clearCredentialContext` plumbing that scopes credentials per async invocation so concurrent workers don't clobber each other. The worker extracts the execution token from `__agentspan_ctx__` (with a workflow-input fallback for sub-agents) and resolves credentials via the server before invoking the handler. Errors map to `CredentialNotFoundError` / `CredentialAuthError` / `CredentialRateLimitError` / `CredentialServiceError`. + +### 6.6 Callbacks (`callback.ts`) + +`CallbackHandler` with six optional async hooks (`onAgentStart`/`End`, `onModelStart`/`End`, `onToolStart`/`End`). Each implemented method registers a SIMPLE worker at `{agentName}_{position}` (`before_agent`…`after_tool`). + +### 6.7 Code execution, CLI, Claude Code, plans, skills + +- **Code execution** (`code-execution.ts`): abstract `CodeExecutor` (+ `asTool()`) with `Local`, `Docker`, `Jupyter`, `Serverless` implementations and a `CommandValidator`. Agent-level `CodeExecutionConfig` / `CliConfig`. +- **CLI tool** (`cli-config.ts`): `makeCliTool()` produces a `run_command` tool with quoting-aware tokenization, command whitelist, exit-code capture, and context-state persistence. +- **Claude Code** (`claude-code.ts`): `ClaudeCode` model wrapper → `"claude-code/"`, `PermissionMode` enum, `resolveClaudeCodeModel()` alias mapping. +- **Plans** (`plans.ts`): typed `PLAN_EXECUTE` builders — `Plan`, `Step`, `Op`, `Generate`, `Validation`, `Action`, `Ref`, `Context` — each with `toJSON()`; `coercePlan()` normalizes a `Plan`-or-object. A static plan supplied via `RunOptions.plan` is sent as `static_plan` and wins over the planner LLM. +- **Skills** (`skill.ts`): `skill()` loads an Agent Skill from a `SKILL.md` directory (frontmatter, `*-agent.md` sub-agents, scripts, cross-skill refs), returning an `Agent` marked `_framework: "skill"`. `loadSkills()` batch-loads; `createSkillWorkers()` builds script-execution workers plus a `read_skill_file` tool. + +### 6.8 Execution API surface + +`RunOptions`: `sessionId`, `media`, `idempotencyKey`, `timeoutSeconds`, `signal` (AbortSignal), `credentials`, `context`, `plan`, `model`. `AgentHandle`: `executionId`, `correlationId`, `getStatus`, `wait(pollIntervalMs?)`, `respond/approve/reject/send`, `pause`, `resume`, `cancel`, `stream`. Control-plane-only execution (no local workers) is available through `AgentClient` → `ClientHandle` (`agent-client.ts`), and read-only execution inspection through `WorkflowClient` (`workflow-client.ts`). Cron scheduling for deployed agents: `Schedule` + `ScheduleClient` (`schedule.ts`) with a declarative `reconcile()`, plus the `schedules` module facade. + +--- + +## 7. Framework auto-detection & integration + +See [`../../framework-integration.md`](../../framework-integration.md) for the server-side contract. + +### 7.1 Detection (`frameworks/detect.ts`) + +`detectFramework(agent)` returns `FrameworkId | null` via duck-typing (no framework imports): + +| Result | Signature checked | +|--------|-------------------| +| `"skill"` | `_framework === "skill"` | +| `null` (native) | `instanceof Agent` | +| `"langgraph"` | `.invoke()` + (`.getGraph()` or `.nodes`) | +| `"langchain"` | `.invoke()` + `.lc_namespace` | +| `"openai"` | `name` + `instructions` + `model` + `tools` + `handoffs` + `asTool` | +| `"google_adk"` | `model` + `instruction` + ADK-specific props | + +LangGraph, LangChain, OpenAI Agents, Google ADK, and skills are auto-detected and run through the passthrough path. **Vercel AI SDK** integration is delivered through the `./vercel-ai` wrapper rather than runtime auto-detection. + +### 7.2 Passthrough path + +For framework objects, `run()`/`start()` call `_serializeFramework` → `[rawConfig, WorkerInfo[]]`, register the extracted workers, start polling, then `POST /agent/start { framework, rawConfig, prompt, sessionId?, credentials? }`. The server normalizer compiles a passthrough `WorkflowDef` whose SIMPLE task is served by the registered worker; events stream back over SSE like any native run. + +- `serializeLangGraph` (`langgraph-serializer.ts`) has three paths: full extraction (model + ToolNode → AI_MODEL + SIMPLE), graph-structure extraction (custom `StateGraph` nodes/edges/reducers/retry policies, detecting LLM and subgraph nodes by patching `invoke`), and single-SIMPLE passthrough. +- `serializeLangChain` (`langchain-serializer.ts`) checks for `_agentspan` wrapper metadata first, else extracts model + tools, else passes through. +- `serializeFrameworkAgent` (`serializer.ts`) handles OpenAI Agents and Google ADK generically. + +### 7.3 Drop-in wrappers (subpath exports) + +- **`./vercel-ai`** (`wrappers/ai.ts`): re-exports the `ai` module and wraps `generateText`/`streamText` to intercept model/tools/system/prompt, compile to an `Agent`, run on `AgentRuntime`, and map results back to AI SDK shape. +- **`./langgraph`** (`wrappers/langgraph.ts`): `createReactAgent()` proxies the original and stamps `_agentspan` metadata so the serializer fast-paths it. +- **`./langchain`** (`wrappers/langchain.ts`): `createAgentExecutor()` / runnable-metadata helpers that capture LLM/tools/instructions as `_agentspan` metadata. + +Wrappers infer the `provider/model` string from LLM class/model names (Anthropic, Google, Bedrock, OpenAI). Framework packages are lazy-loaded on first use. + +--- + +## 8. Build & packaging + +`tsup` produces dual ESM + CJS with `.d.ts` for five entry points (§2.4); `npm run build` verifies the output with `scripts/verify-dist.mjs`. Node 18+ is the supported runtime. The package marks Node-only modules (worker, credentials, code execution) as such; browser consumers can use the REST/SSE surface but not worker polling, tool execution, or credential resolution. Lint/format via ESLint + Prettier; typecheck via `tsc --noEmit`. + +--- + +## 9. Language-specific design choices & gotchas + +- **Environment variables** (`config.ts`, all `AGENTSPAN_*`): `SERVER_URL` (default `http://localhost:6767/api`), `API_KEY`, `AUTH_KEY`, `AUTH_SECRET`, `WORKER_POLL_INTERVAL` (100ms), `WORKER_THREADS` (1), `AUTO_START_WORKERS`/`AUTO_START_SERVER`/`DAEMON_WORKERS`/`STREAMING_ENABLED` (true), `CREDENTIAL_STRICT_MODE` (false), `LLM_RETRY_COUNT` (3), `LOG_LEVEL` (`INFO`). `.env` is loaded on import via `dotenv`. +- **URL normalization:** `serverUrl` is stripped of trailing slashes and gets `/api` appended if missing. The Conductor worker client polls the base URL (with `/api` stripped). +- **Composition without operator overloading:** termination uses `.and()`/`.or()`; agent chaining uses `.pipe()` (which flattens `a.pipe(b).pipe(c)` into one sequential agent, not a nested tree). +- **Swarm transfers** are generated server-side from `strategy: 'swarm'`; the SDK only registers the no-op/blocked transfer workers and the `check_transfer`/`handoff_check` workers. Don't add `transfer_to_*` tools manually. +- **State mutation capture** appends `_state_updates`; the server persists state and strips the key from user-visible output. +- **Output "junk" repair:** the runtime treats `{result: null, finishReason: ...}` and `{result: []}` as junk and falls back to the execution's last assistant message / workflow output. +- **Token usage** is aggregated recursively across `SUB_WORKFLOW` tasks (`workflow-client.ts` / `runtime._collectTokensById`). +- **Idempotency:** `RunOptions.idempotencyKey` maps to Conductor `correlationId`; the server dedupes against RUNNING/COMPLETED (never FAILED) executions. `AgentHandle.correlationId` is a fresh UUID per call. +- **Cancellation/timeout:** every network call accepts an `AbortSignal` (`AbortSignal.timeout(ms)` or a manual `AbortController`). +- **`Status` is terminal-only** (`COMPLETED|FAILED|TERMINATED|TIMED_OUT`); `AgentStatus.status` is a `string` and may carry non-terminal Conductor states — prefer the `isComplete`/`isRunning`/`isWaiting` flags. +- **Stateful domains:** when an agent (or any descendant) is stateful, the run mints a `runId` used as the Conductor worker `domain`, so every worker for that execution polls in an isolated domain. +- **Tracing:** `isTracingEnabled()` only probes `OTEL_EXPORTER_OTLP_ENDPOINT`/`OTEL_SERVICE_NAME`; OTel SDK wiring is the caller's responsibility. +- **CLI helper scripts** live in `cli-bin/` (`deploy.ts`, `discover.ts`) and are invoked via `node`/`tsx`; there is no separately published binary. + +--- + +## 10. Testing + +> Full validation & e2e design: [typescript-validation.md](../../validation/typescript-validation.md) + +Vitest, configured in `vitest.config.ts` (decorator support, 60s timeout, `forks` pool with up to 3 workers, JUnit reporter to `e2e-results/junit-ts.xml`). The package import is aliased to `src/index.ts` for in-tree testing. + +- **Unit** (`tests/unit/`, ~45 files): per-module coverage — agent, tool, serializer, worker, runtime, stream, guardrail, termination, handoff, memory, credentials, callback, code-execution, cli-config, schedule, plans, skill, config, result, agent-client-auth, concurrent-injection, context-passing, planner-context, swarm-workers, kitchen-sink-structural, plus `frameworks/`, `wrappers/`, `testing/`, and `validation/` subtrees. +- **E2E** (`tests/e2e/`, ~24 suites, require a running server): basic validation, tool/CLI/MCP/HTTP/PDF/media tools, guardrails (+ matrix), handoffs, multi-agent matrix, LangGraph, termination/gates, callbacks, lease extension, stateful domain, behavioral correctness, skills, streaming, token usage, plan-execute, scheduling, wait-for-message tool, and the agent client. Per-language e2e wiring is documented in [`../../validation/typescript-validation.md`](../../validation/typescript-validation.md). +- **Testing utilities** (`./testing` export): `mockRun()`, `expectResult()` fluent assertions, individual `assert*` helpers, `record()`/`replay()` fixtures, `validateStrategy()`, and a `CorrectnessEval` LLM judge. +- **Fixtures:** `tests/_configs/*.json` hold expected wire-format snapshots; `tests/fixtures/skills/` provide sample SKILL.md trees. diff --git a/docs/sdk-design/typescript.md b/design/sdk-design/languages/typescript.md similarity index 99% rename from docs/sdk-design/typescript.md rename to design/sdk-design/languages/typescript.md index c82a0aebe..487dbd13b 100644 --- a/docs/sdk-design/typescript.md +++ b/design/sdk-design/languages/typescript.md @@ -1,8 +1,9 @@ # TypeScript SDK Translation Guide **Date:** 2026-03-23 -**Base spec:** `docs/sdk-design/2026-03-23-multi-language-sdk-design.md` +**Base spec:** `design/sdk-design.md` **Reference implementation:** `sdk/python/examples/kitchen_sink.py` +**As-built internals:** [typescript-implementation.md](typescript-implementation.md) --- @@ -1362,7 +1363,7 @@ The validation runner executes kitchen sink examples against multiple models con ```toml # validation/runs.toml [judge] -model = "openai/gpt-4o-mini" +model = "anthropic/claude-sonnet-4-6" max_output_chars = 3000 max_tokens = 300 rate_limit = 0.5 @@ -1579,7 +1580,7 @@ const piiGuardrail = new RegexGuardrail({ const biasGuardrail = new LLMGuardrail({ name: "bias_detector", - model: "openai/gpt-4o-mini", + model: "anthropic/claude-sonnet-4-6", policy: "Check for biased language or stereotypes. If found, provide corrected version.", position: "output", onFail: "fix", diff --git a/design/sentinel-agents.md b/design/sentinel-agents.md new file mode 100644 index 000000000..216f390fc --- /dev/null +++ b/design/sentinel-agents.md @@ -0,0 +1,785 @@ +# Sentinel Agents + +**Status:** Consolidated 2026-06-26 (scheduling shipped; other triggers roadmap) + +**Scope:** Sentinel Agents are always-on, event-driven agents — regular agents (model + tools + instructions) augmented with **triggers** that define when and how they activate. Where today's agentic frameworks are request-response (a human prompts, the agent runs once, returns), sentinels are autonomous background processes with LLM brains: a scheduled health-checker, a log watcher that files tickets on errors, an incident responder that wakes on a PagerDuty alert, a PR reviewer triggered by a webhook. Each needs **activation** (something to wake it), **context** (the triggering data injected into its prompt), **tools** (actions it can take), **durability** (retry, audit, timeout), and **lifecycle management** (deploy, pause, resume, undeploy, observe). This doc defines the trigger model, the **shipped** Phase 1 (declarative cron scheduling, across all four SDKs and the UI), and the roadmap for the remaining trigger types. See also [`agentspan-design.md`](agentspan-design.md), [`sdk-design.md`](sdk-design.md), and [`stateful-agents.md`](stateful-agents.md). + +--- + +## 1. Scope & Vision + +A **Sentinel Agent** is a regular agent with one addition — **triggers** that define when and how it activates: + +``` +Sentinel Agent = Agent + Triggers +``` + +The agent definition describes **what** it does. Triggers describe **when** it runs. + +The activation layer sits in front of agent execution. A trigger fires, injects context into the prompt template, the agent executes (LLM ↔ tools, with guardrails and optional memory), and the result drives an action — an alert, a fix, a report, a ticket, a restart. + +``` +┌──────────────────────────────────────────────────────────────┐ +│ ACTIVATION LAYER │ +│ │ +│ ┌──────────┐ ┌───────────┐ ┌─────────┐ ┌───────────────┐ │ +│ │ Schedule │ │ Event │ │ Webhook │ │ Source Watch │ │ +│ │ (cron) │ │ (pub/sub) │ │ (HTTP) │ │ (file/stream) │ │ +│ └────┬─────┘ └─────┬─────┘ └────┬────┘ └───────┬───────┘ │ +│ │ │ │ │ │ +│ └──────────────┴────────────┴──────────────┘ │ +│ │ │ +│ ┌──────▼──────┐ │ +│ │ TRIGGER │ ← context injected │ +│ │ (prompt) │ into prompt template │ +│ └──────┬──────┘ │ +│ │ │ +│ ┌────────────▼─────────────┐ │ +│ │ AGENT EXECUTION │ │ +│ │ ┌─────┐ ┌─────┐ ┌────┐ │ │ +│ │ │ LLM │→│Tools│→│ LLM│ │ │ +│ │ └─────┘ └─────┘ └────┘ │ │ +│ │ + guardrails + memory │ │ +│ └────────────┬─────────────┘ │ +│ │ │ +│ ┌──────▼──────┐ │ +│ │ ACTION │ (alert, fix, report, │ +│ │ (output) │ create ticket, restart...) │ +│ └─────────────┘ │ +└──────────────────────────────────────────────────────────────┘ +``` + +### Landscape & the gap + +No current framework provides a clean, unified model for always-on, multi-trigger agents with durable execution and simple deployment. + +| Capability | AutoGen | CrewAI | LangGraph | **Ours (Goal)** | +|---|---|---|---|---| +| Cron scheduling | - | - | Partial | **Yes (shipped)** | +| Event triggers | Internal | Internal | Webhooks | **Yes (Conductor events)** | +| Webhook triggers | - | - | Status only | **Yes** | +| File/log watching | - | - | - | **Yes (local daemon)** | +| Stream watching (Kafka/Redis) | - | - | - | **Yes (local consumer)** | +| Multi-trigger per agent | - | - | - | **Yes** | +| Durable execution (retry, audit) | - | - | Yes | **Yes (Conductor)** | +| Simple deployment | - | - | Managed service | **Yes (pip + one command)** | + +- **AutoGen v0.4 (AG2)** — async, event-driven actor model with rich event surfacing. *Gap*: no deployment model, no external triggers; you run it, it runs once. +- **CrewAI** — Crews (agent groups) vs. Flows (event-driven pipelines). *Gap*: "events" are internal flow routing, not external activation; no cron/file/webhook out of the box. +- **LangGraph Platform** — background runs, task queue, webhook status callbacks, mentions cron. *Gap*: a managed deployment service, not a framework primitive; limited event sources; no local daemon patterns. +- **OmniDaemon (research)** — daemon + topic subscription (e.g. Redis streams). *Gap*: single event-source type; no scheduling, no file watching. +- **Microsoft Sentinel (security)** — sidecar deployment, continuous behavioral monitoring, hybrid rule + LLM auditing. *Gap*: domain-specific, not general-purpose. + +## 2. Trigger Model + +Five trigger types span the spectrum from fully server-managed to local-daemon-driven. **Schedule is shipped today (Phase 1)**; the rest are roadmap (Section 4). + +| Trigger | Activation | Deployment | Status | +|---|---|---|---| +| **Schedule** | cron expression | server-side | **Shipped** | +| **EventTrigger** | named pub/sub event | server-side | Roadmap | +| **WebhookTrigger** | matching HTTP request | server-side | Roadmap | +| **FileWatch** | local file matches pattern | local watcher | Roadmap | +| **StreamWatch** | message on a stream | local consumer | Roadmap | + +#### Schedule +Runs the agent on a cron expression. Server-side — the orchestration platform handles timing. See Section 3 for the shipped API. + +``` +Schedule: + cron: "*/5 * * * *" # every 5 minutes + prompt: "Run health checks on all production services." + timezone: "UTC" # optional + start_time: null # optional window start + end_time: null # optional window end + catch_up: false # run missed executions? +``` + +**Maps to**: Conductor `SchedulerClient.save_schedule()` / any execution engine's cron scheduler. + +#### Event Trigger +Runs the agent when a named event fires. The event payload is injected into the prompt. + +``` +EventTrigger: + event: "pagerduty:incident" # event source/name + condition: "event.severity == 'critical'" # optional filter + prompt: "Critical incident: ${event.title}\nDetails: ${event.description}" +``` + +**Maps to**: Conductor `EventHandler` / Kafka consumer / any pub-sub system. + +#### Webhook Trigger +Runs the agent when an HTTP request arrives matching certain criteria. + +``` +WebhookTrigger: + matches: # payload matching criteria + type: "pull_request" + action: "opened" + prompt: "New PR opened: ${webhook.payload}" +``` + +**Maps to**: Conductor webhook receiver / any HTTP endpoint that starts a workflow. + +#### File Watch +Runs the agent when a local file matches a pattern. Requires a local watcher process. + +``` +FileWatch: + path: "/var/log/myapp/*.log" # file path or glob + pattern: "(ERROR|CRITICAL)" # regex to match + prompt: "Log error detected:\n\n{matched_lines}" + debounce: 30 # seconds between triggers (prevent storm) + lookback: 50 # lines of context around match +``` + +**Maps to**: Local daemon thread that tails files → calls workflow start API on match. + +#### Stream Watch +Runs the agent when a message appears on a message stream. + +``` +StreamWatch: + source: "kafka://alerts-topic" # or redis://stream, sqs://queue, etc. + filter: "message.level == 'error'" + prompt: "Alert from stream: {message}" +``` + +**Maps to**: Local consumer thread → calls workflow start API on message. + +### Prompt templating + +Triggers inject context into the agent's prompt via template variables: + +| Trigger Type | Available Variables | +|---|---| +| Schedule | `{run_time}`, `{run_count}`, `{last_run_time}` | +| EventTrigger | `${event.*}` (event payload fields) | +| WebhookTrigger | `${webhook.payload}`, `${webhook.headers}` | +| FileWatch | `{matched_lines}`, `{filepath}`, `{line_number}`, `{match}` | +| StreamWatch | `{message}`, `{topic}`, `{offset}`, `{timestamp}` | + +Server-side triggers use `${...}` (Conductor expression syntax, resolved server-side). Local triggers use `{...}` (resolved by the local watcher before workflow start). + +## 3. Phase 1 — Scheduling (SHIPPED) + +**Status: complete across all four SDKs (Python, TypeScript, Java, C#) and the UI (2026-06).** Users can put an agent on one or more cron schedules from code, with full lifecycle control (deploy, list, pause/resume, delete, ad-hoc run-now). Schedules survive process restarts; the orchestration server (Conductor) handles timing. + +### 3.1 Model + +``` +Agent ──deploy──► WorkflowDef + ▲ + │ startWorkflowRequest.name = agent.name + │ + ┌─────┴─────┬───────────┐ + Schedule Schedule Schedule ← N independent crons per agent + (name="A") (name="B") (name="C") +``` + +- One `Schedule` = one cron expression + one input + one name. +- An agent can have **N schedules**; pause/resume/delete each independently. +- **Ownership is implicit**: a schedule "belongs to" an agent iff `startWorkflowRequest.name == agent.name`. No tags or metadata needed — Conductor's `findAllSchedules(workflowName)` does the lookup. +- Server-side scheduler is **Conductor** (`/api/scheduler/*`). The SDK is a thin typed wrapper. + +### 3.2 Conductor surface this builds on + +Verified against [conductor-oss/conductor](https://github.com/conductor-oss/conductor): + +| SDK call | Conductor endpoint | Source | +|---|---|---| +| Save / upsert | `POST /api/scheduler/schedules` | `SchedulerResource.java:62` | +| List for agent | `GET /api/scheduler/schedules?workflowName={agent}` | `SchedulerResource.java:69` | +| Get one | `GET /api/scheduler/schedules/{name}` | `SchedulerResource.java:93` | +| Delete | `DELETE /api/scheduler/schedules/{name}` | `SchedulerResource.java:99` | +| Pause | `PUT /api/scheduler/schedules/{name}/pause?reason=...` | `SchedulerResource.java:110` | +| Resume | `PUT /api/scheduler/schedules/{name}/resume` | `SchedulerResource.java:119` | +| Preview next N fires | `GET /api/scheduler/nextFewSchedules?cronExpression=...&limit=N` | `SchedulerResource.java:130` | +| Run now (ad-hoc) | `POST /api/workflow/{agent.name}` (bypasses scheduler) | core workflow API | + +The `WorkflowSchedule` payload sent in `POST /schedules`: + +```json +{ + "name": "weekday-9am", + "cronExpression": "0 9 * * MON-FRI", + "zoneId": "America/Los_Angeles", + "paused": false, + "runCatchupScheduleInstances": false, + "scheduleStartTime": null, + "scheduleEndTime": null, + "description": "Daily digest", + "startWorkflowRequest": { + "name": "daily_digest", + "version": null, + "input": { "channel": "#eng" }, + "correlationId": null + } +} +``` + +### 3.3 Schedule object — fields + +| Field | Type | Required | Default | Notes | +|---|---|---|---|---| +| `name` | string | **yes** | — | Unique per agent. SDK auto-prefixes the wire name as `{agent.name}-{name}` to satisfy Conductor's org-wide uniqueness constraint while preserving the per-agent mental model. Raise at construction if omitted. | +| `cron` | string | **yes** | — | 5- or 6-field cron (seconds optional). Server validates. | +| `timezone` | string | no | `"UTC"` | IANA tz id, maps to `zoneId`. | +| `input` | object | no | `{}` | Workflow input. | +| `catchup` | bool | no | `false` | Maps to `runCatchupScheduleInstances`. Replay missed fires on resume. | +| `paused` | bool | no | `false` | Start in paused state. | +| `start_at` | datetime | no | `null` | Window start (ms since epoch). | +| `end_at` | datetime | no | `null` | Window end. | +| `description` | string | no | `null` | Human-readable note. | + +**Not exposed in v1**: `overlap` (Conductor fires every tick — agentspan-side skip/queue is future work), `cronSchedules` multi-cron list (covered by N schedules). + +### 3.4 Lifecycle semantics + +#### Deploy is declarative, scoped to this agent + +```text +deploy(agent, schedules=...) +``` + +| `schedules=` value | Behavior | +|---|---| +| omitted / `None` | Leave existing schedules untouched. | +| `[]` (empty list) | Delete **all** schedules whose `workflowName == agent.name`. | +| `[Schedule(...), ...]` | **Upsert** the listed schedules; delete any other schedule whose `workflowName == agent.name`. | + +Reconciliation algorithm: + +``` +existing = SchedulerClient.getAllSchedules(workflowName=agent.name) +desired = schedules +to_delete = {s.name for s in existing} - {s.name for s in desired} +to_upsert = desired +for s in to_delete: deleteSchedule(s) +for s in to_upsert: saveSchedule(s) +``` + +This works precisely because agent name = workflow name. No tagging scheme. + +#### Module-level lifecycle API + +All operations are keyed by schedule **name** — no handles to pass around, survives process restart. + +```text +schedules.list(agent=name) -> [ScheduleInfo] +schedules.get(name) -> ScheduleInfo +schedules.pause(name, reason=None) +schedules.resume(name) +schedules.delete(name) +schedules.run_now(name) # bypasses scheduler; returns execution id immediately +schedules.run_now(name, wait=True) # synchronous wait variant — block until completion (all four SDKs) +schedules.preview_next(cron, n=5) # for UI / drawer +``` + +`ScheduleInfo` returned by `get` / `list`: + +``` +ScheduleInfo { + name, cron, timezone, input, paused, paused_reason, + catchup, start_at, end_at, description, + next_run, last_run, # epoch ms (server-computed) + create_time, created_by, update_time, updated_by, + agent, # = workflow name +} +``` + +#### Overlap + +Fixed to `allow` in v1 (Conductor's native behavior). Every cron tick starts a new workflow execution even if the prior one is still running. Skip-if-running and queue policies are future agentspan-layer features. + +#### Errors + +- Duplicate `name` within the same agent → SDK raises `ScheduleNameConflict` before the wire call. Across agents, names are isolated by the `{agent.name}-` prefix, so no collision possible. +- Bad cron → 400. SDK surfaces `InvalidCronExpression` with the server's parse error. +- Schedule not found on `pause`/`resume`/`delete`/`get` → 404 → `ScheduleNotFound`. + +### 3.5 Language SDK surfaces + +Same semantics, idiomatic shape per language. All four wrap the same Conductor REST surface. + +#### Python + +```python +from conductor.ai.agents import Agent, deploy, schedules +from conductor.ai.agents.schedule import Schedule + +agent = Agent(name="daily_digest", ...) + +deploy( + agent, + schedules=[ + Schedule( + name="weekday-9am", + cron="0 9 * * MON-FRI", + timezone="America/Los_Angeles", + input={"channel": "#eng"}, + ), + Schedule(name="friday-5pm", cron="0 17 * * FRI", input={"channel": "#all-hands"}), + ], +) + +schedules.list(agent="daily_digest") +schedules.pause("weekday-9am", reason="rate limit cooldown") +schedules.resume("weekday-9am") +schedules.run_now("weekday-9am") +schedules.delete("weekday-9am") +schedules.preview_next("0 9 * * MON-FRI", n=5) +``` + +`Schedule` is a `@dataclass(frozen=True)` (matches repo convention — no Pydantic). All names snake_case. Async siblings: `schedules.list_async`, `pause_async`, etc., plus `deploy_async(..., schedules=...)`. + +#### TypeScript + +```ts +import { Agent, deploy, schedules, Schedule } from "@conductor-oss/conductor-agent-sdk"; + +const agent = new Agent({ name: "dailyDigest", /* ... */ }); + +await deploy(agent, { + schedules: [ + new Schedule({ + name: "weekday-9am", + cron: "0 9 * * MON-FRI", + timezone: "America/Los_Angeles", + input: { channel: "#eng" }, + }), + new Schedule({ name: "friday-5pm", cron: "0 17 * * FRI", input: { channel: "#all-hands" } }), + ], +}); + +await schedules.list({ agent: "dailyDigest" }); +await schedules.pause("weekday-9am", { reason: "rate limit cooldown" }); +await schedules.resume("weekday-9am"); +await schedules.runNow("weekday-9am"); +await schedules.delete("weekday-9am"); +await schedules.previewNext("0 9 * * MON-FRI", { n: 5 }); +``` + +Constructor takes a single options object (camelCase). Field renames: `timezone` (not `tz`), `catchup`, `startAt`, `endAt`. All operations return Promises. Type exported as `ScheduleOptions` for the constructor and `ScheduleInfo` for the runtime view. + +#### Java + +```java +import org.conductoross.conductor.ai.Agent; +import org.conductoross.conductor.ai.AgentRuntime; +import org.conductoross.conductor.ai.schedule.Schedule; +import org.conductoross.conductor.ai.schedule.Schedules; + +Agent agent = Agent.builder().name("daily_digest")./*...*/.build(); + +AgentRuntime runtime = new AgentRuntime(); +runtime.deploy( + agent, + List.of( + Schedule.builder() + .name("weekday-9am") + .cron("0 9 * * MON-FRI") + .timezone("America/Los_Angeles") + .input(Map.of("channel", "#eng")) + .build(), + Schedule.builder() + .name("friday-5pm") + .cron("0 17 * * FRI") + .input(Map.of("channel", "#all-hands")) + .build())); + +Schedules schedules = runtime.schedules(); +schedules.list("daily_digest"); +schedules.pause("weekday-9am", "rate limit cooldown"); +schedules.resume("weekday-9am"); +schedules.runNow("weekday-9am"); // name-keyed; fire-and-return execution id +schedules.runNowAndWait("weekday-9am"); // synchronous wait variant (returns AgentResult) +schedules.delete("weekday-9am"); +schedules.previewNext("0 9 * * MON-FRI", 5); +``` + +`Schedule` uses Lombok `@Builder` (mirrors `WorkflowSchedule.java` from Conductor). `Schedules` is reached via `runtime.schedules()` rather than a top-level static — fits the existing `AgentRuntime`-centric Java idiom. Overloaded `deploy(Agent agent, List schedules)` extends the current `deploy(Agent...)`. + +#### C# + +```csharp +using Conductor.AI; +using Conductor.AI.Scheduling; + +var agent = new Agent { Name = "daily_digest", /* ... */ }; + +await using var runtime = new AgentRuntime(); +await runtime.DeployAsync( + agent, + schedules: new[] + { + new Schedule + { + Name = "weekday-9am", + Cron = "0 9 * * MON-FRI", + Timezone = "America/Los_Angeles", + Input = new { channel = "#eng" }, + }, + new Schedule + { + Name = "friday-5pm", + Cron = "0 17 * * FRI", + Input = new { channel = "#all-hands" }, + }, + }); + +var schedules = runtime.Schedules; +await schedules.ListAsync(agent: "daily_digest"); +await schedules.PauseAsync("weekday-9am", reason: "rate limit cooldown"); +await schedules.ResumeAsync("weekday-9am"); +await schedules.RunNowAsync("weekday-9am"); // name-keyed; fire-and-return execution id +await schedules.RunNowAsync("weekday-9am", wait: true); // synchronous wait variant (returns AgentResult) +await schedules.DeleteAsync("weekday-9am"); +await schedules.PreviewNextAsync("0 9 * * MON-FRI", n: 5); +``` + +`Schedule` is a property-init record-style class. All operations async-first (sync wrappers mirror existing `AgentRuntime` style). `Schedules` accessor on `AgentRuntime` parallels Java. + +### 3.6 UI + +Two surfaces. Both back onto the same REST endpoints. + +#### Agent detail → Schedules tab + +``` +┌─ Agent: daily_digest ─────────────────────────────────────────────────┐ +│ [ Overview ] [ Executions ] [ Schedules ] [ Versions ] [ Code ] │ +│ ───────────────────────────────────────────────────────────────────── │ +│ [+ New] │ +│ ● weekday-9am 0 9 * * MON-FRI PT next: Tue 9:00 AM │ +│ last: ✓ 2026-05-26 9:00 (12.4s) [Pause] [Run now] [⋯] │ +│ │ +│ ◐ friday-5pm 0 17 * * FRI UTC PAUSED (rate limit cooldown) │ +│ last: ✓ 2026-05-22 17:00 [Resume] [Run now] [⋯] │ +└───────────────────────────────────────────────────────────────────────┘ +``` + +Status glyph: ● active · ◐ paused · ⊘ expired. Row click → detail drawer. + +#### New / edit drawer + +``` +┌─ New schedule ─────────────────────────────────────┐ +│ Name * [ weekday-9am ] │ +│ Cron * [ 0 9 * * MON-FRI ] │ +│ ⓘ "At 9:00 AM, Mon–Fri" │ +│ Next: Tue 9:00 · Wed 9:00 · ... │ +│ Timezone [ America/Los_Angeles ▾ ] │ +│ Input (JSON) ┌──────────────────────────┐ │ +│ │ { "channel": "#eng" } │ │ +│ └──────────────────────────┘ │ +│ Window Start [ — ] End [ — ] (opt) │ +│ [ ] Catch up missed runs on resume │ +│ [ ] Start paused │ +│ │ +│ [ Cancel ] [ Save ] │ +└────────────────────────────────────────────────────┘ +``` + +Cron preview uses `GET /api/scheduler/nextFewSchedules` and the existing `cronExpressionHelpers.ts`. + +#### Schedule detail drawer + +- Header: name · cron · tz · status · `[Pause/Resume]` `[Run now]` `[Edit]` `[Delete]` +- Tabs: + - **Executions** — table of past runs (started, duration, status, workflow id → click through) + - **Definition** — read-only JSON + - **History** — audit trail (created / paused with reason / edited) + +#### Global Schedules list + +`Agent` column + filter on `ui/src/pages/scheduler/`. Same row controls. The cross-agent view; the agent-detail tab is a filtered slice. + +### 3.7 Validation evidence + +- Conductor REST surface — `scheduler/corexx/src/main/java/io/orkes/conductor/scheduler/rest/SchedulerResource.java` (verified all endpoints exist). +- `findAllSchedules(orgId, workflowName)` — `scheduler/core/.../dao/scheduler/SchedulerDAO.java:36`. +- `WorkflowSchedule` model fields — `scheduler/corexx/.../model/WorkflowSchedule.java`. +- conductor-python already has `SchedulerClient` (`save_schedule`, `get_all_schedules(workflow_name=...)`, `delete_schedule`, `pause_schedule`, `resume_schedule`) — agentspan SDKs wrap it. +- agent.name → workflow name — `server/.../AgentService.java:222` (`def.getName()` returned as `agentName`). + +### 3.8 Resolved design questions (Phase 1) + +1. **Module path** → `conductor.ai.agents.schedule.Schedule`. Ships only what exists today; if/when Webhook/Event triggers land, they get their own modules and we revisit a `triggers/` umbrella. +2. **`run_now` blocking** → the default returns the execution id immediately. Agents can run for minutes; blocking is the wrong default for a UI button or scripted invocation. **`run_now` is now name-keyed in all four SDKs, and all four expose an opt-in synchronous wait variant**: Python `run_now(name, wait=True)`, TS `runNow(name, {wait})` / `runNowAndWait(name)`, Java `runNow(name)` / `runNowAndWait(name)`, C# `RunNowAsync(name)` / `RunNowAsync(name, wait: true)`. The wait variant returns an `AgentResult` **uniformly across all four SDKs** — no return-type divergence. +3. **`nextRunTime` when paused-on-create** → verified against Conductor source (`scheduler/core/.../SchedulerService.java:732`): `setNextRunTimeInEpoch(...)` is called unconditionally on save; the `isPaused()` check only gates the queue-message push that triggers the fire. The UI's "Next: ..." column is reliable for paused schedules. No SDK or UI accommodation needed. +4. **Schedule name scoping** → unique **per agent**, not globally. The SDK auto-prefixes the wire name to `{agent.name}-{name}` at `deploy()` time so users write `Schedule(name="daily")` ergonomically while Conductor's org-wide uniqueness is satisfied. The prefixed name is the canonical identifier returned by `list()`/`get()` and accepted by `pause`/`resume`/`delete`/`run_now`. The `ScheduleInfo` dataclass exposes both `name` (prefixed, wire) and `short_name` (the user's original) for display. + +## 4. Roadmap — Event / Webhook / File / Stream Triggers + +The remaining four trigger types extend the same `deploy(agent, triggers=[...])` umbrella. Server-side triggers (Event, Webhook) are fully managed by the orchestration server; local source watchers (FileWatch, StreamWatch) require a local daemon. + +### 4.1 Deployment architecture + +#### Server-side triggers (Event, Webhook) + +Fully managed by the orchestration server. No local process needed beyond initial registration. + +``` +Developer Orchestration Server + │ │ + │ deploy(agent, triggers) │ + │─────────────────────────────►│ + │ │ 1. Register workflow definition + │ │ 2. Create schedule (cron) + │ │ 3. Register event handlers + │ │ 4. Start tool workers + │ DeploymentHandle │ + │◄─────────────────────────────│ + │ │ + │ │ [cron fires / event arrives] + │ │ → Start agent execution + │ │ → LLM + tools execute + │ │ → Result stored + │ │ + │ handle.executions() │ + │─────────────────────────────►│ + │ [list of past runs] │ + │◄─────────────────────────────│ +``` + +#### Local source watchers (FileWatch, StreamWatch) + +These require a local daemon process that watches the source and triggers the agent. + +``` +┌─────────────────────────────────┐ ┌───────────────────────┐ +│ Local Watcher Process │ │ Orchestration Server │ +│ │ │ │ +│ ┌────────────────────────────┐ │ │ │ +│ │ FileWatch Thread │ │ │ │ +│ │ tail -F /var/log/app.log │──┼─match─┼──► start_workflow() │ +│ │ pattern: "ERROR" │ │ │ prompt = "..." │ +│ │ debounce: 30s │ │ │ → Agent runs │ +│ └────────────────────────────┘ │ │ │ +│ │ │ │ +│ ┌────────────────────────────┐ │ │ │ +│ │ StreamWatch Thread │ │ │ │ +│ │ Kafka consumer: alerts │──┼──msg──┼──► start_workflow() │ +│ │ filter: level == 'error' │ │ │ prompt = "..." │ +│ └────────────────────────────┘ │ │ │ +│ │ │ │ +│ ┌────────────────────────────┐ │ │ │ +│ │ Tool Workers │ │ │ │ +│ │ Serving @tool functions │◄─┼───────┼── poll for tasks │ +│ └────────────────────────────┘ │ │ │ +│ │ │ │ +│ runtime.wait() ← blocks here │ │ │ +└─────────────────────────────────┘ └───────────────────────┘ +``` + +The local process runs: +1. **Watcher threads** — one per FileWatch/StreamWatch trigger +2. **Tool workers** — serving the agent's @tool functions to the orchestration server +3. **Main thread** — `runtime.wait()` blocks, keeping everything alive + +#### How you install and run this + +**Scenario: "Tail a log file and trigger an agent on errors"** + +```bash +# 1. Install +pip install conductor-agent-sdk # or: npm install @conductor-oss/conductor-agent-sdk + +# 2. Write the sentinel (sentinel.py / sentinel.ts) +# Define agent + tools + FileWatch trigger + +# 3. Run +python sentinel.py # foreground, for dev/testing +# or +conductor-agent deploy sentinel.py # daemonize (future CLI) +# or +docker run -v /var/log:/var/log:ro my-sentinel # containerized +# or +systemctl start conductor-sentinel@log_monitor # systemd service +``` + +**What `runtime.wait()` does:** blocks the main thread, keeps file watchers alive, keeps tool workers polling, handles graceful shutdown on SIGTERM/SIGINT, and logs trigger events and agent executions. + +#### Multi-instance / HA deployment + +| Trigger Type | Multi-Instance Behavior | +|---|---| +| **Schedule** | Orchestration server ensures exactly-once execution. Safe to run multiple instances. | +| **Event** | Event handler registered once. Server routes to one execution instance. | +| **Webhook** | Server-side. Single handler. | +| **FileWatch** | Local — each instance watches independently. Needs **distributed lock** or **leader election** to prevent duplicate triggers. | +| **StreamWatch** | Use consumer groups (Kafka) or competing consumers (SQS) for natural dedup. | + +### 4.2 Lifecycle management (multi-trigger) + +When an agent is deployed with non-schedule triggers, a handle is returned for ongoing management: + +``` +DeploymentHandle: + name: string # agent name + registered_name: string # compiled workflow name + triggers: Trigger[] # active triggers + status: "running" | "paused" | "stopped" + + pause() # pause all triggers + resume() # resume triggers + undeploy() # stop + remove all triggers + cleanup + + executions(limit=10) # list recent agent runs + last_execution() # most recent run result +``` + +Observability: deployed sentinels expose execution history (when it ran, what triggered it, outcome), trigger status (active? last fire? error count?), metrics (runs/hour, avg duration, tokens, tool calls), and structured logs. Access via API (`handle.executions()`, `handle.status`), CLI (`conductor-agent status`, `conductor-agent logs `), and the Web UI (Conductor's workflow execution UI). + +### 4.3 Concrete examples + +#### Log Sentinel (FileWatch + Schedule) + +``` +Agent: + name: log_sentinel + model: openai/gpt-4o + tools: [read_log_context, create_jira_ticket, send_slack_alert] + instructions: | + You are a production log sentinel. When triggered with log errors: + 1. Read surrounding context to understand the error + 2. Assess severity (transient vs. real bug vs. critical outage) + 3. Transient: ignore. Bug: create JIRA ticket. Critical: Slack alert + ticket. + +Triggers: + - FileWatch: + path: /var/log/myapp/error.log + pattern: (ERROR|CRITICAL|FATAL) + debounce: 30 + prompt: | + Log error detected in {filepath} at line {line_number}: + + {matched_lines} + + Analyze this error, check context, and take appropriate action. + + - Schedule: + cron: "0 9 * * 1" # Monday 9am + prompt: Compile a weekly summary of all errors from the past week. +``` + +#### PR Review Sentinel (WebhookTrigger + Schedule) + +``` +Agent: + name: pr_reviewer + model: anthropic/claude-sonnet + tools: [list_open_prs, fetch_pr_diff, post_review_comment] + instructions: | + Review pull requests for code quality, bugs, and security issues. + Post constructive review comments. Approve clean PRs. + +Triggers: + - WebhookTrigger: + matches: {action: "opened", pull_request: {base: {ref: "main"}}} + prompt: "New PR to main: ${webhook.payload.pull_request.title}. Review it." + + - Schedule: + cron: "*/30 * * * *" + prompt: "Check for any unreviewed PRs in the last 30 minutes." +``` + +#### Incident Responder (EventTrigger) + +``` +Agent: + name: incident_responder + model: openai/gpt-4o + tools: [get_metrics, get_logs, restart_service, scale_replicas, notify_oncall] + instructions: | + You are the first responder for production incidents. + 1. Gather metrics and logs to understand the issue + 2. If it's a known pattern (OOM, connection pool), apply the fix + 3. If unknown, gather diagnostics and escalate to on-call + +Triggers: + - EventTrigger: + event: pagerduty:incident + condition: "event.severity == 'critical'" + prompt: | + CRITICAL INCIDENT: ${event.title} + Service: ${event.service} + Description: ${event.description} + Triggered at: ${event.created_at} + + - EventTrigger: + event: prometheus:alert + condition: "event.labels.severity == 'warning'" + prompt: | + Warning alert: ${event.labels.alertname} + ${event.annotations.description} +``` + +#### Omnipresent Ops Agent (all trigger types) + +The most ambitious pattern — a single agent combining every trigger type: + +``` +Agent: + name: ops_sentinel + model: openai/gpt-4o + tools: [ + check_health, get_metrics, query_logs, + restart_pod, scale_service, + create_ticket, send_alert, + run_db_query, check_cert_expiry + ] + instructions: | + You are the AI ops team member. You have multiple responsibilities: + - Health checks (scheduled) + - Error triage (log watching) + - Alert response (event-driven) + - Deployment verification (webhook-driven) + Fix what you can autonomously. Escalate what you can't. + +Triggers: + - Schedule: + cron: "*/5 * * * *" + prompt: "Run health checks on all production services." + + - Schedule: + cron: "0 8 * * *" + prompt: "Morning report: summarize overnight incidents, current system health, upcoming cert expirations." + + - FileWatch: + path: /var/log/k8s/*.log + pattern: "OOMKilled|CrashLoopBackOff|ImagePullBackOff" + prompt: "K8s issue detected:\n\n{matched_lines}\n\nDiagnose and fix if possible." + + - EventTrigger: + event: prometheus:alert + prompt: "Alert: ${event.labels.alertname}\n${event.annotations.description}" + + - WebhookTrigger: + matches: {source: "github", action: "deployment"} + prompt: "Deployment to ${webhook.payload.environment}: verify health." +``` + +### 4.4 Roadmap phases + +**Phase 2 — Event & Webhook Triggers.** `EventTrigger` class → registers event handler with the orchestration server. `WebhookTrigger` class → registers webhook handler. Prompt template interpolation with event/webhook payload. *Example*: incident responder triggered by PagerDuty events. + +**Phase 3 — Local File Watching.** `FileWatch` trigger class. Local file tailing engine (efficient, handles rotation and glob patterns). Pattern matching + debounce. On match → `runtime.start(agent, prompt=rendered_template)`. *Example*: log sentinel watching error.log. + +**Phase 4 — Stream Watching.** `StreamWatch` trigger class. Connector interface for Kafka, Redis Streams, SQS, etc. Consumer-group support for multi-instance dedup. *Example*: data-pipeline monitor on a Kafka topic. + +**Phase 5 — CLI & Observability.** `conductor-agent deploy ` (daemonize), `status` (list deployed sentinels), `logs ` (stream execution logs), `pause/resume/undeploy `. Dashboard integration with execution history. + +**Phase 6 — Advanced Patterns.** Cost controls (max runs/hour, spend threshold, auto-pause); concurrency policy (skip-if-running / queue / parallel); state across runs (automatic memory injection for scheduled agents — see [`stateful-agents.md`](stateful-agents.md)); multi-instance coordination (distributed locking for FileWatch in HA); chained sentinels (one sentinel's output triggers another). + +### 4.5 Open design questions + +1. **Prompt templating syntax**: Server-side triggers naturally use orchestration engine expressions (`${event.field}`). Local triggers resolve before the workflow starts (`{matched_lines}`). Unify, or keep the natural split? +2. **Concurrency on schedule overlap**: If a cron fires while the previous run is still executing — skip, queue, or parallel? Default recommendation: **skip-if-running** to prevent agent pile-up and runaway costs. (Schedule overlap is `allow` in shipped Phase 1; this is the Phase 6 agentspan-layer feature.) +3. **State persistence across scheduled runs**: Should sentinel agents automatically get conversation memory? Recommendation: **opt-in** via the memory parameter — memory adds cost and complexity. +4. **FileWatch reliability**: The local watcher process is a single point of failure. Options: (a) watchdog/health checks, (b) systemd auto-restart, (c) heartbeat to the orchestration server that alerts on missed heartbeats. +5. **Cost guardrails**: Sentinel agents can run thousands of times per day. Should triggers support max executions per hour, a monthly spend cap, auto-pause on threshold? +6. **Trigger composition**: Can triggers have dependencies? E.g., "only trigger on FileWatch if the last Schedule run found issues." Adds complexity — defer to Phase 6. +7. **Multi-SDK consistency**: Trigger classes and deployment API should be identical in structure across Python, TypeScript, Java, and C# SDKs, differing only in language idiom — as already achieved for Phase 1 scheduling. diff --git a/docs/design/stateful-agents.md b/design/stateful-agents.md similarity index 73% rename from docs/design/stateful-agents.md rename to design/stateful-agents.md index 7a134b3ac..e3ebdf315 100644 --- a/docs/design/stateful-agents.md +++ b/design/stateful-agents.md @@ -1,5 +1,13 @@ # Stateful Agents — Task-to-Domain Routing +**Status:** Consolidated 2026-06-26 + +**Scope:** How Agentspan isolates tool-worker routing per concurrent agent +execution so multiple running instances of the same agent script don't cross-route +tasks. For the general execution/worker-dispatch model see +[agentspan-design.md](agentspan-design.md); for always-on/scheduled execution see +[sentinel-agents.md](sentinel-agents.md). + ## Problem When multiple concurrent instances of the same agent script run simultaneously @@ -21,6 +29,16 @@ Mark tools — or a whole agent — as **stateful**. At execution time: Non-stateful tools (HTTP, MCP, or `@tool` without `stateful=True` on a non-stateful agent) are unaffected — they continue to use the default domain. +> **Cross-SDK note.** The `stateful` flag's domain-routing semantics described +> here ARE correctly implemented and consistent across all four SDKs (Python, +> TypeScript, Java, C#): each execution gets a unique `run_id` that isolates the +> stateful tool workers to their own Conductor domain. **However**, the Java +> (`Agent.java`) and C# (`Agent.cs`) doc-COMMENTS on the `stateful` flag describe +> a different/overloaded meaning — conversation-history persistence / +> `WaitForMessageTool` — which is NOT what the flag does. This is a known +> code-comment inconsistency only; the behavior actually wired by all four SDKs is +> the per-execution `run_id` → worker-domain isolation documented above. + ## API Two equivalent ways to opt in: diff --git a/design/tool-execution-and-credentials-design.md b/design/tool-execution-and-credentials-design.md new file mode 100644 index 000000000..1b164e786 --- /dev/null +++ b/design/tool-execution-and-credentials-design.md @@ -0,0 +1,1064 @@ +# Tool Execution and Credentials Design + +**Status:** Consolidated 2026-06-26 + +**Scope:** This is the canonical design for two coupled subsystems in Agentspan. The first half covers **tool and code execution** — how an agent's LLM runs code via the `execute_code` tool, the executor types (local, Docker, Jupyter, serverless), the interpreter table, command validation, timeouts, and how tools register as Conductor workers. The second half covers **credentials and secrets** — the encrypted-at-rest store, execution-token auth for distributed workers, per-user LLM keys, output masking on the read path, the SDK secret-injection contract every SDK must honor, and the **Secrets** management UI. The two halves meet at secret injection into tools: a tool declares the secrets it needs, and the credentials pipeline resolves and injects them at execution time. Siblings: [agentspan-design.md](agentspan-design.md) (overall architecture), [api-design.md](api-design.md) (REST surface), [sdk-design.md](sdk-design.md) (cross-SDK contracts), [framework-integration.md](framework-integration.md) (framework passthrough). Framework-passthrough credential injection detail is shared with [framework-integration.md](framework-integration.md). + +--- + +# Part 1 — Tool & Code Execution + +Local code execution lets an agent's LLM run code on the user's machine (or in a sandbox) via an `execute_code` tool. The LLM sends code + language; a **worker** on the SDK side executes it and returns stdout/stderr/exit code. The full four-executor family (Local + Docker + Jupyter + Serverless) now ships in **all four SDKs**; the only cross-SDK difference is the Jupyter *mechanism*, which is language-idiomatic — see §2.10 for the as-built matrix. There is no Go SDK (Python, TypeScript, Java, and C#/.NET only). + +``` +LLM ──tool_call──► Conductor ──task──► SDK Worker ──subprocess──► Result + (execute_code) (SIMPLE) (temp file) +``` + +## 2.1 ExecutionResult + +A data object returned by every executor. + +| Field | Type | Description | +|-------------|---------|---------------------------------------------| +| `output` | string | Captured stdout | +| `error` | string | Captured stderr | +| `exit_code` | int | Process exit code (0 = success) | +| `timed_out` | bool | Whether execution hit the timeout | + +**Derived property:** `success` = `exit_code == 0 && !timed_out` + +## 2.2 CodeExecutor (interface / abstract base) + +Every SDK must implement this interface: + +``` +interface CodeExecutor { + execute(code: string) -> ExecutionResult +} +``` + +**Constructor parameters:** + +| Param | Type | Default | Description | +|---------------|--------|------------|---------------------------------------| +| `language` | string | `"python"` | Target interpreter language | +| `timeout` | int | `30` | Max execution time in seconds | +| `working_dir` | string | `null` | Working directory for the subprocess | + +## 2.3 Executor implementations + +### 2.3a LocalCodeExecutor + +Runs code in a local subprocess via a temp file. + +**Algorithm:** + +1. If `code` is empty/null, return `ExecutionResult(output="No code provided. Nothing to execute.", exit_code=0)`. +2. Map `language` to interpreter command using the interpreter table (below). +3. Write `code` to a temp file with the appropriate extension. +4. Run `subprocess(interpreter, temp_file)` with: + - `timeout` applied + - `working_dir` as cwd (if set) + - stdout and stderr captured separately +5. Return `ExecutionResult(stdout, stderr, exit_code)`. +6. On timeout: return `ExecutionResult(error="...", exit_code=-1, timed_out=true)`. +7. **Always** delete the temp file in a finally block. + +**Interpreter table:** + +| Language | Command(s) | File Extension | +|----------------|-----------------|----------------| +| `python` | `python3` | `.py` | +| `python3` | `python3` | `.py` | +| `bash` | `bash` | `.sh` | +| `sh` | `sh` | `.sh` | +| `node` | `node` | `.js` | +| `javascript` | `node` | `.js` | +| `ruby` | `ruby` | `.rb` | + +> **Portability note:** On Windows, `python3` may not exist; fall back to `python`. For Node.js, use `node` on all platforms. + +### 2.3b DockerCodeExecutor + +Runs code inside a Docker container for isolation. + +**Algorithm:** + +1. Build Docker command: + ``` + docker run --rm -i [--network=none] [--memory LIMIT] + [-v host:container:ro ...] IMAGE INTERPRETER -c CODE + ``` +2. Pass code via stdin (not a temp file — avoids volume mounts for code). +3. Capture stdout/stderr from the container. +4. Add extra timeout buffer (e.g. +10s) for container startup. +5. Default: `--network=none` (disable network). + +**Constructor extras:** + +| Param | Type | Default | +|-------------------|-------------------|---------------------| +| `image` | string | `"python:3.12-slim"`| +| `network_enabled` | bool | `false` | +| `memory_limit` | string | `null` | +| `volumes` | map| `{}` | + +### 2.3c JupyterCodeExecutor + +Uses a Jupyter kernel for stateful execution (state persists between calls). + +> **Note:** This is the exception to the "isolated per call" rule. A `JupyterCodeExecutor` now ships in **all four SDKs**, but the underlying mechanism is language-idiomatic: Python runs in-process via `jupyter_client`; TypeScript shells out to the `jupyter run` CLI; Java and C# talk to a Jupyter **Kernel Gateway** over HTTP. The executor is present everywhere — only the runtime it requires differs by language. + +### 2.3d ServerlessCodeExecutor + +Delegates execution to an HTTP endpoint. + +**Request:** +```json +POST /execute +{ + "code": "...", + "language": "python", + "timeout": 30 +} +``` + +**Response:** +```json +{ + "output": "...", // or "stdout" + "error": "...", // or "stderr" + "exit_code": 0 +} +``` + +This is the most portable executor — any SDK can implement an HTTP client. + +## 2.4 CodeExecutionConfig + +Declarative configuration attached to an Agent. + +| Field | Type | Default | Description | +|---------------------|----------------|--------------|------------------------------------| +| `enabled` | bool | `true` | Whether code execution is active | +| `allowed_languages` | list\ | `["python"]` | Languages the LLM may use | +| `allowed_commands` | list\ | `[]` | Allowed shell commands (empty = no restriction) | +| `executor` | CodeExecutor | `null` | Executor instance (null = auto-create LocalCodeExecutor) | +| `timeout` | int | `30` | Seconds | +| `working_dir` | string | `null` | Working directory | + +## 2.5 CommandValidator + +Best-effort regex-based validator that checks code for shell command invocations against an allowed-command whitelist. + +**Important:** This is NOT a security boundary. For untrusted code, use DockerCodeExecutor or ServerlessCodeExecutor. + +**Validation rules per language:** + +- **Python:** Scan for `subprocess.run/call(["CMD"...])`, `os.system("CMD")`, `os.popen("CMD")`, Jupyter `!CMD` syntax. +- **Bash/sh:** Extract command names from the script (skip builtins like `if`, `echo`, `export`, etc.), check each against the whitelist. +- **Other languages:** Skip validation (no patterns defined). + +## 2.6 The `execute_code` tool + +A tool function registered as a Conductor SIMPLE worker. + +**Tool schema:** + +```json +{ + "name": "execute_code", + "description": "Execute code in a sandboxed environment. Supported languages: {langs}. Timeout: {timeout}s.", + "parameters": { + "code": { "type": "string", "description": "The code to execute" }, + "language": { "type": "string", "default": "python", "description": "Programming language" } + } +} +``` + +**Output format:** + +The tool always returns structured JSON (never raises on code errors): + +```json +{"status": "success", "stdout": "hello world\n", "stderr": ""} +{"status": "error", "stdout": "", "stderr": "NameError: name 'x' is not defined\nExit code: 1"} +``` + +When the tool returns a `dict`, the worker sets it directly as `task_result.output_data` — the server passes `outputData` straight through to the LLM as the tool result. + +**Execution flow:** + +``` +1. Receive task with { code, language } +2. If code is empty/null → COMPLETE with {"status":"success","stdout":"No code provided...","stderr":""} +3. If language not in allowed_languages → raise ValueError (FAILED — tool misconfiguration) +4. If allowed_commands is set → CommandValidator.validate(code, language) + If violation → raise ValueError (FAILED — tool misconfiguration) +5. Create executor for the language (LocalCodeExecutor per invocation, + since each language needs its own interpreter) +6. result = executor.execute(code) +7. If result.success → COMPLETE with {"status":"success","stdout":"...","stderr":"..."} +8. If !result.success → COMPLETE with {"status":"error","stdout":"...","stderr":"..."} +``` + +**Key behavior:** Code execution errors always complete the task so the LLM receives the error as a normal tool result and can self-correct without wasting Conductor retries. Only tool misconfiguration errors (invalid language, disallowed commands) fail the task. + +## 2.7 Agent integration + +### Shorthand API + +Every SDK should support a simple boolean flag: + +```python +# Python +Agent(name="coder", model="...", local_code_execution=True) + +// JavaScript +new Agent({ name: "coder", model: "...", localCodeExecution: true }) + +// Java +Agent.builder().name("coder").model("...").localCodeExecution(true).build() +``` + +This auto-creates a `CodeExecutionConfig` with defaults and attaches the `execute_code` tool to the agent. + +### Extended API + +For fine-grained control: + +```python +# Python +Agent( + name="coder", + model="...", + code_execution=CodeExecutionConfig( + allowed_languages=["python", "bash"], + allowed_commands=["pip", "ls"], + executor=DockerCodeExecutor(image="python:3.12-slim"), + timeout=60, + ), +) +``` + +### Serialization + +When the agent config is sent to the server for compilation, the code execution config is serialized as: + +```json +{ + "codeExecution": { + "enabled": true, + "allowedLanguages": ["python", "bash"], + "allowedCommands": ["pip", "ls"], + "timeout": 60 + } +} +``` + +The `executor` field is NOT serialized — it lives only on the SDK side. The server uses this config to inject instructions into the LLM system prompt (see below). + +## 2.8 Server-side (Java) + +The server does not execute code. It: + +1. Reads `codeExecution` from the agent config. +2. Injects instructions into the LLM system prompt via `AgentCompiler.buildCodeExecInstructions()`: + ``` + You have code execution capabilities. Use the execute_code tool to write + and run code. Supported languages: python, bash. + Each execution runs in an isolated environment — no state, variables, or + imports persist between calls. + Always include all necessary imports at the top of every code block + (e.g. import subprocess, import os, import json). + Allowed shell commands: pip, ls. Do not use other commands. + ``` +3. The `execute_code` tool appears in the LLM's tool spec as a SIMPLE Conductor task. The SDK-side worker picks it up and executes it. + +## 2.9 Worker registration + +Each SDK must: + +1. Detect agents that have code execution enabled. +2. Register the `execute_code` function as a Conductor worker (SIMPLE task). +3. Start polling for tasks. + +The worker must handle: +- Empty/null code (return success with message) +- Language validation +- Command validation (if configured) +- Execution via the configured executor +- Timeout handling +- Error formatting for LLM consumption + +### Implementation checklist for new SDKs + +- [ ] `ExecutionResult` data class with `output`, `error`, `exit_code`, `timed_out`, `success` +- [ ] `CodeExecutor` interface with `execute(code) -> ExecutionResult` +- [ ] `LocalCodeExecutor` — subprocess + temp file, interpreter table, cleanup +- [ ] `DockerCodeExecutor` — Docker container execution (optional) +- [ ] `ServerlessCodeExecutor` — HTTP endpoint delegation (optional) +- [ ] `CodeExecutionConfig` data class +- [ ] `CommandValidator` with Python and Bash patterns +- [ ] `execute_code` tool function with the execution flow above +- [ ] Agent shorthand: `localCodeExecution: true` flag +- [ ] Config serialization to JSON for server compilation +- [ ] Conductor worker registration and polling +- [ ] Tests: empty code, language validation, command validation, execution success/failure/timeout + +## 2.10 As-built executor coverage per SDK + +The four-executor family above now ships in **all four SDKs**. The only cross-SDK difference is the Jupyter *mechanism* (the executor is present everywhere; the runtime it drives is language-idiomatic): + +| SDK | Local | Docker | Jupyter | Serverless | CommandValidator | Jupyter mechanism | +|---|:-:|:-:|:-:|:-:|:-:|---| +| **Python** | ✓ | ✓ | ✓ | ✓ | ✓ | In-process `jupyter_client` kernel. | +| **TypeScript** | ✓ | ✓ | ✓ | ✓ | ✓ | Shells out to the `jupyter run` CLI. | +| **Java** | ✓ | ✓ | ✓ | ✓ | ✓ | Jupyter Kernel Gateway over HTTP. | +| **C# / .NET** | ✓ | ✓ | ✓ | ✓ | ✓ | Jupyter Kernel Gateway over HTTP. C# `WorkerManager` now delegates local execution to `LocalCodeExecutor` (the earlier inline `ExecuteLocalCodeAsync` path). | + +Source locations: + +- Python: `sdk/python/src/conductor/ai/agents/code_executor.py` (+ `code_execution_config.py`). +- TypeScript: `sdk/typescript/src/code-execution.ts`. +- Java: `sdk/java/.../execution/{CodeExecutor,LocalCodeExecutor,DockerCodeExecutor,JupyterCodeExecutor,ServerlessCodeExecutor,CliCommandExecutor}.java`. +- C#: `sdk/csharp/src/Conductor.AI/{LocalCodeExecutor,DockerCodeExecutor,JupyterCodeExecutor,ServerlessCodeExecutor}.cs`; `WorkerManager` delegates local execution to `LocalCodeExecutor`. + +--- + +# Part 2 — Credentials & Secrets + +This half reflects what the codebase does today, not historical proposals. + +## 3. Goals + +- **Frictionless local dev** — env vars work without any setup. +- **Multi-user safe** — two users on the same server use distinct keys. +- **Distributed-worker safe** — workers resolve per-execution credentials via a short-lived token, never see the user's session. +- **One pipeline** — same resolution code path for LLM keys, tool credentials, HTTP/MCP headers, CLI tools, and framework passthroughs. +- **Pluggable** — `CredentialStoreProvider` interface lets Enterprise swap in AWS SM / HashiCorp Vault / Azure KV without touching OSS code. + +## 3.1 Backend architecture + +### Module layout (server) + +The server namespace is `dev.agentspan.*`. The credential implementation classes live under `server/conductor-agentspan/.../runtime/credentials/` (interfaces/SPI in `runtime/spi/`, controllers in `runtime/controller/`); the OSS-only built-in store implementations live in the `conductor-agentspan-server` module. + +| Class | Responsibility | +|---|---| +| `CredentialStoreProvider` (iface, `runtime/spi`) | `get/set/delete/list` over an opaque backend | +| `EncryptedDbCredentialStoreProvider` (server module) | OSS default — AES-256-GCM in SQLite/Postgres | +| `CredentialResolutionService` | Single authority: `(userId, name) → plaintext` — flat lookup + dotted-JSONPath | +| `ExecutionTokenService` | Mint/validate HMAC-SHA256 execution tokens; in-memory `jti` deny-list | +| `KnownProviderEnvVars` | The ~35 well-known provider env-var names to seed | +| `CredentialEnvSeeder` (server module) | One-shot startup seeder that copies those env vars into the default-user store | +| `SecretOutputMasker` (iface, `runtime/spi`) | `mask(executionId, userId, payload)` — output redaction hook | +| `NoOpSecretOutputMasker` (server module) | OSS default — returns the payload unchanged (no masking in OSS) | +| `CredentialMaskingResponseAdvice` (`runtime/controller`) | `@ControllerAdvice` that delegates to `SecretOutputMasker` on execution-read URIs | +| `SecretAwareHttpTask` / `Config` | Resolves `${NAME}` in HTTP-task headers before dispatch | +| `SecretAwareMcpService` | Resolves `#{NAME}` in MCP tool headers | +| `controller/SecretController + WorkerController` | REST surface (management at `/api/secrets`, worker resolve at `/api/workers/secrets`) | + +> **Naming:** the user-facing REST API (`/api/secrets`) and the UI page ("Secrets") keep the *Secrets* name for Conductor parity; the internal store/resolution/masking classes are named `Credential*`. + +### Data model + +The OSS schema (`schema-credentials.sql`) defines exactly **one** table: + +```sql +credentials_store( + user_id TEXT, + name TEXT, -- e.g. "GITHUB_TOKEN" + encrypted_value BLOB, -- [12B IV][ciphertext + 16B GCM tag] + created_at TEXT, updated_at TEXT, + PRIMARY KEY(user_id, name) +) +``` + +There are **no** `users`, `api_keys`, `secret_tags`, or `secret_disclosures` tables in OSS. (Agentspan' standalone server runs anonymous; tag-based RBAC and per-execution disclosure tracking for output masking are **Enterprise** features. The schema file's own comment notes that `credential_disclosures` is enterprise-only and not in the OSS schema.) + +The earlier `credentials_binding` table (logical-key → store-name indirection) was removed for parity with Conductor's flat-name secrets API. `credentials_store` is the canonical table name; a transient dev name (`secrets_store`) was never the canonical form. + +Schema: `server/conductor-agentspan-server/src/main/resources/schema-credentials.sql`. + +### Encryption at rest + +- **Algorithm:** AES-256-GCM (authenticated). +- **IV:** 12 random bytes per value. +- **Blob layout:** `[IV 12B][ciphertext + 16B GCM tag]` — Conductor-portable. +- **Master key:** 32 bytes, sourced from `AGENTSPAN_MASTER_KEY` (base64) in production; auto-generated to `~/.agentspan/master.key` (mode `0600`) on localhost for dev. +- **Rotation:** `agentspan admin credentials re-encrypt --old-key … --new-key …`. +- **Loss:** unrecoverable — self-hosters must back up the key. + +## 3.2 Execution token + +Workers never present the user's JWT or API key to `/api/workers/secrets`. The server mints an execution-scoped token at workflow start and embeds it in Conductor workflow variables as `__agentspan_ctx__`. + +``` +jti UUID unique ID, used for revocation deny-list +sub userId resolution lookup key +wid executionId audit trail +iat issued-at +exp iat + max(1h, agent.timeout_seconds) +scope "credentials" narrow, single-purpose +sig HMAC-SHA256 (master) +``` + +- **TTL:** `max(1h, execution timeout)` — long-running agents don't expire mid-run. +- **Revocation:** server keeps an in-memory deny-list keyed by `jti`. On execution cancel/terminate the `jti` is added; entries self-prune at `exp`. OSS = process-local; Enterprise can durably persist. +- **Declared-name binding:** at dispatch time the token records the set of secret names declared by the tool/agent. The resolve endpoint rejects names outside that set — bounds the blast radius of a compromised token. **Prefix-permissive for JSONPath:** if the parent `GCP_SVC` is declared, requests for `GCP_SVC.project_id` are allowed (the dot boundary is required — `FOO` does not permit `FOOBAR.x`). Rationale: JSONPath access doesn't expand the blast radius, since the parent secret already grants the whole blob. +- **Rate limit:** 120 calls/min/token (configurable). + +### Resolution pipeline + +`CredentialResolutionService.resolve(userId, name)`: + +1. **Flat name** (no `.`): `storeProvider.get(userId, name)` → return value (or null). +2. **Dotted name** (Conductor-parity JSONPath): split on first `.`. Fetch the base secret, parse it as JSON, walk the remaining dotted path via Jackson, return the leaf as a string (text nodes unquoted; other types as compact JSON). Returns null if the base isn't JSON or the path doesn't resolve. + +Examples: + +``` +GCP_SVC → raw stored value +GCP_SVC.project_id → field "project_id" from JSON-valued GCP_SVC +BLOB.auth.oauth.client_id → deeply nested extraction +GCP_SVC.does_not_exist → null +FLAT_TOKEN.field → null (FLAT_TOKEN isn't JSON) +``` + +Constraint: dotted resolution always splits on the **first** `.`. Don't put dots in secret names — store them under dot-free names and address fields via dotted paths. + +No indirection layer beyond JSONPath. (Earlier designs had a `logical_key → store_name` binding table; it was removed for parity with Conductor's flat-namespace secrets API. Multi-environment use cases switch by changing the stored value, not by rebinding.) + +The server itself does **not** perform an env-var fallback. Env-var convenience is provided by: + +- **`CredentialEnvSeeder`** (driven by the `KnownProviderEnvVars` list) — at server startup, copies any of the ~35 well-known provider env vars (OpenAI, Anthropic, AWS, GCP, etc.) into the default user's credentials store. So `export OPENAI_API_KEY=…` still "just works" without any setup. +- **SDK fallback** — when `secret_strict_mode=false`, missing names from `/resolve` fall back to `os.environ` in the worker process (local-dev compat). + +## 3.3 API surface + +Two namespaces, two auth primitives. **The path itself documents which auth is required.** See [api-design.md](api-design.md) for the full REST surface. + +| Namespace | Auth | Consumer | Purpose | +|---|---|---|---| +| `/api/secrets/*` | Login JWT or API key (`AuthFilter`) | UI, CLI, humans | Management — create / read / update / delete / list secrets | +| `/api/workers/*` | Execution token (`ExecutionTokenService`) | Distributed workers | Runtime — pull declared secrets for the current execution | + +This split is intentional. Earlier designs put both under `/api/secrets` with `/resolve` as a subpath, but that hid the auth-boundary difference behind a path segment. `/api/workers/*` is reserved for future token-mediated worker endpoints (heartbeat, lease extension, handoff, …) using the same execution-token primitive. + +### Conductor-parity surface + +Mirrors `io.orkes.conductor.server.rest.SecretResource`. + +| Method | Path | Body | Returns | +|---|---|---|---| +| `POST` | `/api/secrets` | — | `List` of names | +| `GET` | `/api/secrets` | — | `List` (RBAC-filtered; same set as POST in OSS) | +| `GET` | `/api/secrets/{key}` | — | plaintext value (`text/plain`); `404` if missing | +| `PUT` | `/api/secrets/{key}` | raw string | `200` (upsert) | +| `DELETE` | `/api/secrets/{key}` | — | `204` | +| `GET` | `/api/secrets/{key}/exists` | — | `true` / `false` | + +> Tag CRUD (`/api/secrets/{key}/tags`) is **not** in OSS — `secret_tags` is an Enterprise feature. + +`GET /{key}` returns plaintext (Conductor parity). Every read is audit-logged. RBAC will gate this in Enterprise; in OSS, anyone with management auth can read or overwrite, so hiding plaintext on GET would be theater. + +### V2 listing (Agentspan extension, mirrors Conductor V2) + +| Method | Path | Returns | +|---|---|---| +| `GET` | `/api/secrets/v2` | `List` — name + partial + created_at + updated_at | + +`partial` follows the OpenAI/GitHub convention: first-4 + `…` + last-4. The UI uses this endpoint for the secrets table; the v1 `POST /api/secrets` is reserved for strict-parity callers. + +### Worker secret fetch (Agentspan-only) + +Conductor has no equivalent — its workers receive substituted plaintext at task dispatch. Agentspan workers are out-of-process (often user-written, sometimes on untrusted infra), so they pull declared secrets at runtime using the execution token embedded in `__agentspan_ctx__`. + +Lives at `/api/workers/secrets` (not `/api/secrets/resolve`) because its auth model — execution token, not login session — is fundamentally different from the other `/api/secrets/*` endpoints. The path makes the boundary visible. + +``` +POST /api/workers/secrets +{ + "token": "", + "names": ["GITHUB_TOKEN", "OPENAI_API_KEY"] +} + +200 → { "GITHUB_TOKEN": "ghp_…", "OPENAI_API_KEY": "sk-…" } + (missing names omitted; SDK chooses env fallback or error per strict_mode) +401 → token expired / revoked / invalid — no env fallback +429 → rate limit — no env fallback +5xx → server error +``` + +Every resolve call is audit-logged: `{userId, executionId, taskId, names, timestamp, ip}`. + +## 3.4 Secret injection by tool type + +Credentials are declared at definition time and resolved at execution time. The injection mechanism varies by where the credential needs to land. + +| Tool kind | Where resolved | Where injected | Mechanism | +|---|---|---|---| +| `@tool(secrets=[...])` | Worker, in-process | `os.environ` for the call | Server fetch → `inject_via_env` (lock-around-invoke) — see §4 | +| `Agent(cli_commands=True)` | Worker, in-process | `os.environ` for the call | Auto-mapped from `CLI_CREDENTIAL_MAP`, same helper | +| HTTP tool (system task) | Server | Request headers | `${NAME}` rewritten by `SecretAwareHttpTask` | +| MCP tool (system task) | Server | Tool-server headers | `#{NAME}` rewritten by `SecretAwareMcpService` | +| Framework passthrough (LangGraph/LangChain/OpenAI/ADK) | Worker, in-process | `os.environ` for `invoke()` | Same `inject_via_env` helper as native tools — see [framework-integration.md](framework-integration.md) | +| External worker | Caller's process | Caller's responsibility | Direct `POST /api/workers/secrets` with `__agentspan_ctx__` token | +| LLM provider keys | Server | Provider client init | Same pipeline via `AIModelProvider` | +| Vector DB keys | Server | Provider client init | Same pipeline via `VectorDBProvider` | + +All in-process injection paths (the first three rows) share a single process-wide lock so concurrent invocations don't clobber each other's env. The lock is the only safety mechanism — there's no subprocess isolation. For throughput, scale by running additional worker processes. The full SDK injection contract is §4. + +### SDK declaration + +```python +# Declare secrets the tool needs — server resolves at runtime, value reaches +# os.environ for the duration of this call. +@tool(secrets=["GITHUB_TOKEN"]) +def fetch_issues(repo: str) -> str: + token = os.environ["GITHUB_TOKEN"] + ... + +# Same secrets, read via the contextvars accessor instead of env. +# Prefer this when the underlying SDK accepts an explicit api_key — avoids +# the env-mutation lock entirely. +@tool(secrets=["OPENAI_API_KEY"]) +def call_openai(prompt: str) -> str: + key = get_secret("OPENAI_API_KEY") + client = OpenAI(api_key=key) + ... + +# Agent-level — auto-mapped for known CLIs +Agent(cli_commands=True, cli_allowed_commands=["gh", "git"]) +# → resolves GITHUB_TOKEN, GH_TOKEN automatically +``` + +### Worker flow + +``` +Conductor poll → task picked up + │ + ├─ Read __agentspan_ctx__ → execution token + ├─ Compute needed names: declared @tool/Agent set ∪ CLI auto-map + ├─ WorkerCredentialFetcher.fetch(token, names): + │ POST /api/workers/secrets + │ ├─ 200 + missing names → raise CredentialNotFoundError (terminal) + │ ├─ 401 → raise CredentialAuthError (terminal) + │ ├─ 429 → raise CredentialRateLimitError (terminal) + │ └─ 5xx → raise CredentialServiceError (terminal) + └─ inject_via_env(secrets, lambda: tool_fn(**kwargs)): + with process_wide_lock: + save previous os.environ values + os.environ.update(secrets) + try: return tool_fn(**kwargs) + finally: restore previous os.environ values +``` + +### HTTP / MCP placeholder rewriting (server-side) + +For tools that execute as Conductor system tasks (no worker process owns them), the server resolves credentials before dispatching the HTTP/MCP call. Both HTTP and MCP use the `#{NAME}` sigil (regex `#\{[\w.]+}`): + +``` +# HTTP task headers +Authorization: Bearer #{GITHUB_TOKEN} +X-Project: #{GCP_SVC.project_id} # JSONPath into a JSON-valued secret + +# MCP tool headers +X-API-Key: #{OPENAI_API_KEY} +X-Client-Id: #{BLOB.auth.oauth.client_id} # nested JSONPath +``` + +Dotted names go through the same `CredentialResolutionService` as worker-side resolution, so the JSONPath syntax is uniform across all four call paths (worker `/api/workers/secrets`, HTTP placeholder, MCP placeholder, server-side LLM/VectorDB providers). + +This means credential material **never leaves the server** for system-task tools — workers don't see it, neither does Conductor (placeholders are rewritten on egress). + +### Framework passthrough (LangGraph / LangChain / OpenAI SDK / Google ADK) + +Framework agents run third-party code in-process and read keys from `os.environ` (e.g. `langchain_openai` reads `OPENAI_API_KEY` itself). The runtime resolves declared credentials, temporarily sets them in `os.environ` around the framework invocation, then restores prior state. This path is **single-threaded by construction** — the worker holds a process-wide lock during the framework call to prevent credential bleed across concurrent executions. Multi-threaded scaling for framework agents requires separate worker processes. The full mechanism and per-framework specifics live in [framework-integration.md](framework-integration.md); the injection contract every SDK must honor is §4 below. + +### Per-user LLM provider keys + +LLM provider keys and Vector DB keys are resolved **server-side** through the same `CredentialResolutionService` pipeline (last two rows of the table above). `AIModelProvider` / `VectorDBProvider` resolve `(userId, name)` at client-init time, so two users on one server transparently use distinct keys and credential material never reaches a worker process for these paths. This is the foundation for per-user / per-tenant model routing. + +## 3.5 Output masking (defense in depth on the read path) + +Even with all the controls above, a tool's *output* can leak a secret value verbatim — e.g. `gh` prints `error: authentication failed (token: ghp_realtoken123)` to stderr, and that string ends up in the Conductor task output. Anyone with execution-read permission would then see the plaintext. + +> **OSS is a no-op; output masking + disclosure tracking are ENTERPRISE.** The OSS server wires the read-path hook (`CredentialMaskingResponseAdvice`, a Spring `@ControllerAdvice`), but it delegates to the `SecretOutputMasker` SPI whose OSS implementation (`NoOpSecretOutputMasker`) returns the payload unchanged. There is no `secret_disclosures`/`credential_disclosures` table in the OSS schema (§3.1). The mechanism described below is the **Enterprise** behavior, which plugs a real masker into the same hook. + +The Enterprise masker closes that gap: + +1. **Disclosure tracking** — `WorkerController.resolveSecrets` writes one disclosure row per successfully resolved name, scoped to the execution id + user id (Enterprise table). +2. **Read-side redaction** — `CredentialMaskingResponseAdvice` activates on per-execution read URIs: `/api/agent/executions/{id}` (+ `/full`, `/tasks`), `/api/agent/execution/{id}`, and the bare-id `/api/agent/{id}/status`. The Enterprise masker pulls the disclosed names, fetches their **current** plaintext from the credential store, parses the response body as JSON, walks every string node, and replaces each occurrence of a disclosed value with `***NAME***`. Tree-walking (rather than literal `String.replace` on the JSON text) is required so values that contain newlines, quotes, or other JSON-escaped characters are still matched — JSON serialization would have escaped them in the wire payload. + +Key properties (Enterprise masker): + +- **Read-time, not write-time.** Storage rewrite would be irreversible. Rotation handles itself: the always-current store value is what gets masked. +- **Minimum-length floor (8 chars).** Shorter values produce too many false positives in natural-language output. +- **Literal substring replace** (not regex) — safe for values containing metacharacters. +- **JSON-aware** — values containing `"`, `\`, newlines, or other characters that JSON serialization escapes are still masked because the masker matches against unescaped text-node values, not the wire payload. +- **Best effort.** If anything fails (parse error, no user context, no disclosures), the body passes through unchanged. Masking should never block a response. +- **Agentspan-owned paths only by default.** The advice always masks Agentspan' own `/api/agent/*` reads. The raw Conductor `/api/workflow/{id}` read is host-owned, so masking it is **opt-in** via `agentspan.credentials.mask-workflow-reads=true` (default `false`) — this keeps the library from mutating an embedding host's workflow responses just by being on the classpath. +- **Bounded retention.** Enterprise prunes disclosure rows on a retention schedule; older execution payloads remain readable but will not be masked — by design, a long-since-disclosed token should have been rotated anyway. + +What this does **not** cover: + +- **List endpoints** (`GET /api/agent/list`, `GET /api/agent/executions`, `GET /api/agent/executions/search`) — these return aggregate metadata, not per-execution payload bodies. The advice intentionally does **not** activate on list responses: there is no single execution id to scope the disclosure set against, and list rows surface summary fields (status, timestamps, names) rather than task outputs. If a secret can appear in a *list-row* field (e.g. an agent name shaped like an env-var template), file it as a separate masking gap. +- **POST / mutation endpoints** that echo input (e.g. `/{executionId}/respond`, `/{executionId}/signal`) — the input body is what the caller already supplied, so masking it would help nothing; the *task output* it triggers is still masked when read back through the GET path. +- **Live SSE streams** (`/api/agent/stream/{id}`) — events flow through the streaming converter, which the advice doesn't intercept. Follow-up work. +- **Bypassing Agentspan to hit Conductor directly** — Conductor is internal-only per the existing security model. +- **Off-server log files** — worker stdout captured by the orchestrator. Agentspan can't reach into those. + +## 3.6 Developer experience tiers + +| Tier | Setup | What happens | +|---|---|---| +| 0 — local dev | `export OPENAI_API_KEY=…; python agent.py` | Seeder copies env into default-user store at boot; resolve serves it back | +| 1 — set once | `agentspan credentials set OPENAI_API_KEY sk-…` | No env var needed; persists across restarts | +| 2 — SDK auto-auth | `AgentRuntime()` on localhost | Auto-authenticates as default user; zero config | +| 3 — team / enterprise | `agentspan login` (OIDC in Enterprise) | Token in `~/.agentspan/config.json`; CLI + SDK both use it | + +`AgentConfig.secret_strict_mode = True` disables the SDK env-var fallback and the startup seeder — required for compliance-sensitive deployments. Recommended default in Enterprise. + +## 3.7 Security model (summary) + +| Threat | Mitigation | +|---|---| +| Worker process compromise | Token has 1h+ TTL, narrow scope, declared-name binding, revocable | +| Credential bleed across concurrent agent invocations | `inject_via_env` holds a process-wide lock across mutation + invoke + restore. See §4. | +| `/proc/PID/environ` exposure | Env mutations are scoped to the duration of a single tool call and restored synchronously; only present during the locked region. | +| Token replay | `jti` deny-list + `exp` + `wid` | +| Tool exfiltration via egress | Names bounded to declared set; audit trail; rate-limited | +| Conductor variable leakage | Conductor is internal-only; agentspan-server is sole external entry point | +| Master key loss | Documented; backup is operator's responsibility | +| Plaintext leaks via tool output (e.g. CLI error messages echo a token) | **Output masking (Enterprise)** — a real `SecretOutputMasker` behind `CredentialMaskingResponseAdvice` redacts disclosed values from execution-read response bodies (§3.5); OSS masker is a no-op | +| **Cross-tenant leak when SDK is embedded in a host app** (e.g. Django, FastAPI) | **Run agentspan-server as a separate service.** The process-wide env-injection lock is insufficient when arbitrary host-app code can read `os.environ` during the injection window. See §4.6. | + +## 3.8 OSS vs Enterprise boundary + +| Concern | OSS | Enterprise | +|---|:-:|:-:| +| `CredentialStoreProvider` interface, encrypted DB store | ✓ | — | +| Env-var seeding + SDK fallback | ✓ | — | +| Management + `/resolve` APIs | ✓ | — | +| Execution token mint/validate (in-memory deny-list) | ✓ | — | +| CLI auto-mapping registry | ✓ | — | +| Subprocess isolation | ✓ | — | +| HTTP/MCP placeholder resolution | ✓ | — | +| Per-user LLM / VectorDB resolution | ✓ | — | +| Output masking (`SecretOutputMasker`): OSS no-op vs real masker | no-op | ✓ | +| Disclosure tracking (per-execution disclosed-name table) | — | ✓ | +| Secret tags / tag-based RBAC | — | ✓ | +| OIDC / SSO authentication | — | ✓ | +| AWS SM / GCP SM / Azure KV / HashiCorp / CyberArk / Doppler / K8s Secrets | — | ✓ | +| Org / team RBAC, credential policies | — | ✓ | +| Durable audit store, durable token revocation | — | ✓ | + +Enterprise plugs in via the same `CredentialStoreProvider`, `SecretOutputMasker`, and `AuthFilter` interfaces — no OSS changes required. + +--- + +# Part 4 — SDK Secret-Injection Contract + +**Status:** Required for every SDK that supports framework passthrough. +**Audience:** SDK implementors (Python, .NET, TypeScript, Java, future languages). See [sdk-design.md](sdk-design.md) for the broader cross-SDK contract. + +This part defines the contract every Agentspan SDK must follow when injecting resolved secrets into third-party framework agents (LangChain, LangGraph, OpenAI Agents, Claude Agent SDK, Google ADK, Semantic Kernel, etc.). The contract exists because the obvious-looking implementation — mutate process environment, run framework, restore — is fundamentally unsafe under concurrency and has burned every SDK that's tried it. + +## 4.1 The problem + +Frameworks like `langchain_openai.ChatOpenAI()` or `OpenAI()` read the API key from the process environment (`OPENAI_API_KEY`) at client-construction time. To support per-execution secrets, an SDK has to make the framework see a *specific* key value for *this* invocation. + +The naïve approach: set the env var, run the framework, unset. + +``` +# THIS IS THE BROKEN PATTERN — do not implement it +os.environ["OPENAI_API_KEY"] = resolved_value +try: + framework.invoke(...) +finally: + os.environ.pop("OPENAI_API_KEY", None) +``` + +Process-level environment is a **single shared mutable global**. Two concurrent invocations clobber each other: + +``` +T=0 Thread A: os.environ["OPENAI_API_KEY"] = "keyA" +T=1 Thread B: os.environ["OPENAI_API_KEY"] = "keyB" +T=2 Thread A: framework.invoke() → reads env, sees "keyB" ← WRONG TENANT +T=3 Thread B: framework.invoke() +T=4 Thread A: pop OPENAI_API_KEY ← removes B's value too +T=5 Thread B: reads env, sees nothing +``` + +Three failure modes for Thread A's call: wrong key, no key, or wrong-then-no-key mid-stream. This isn't hypothetical — it triggers every time two framework agents run concurrently on one worker process. Conductor polls multiple tasks in parallel by default, so this happens on the very first concurrent invocation. + +A lock around just the *mutation* step doesn't help — the framework reads env *after* the lock is released. The lock must cover **mutation + framework invocation + restoration** as one atomic region. That fixes correctness but serializes everything: one worker process = one framework call at a time. + +## 4.2 The two-tier solution + +Every SDK must implement both tiers and prefer **tier 1** wherever the framework supports it. + +### Tier 1 — Explicit-key injection (preferred, concurrent) + +The framework's model client accepts an explicit `api_key` parameter. Resolve the secret, hand it directly to the client constructor, never touch process environment. + +```python +# Python — preferred +client = ChatOpenAI(api_key=resolved_secrets["OPENAI_API_KEY"]) +``` + +```csharp +// .NET — preferred +var client = new OpenAIClient(apiKey: resolved["OPENAI_API_KEY"]); +``` + +```typescript +// TypeScript — preferred +const client = new ChatOpenAI({ apiKey: resolved["OPENAI_API_KEY"] }); +``` + +No shared global state. Multiple threads can construct independent clients with independent keys. Fully concurrent. **This is the default path.** + +Where tier 1 lands cleanly: + +| Framework | Key parameter | +|---|---| +| LangChain `ChatOpenAI`, `ChatAnthropic`, etc. | `api_key=` on the model constructor | +| LangGraph (uses LangChain models underneath) | same | +| OpenAI SDK (`openai.OpenAI`, `AsyncOpenAI`) | `api_key=` on the client | +| Anthropic SDK | `api_key=` | +| Vercel AI SDK | `apiKey` in the provider config | +| Semantic Kernel | `apiKey:` argument to `AddOpenAIChatCompletion` etc. | + +### Tier 2 — Env-injection with lock-around-full-invoke (fallback, serialized) + +Some SDKs don't accept an explicit key — they only read from process env. Examples: Google ADK (`genai.configure` is process-global), Claude Agent SDK in CLI mode, anything that reads env at module-import time. + +For these, env injection is unavoidable. But the lock **must cover the entire framework invocation**, not just the mutation step: + +```python +# Tier 2 — env injection. Note the lock scope. +with _global_env_lock: + previous = {k: os.environ.get(k) for k in secrets} + os.environ.update(secrets) + try: + result = framework.invoke(...) # ← still inside the lock + finally: + for k, v in previous.items(): + if v is None: os.environ.pop(k, None) + else: os.environ[k] = v +``` + +Trade-off: tier 2 calls are strictly serial within one worker process. Throughput scales by adding worker processes (Conductor replicas), not by adding threads. **Document this limitation in the SDK's per-framework docs.** + +## 4.3 Lock discipline + +For tier 2 implementations: + +1. **One lock per process**, not per execution. The shared resource is `os.environ` (or `process.env`, or `Environment`). All tier-2 framework workers contend for the same lock. +2. **The lock must wrap mutation + invoke + restore.** No yielding control (no `await` outside the lock in async contexts, no manual `Thread.yield()`). +3. **In async contexts, use an async lock** (`asyncio.Lock`, `SemaphoreSlim`, async mutex). Never use a sync lock around an `await` — you'll either deadlock or block the event loop. +4. **The lock applies only to tier-2 paths.** Tier-1 (explicit-key) invocations must NOT acquire the lock. Mixing them defeats the concurrency benefit of tier 1. + +**One process-wide lock, shared across all callers.** Native `@tool` dispatch and framework passthrough MUST contend for the same lock. If you implement two locks (one per path) you reintroduce the bug for the case where a native `@tool` and a framework agent run concurrently. Every SDK's test suite must include the "shared single lock" test (Python's `test_native_dispatch_and_framework_share_one_lock` is the reference shape). + +## 4.4 User-facing API + +To enable tier 1, the SDK's agent-factory API must allow secrets to flow into the user's framework construction code. The recommended shape: + +```python +# Python — factory accepts a `secrets` dict +@agent(secrets=["OPENAI_API_KEY"]) +def my_agent(secrets): # ← new parameter + return AgentExecutor.from_agent_and_tools( + agent=create_openai_functions_agent( + ChatOpenAI(api_key=secrets["OPENAI_API_KEY"]), + tools=[...] + ) + ) +``` + +```typescript +// TypeScript +defineAgent({ + secrets: ["OPENAI_API_KEY"], + build: ({ secrets }) => new AgentExecutor({ + llm: new ChatOpenAI({ apiKey: secrets["OPENAI_API_KEY"] }), + ... + }) +}); +``` + +```csharp +// .NET +[Agent(Secrets = ["OPENAI_API_KEY"])] +static Agent BuildAgent(IReadOnlyDictionary secrets) => + new AgentBuilder() + .WithModel(new OpenAIClient(apiKey: secrets["OPENAI_API_KEY"])) + ... + .Build(); +``` + +**Backwards-compatibility for agents that don't accept the `secrets` argument:** the SDK falls back to tier 2 (env injection with lock-around-invoke). The fallback should log a warning recommending migration to the explicit-key API for concurrency. + +## 4.5 Test contract — every SDK MUST have these + +Two deterministic tests, paired. Both go in the SDK's test suite under a stable filename so the contract is visible. + +### Counterfactual ("buggy" path) + +Implements the broken pattern (no lock around invoke, or lock around mutation only). Uses a synchronization primitive (Barrier, Event, gate) to **force** the race deterministically: Thread A enters its fake invoke and blocks on a barrier; Thread B sets its env value; A is released and reads env. **Assertion: A observes B's value (or empty) — proving the race is observable under this implementation.** + +If this test ever starts passing (A observes its own value despite the race), it means the counterfactual is no longer a real counterfactual. Investigate why before deleting. + +### Fix-verification ("correct" path) + +Uses the same harness but invokes through the SDK's real injection helper. **Assertion: A always observes A's value, even when B is concurrently injecting B's value.** + +### Why deterministic, not stress + +Race tests run with raw `Thread.Start()` and `assertEventually` are flaky — they pass 99% of the time even when broken. The barrier/gate technique makes the bug 100% reproducible. The fix test is 100% deterministic too. No flake, no `repeat(1000)`, no CI heartburn. + +### Reference test names (use these or equivalents) + +- `test_buggy_injection_races` (or `_clobbers_concurrent_value`) +- `test_fixed_injection_isolates_concurrent_calls` + +> **As-built gap (C#/.NET):** the .NET suite (`CredentialInjectionConcurrentTest.cs`) ships only the fix-verification side (`FixedInjection_IsolatesConcurrentCalls` plus restore/exception cases). The paired **counterfactual** ("buggy") test is **missing** and should be added to satisfy this contract. + +## 4.6 Embedded deployments — the contract assumes a dedicated worker process + +Everything in §4.1–§4.5 assumes the SDK runs in a **dedicated Agentspan worker process** — a process whose only job is to poll Conductor and execute agent tools. Under that assumption, tier-2 (env-injection with a process-wide lock) is correct: the only code that reads `os.environ` during the injection window is the framework SDK itself, and concurrent agent invocations serialize via the lock. + +The contract **breaks** when you embed the SDK inside a host application that also runs unrelated code in the same process: Django, FastAPI, Flask, Rails, ASP.NET, a long-running CLI, anything where third-party libraries might read `os.environ` at unpredictable times. The reason isn't subtle: + +### The cross-tenant leak in an embedded process + +``` +Thread A (Agentspan worker) Thread B (e.g. Django request handler) +───────────────────────────── ─────────────────────────────────────── +inject_via_env({OPENAI_API_KEY: "userA"}) +os.environ["OPENAI_API_KEY"] = "userA" + a request from user X invokes: + openai.OpenAI() ← reads OPENAI_API_KEY + → uses userA's key ❌ +framework.invoke() +restore: pop OPENAI_API_KEY + another request reads env → no key +``` + +The lock prevents Agentspan-vs-Agentspan races. It cannot synchronize with arbitrary host-app code reading `os.environ`. Every Django middleware, signal handler, ORM connection initializer, Celery worker bootstrap, third-party library doing lazy env reads — any of them observing env during the injection window picks up the wrong tenant's secret. **This is a real cross-tenant credential leak** in any multi-tenant embedded deployment. + +The lock is the only safety mechanism for tier-2. It's local to Agentspan code paths. It is fundamentally insufficient when the surrounding process runs code Agentspan doesn't control. + +### Recommended architecture for embedded use cases + +**Run `agentspan-server` as a separate service** and have the host application call it as an HTTP client. The host process never holds a secret value, never mutates env, and never contends for the lock with arbitrary code. + +``` +┌─────────────────────────────┐ HTTP ┌──────────────────────────────┐ +│ Host app (Django/FastAPI) │ ───────> │ agentspan-server │ +│ - request handlers │ │ - dedicated worker pool │ +│ - calls AgentRuntime().run │ <─────── │ - inject_via_env is safe │ +│ - NO agent workers here │ │ (no host-app code in proc)│ +└─────────────────────────────┘ └──────────────────────────────┘ +``` + +The Python SDK supports this today — construct `AgentRuntime(server_url=…, auto_start_workers=False)` and the runtime becomes a thin HTTP client. The TS/.NET SDKs have equivalent client-only modes. + +### If you must embed, the discipline required + +If running a separate server isn't an option (single-binary deployment, edge-case constraints), the only safe pattern is **tier-1 explicit-key for every tool, with tier-2 hard-disabled**: + +1. **Every tool reads secrets via the contextvars/thread-local accessor** (`get_secret(name)` in Python, `getCredential(name)` in TS, `ToolContext.getCredential(name)` in Java, `ToolContext.GetCredential(name)` / `Secrets.Get(name)` in C#) — never `os.environ` / `process.env` / `Environment`. The tier-1 accessor is now present in **all four SDKs** (C# added), so the strict embedded discipline can be satisfied in every SDK. (Tier-2 env injection remains available as a fallback — C# `Conductor.AI.CredentialInjection.InjectViaEnvAsync`.) +2. **Every secret value is passed explicitly to the underlying client**: `OpenAI(api_key=key)`, `ChatAnthropic(api_key=...)`, etc. No client construction relies on env-var auto-discovery. +3. **Framework passthrough integrations that require env-only configuration are unsupported in embedded mode.** Specifically: Claude Agent SDK CLI mode, Google ADK `genai.configure`, anything that reads env at module-import time. Use only frameworks that accept an explicit `api_key=` parameter. +4. **Hard-disable tier-2 with a config flag** (planned: `AGENTSPAN_DISALLOW_ENV_INJECTION=1`). When set, `inject_via_env` (and equivalents) raise instead of mutating env. Provides loud failure instead of silent leak. +5. **Test the host app for env-read leakage.** Add a test that runs two concurrent agent invocations with different secrets and asserts no host-app code observed a transient value. This is hard but worth doing once if you're committed to embedding. + +The contextvars accessor is per-async-task / per-thread, so it doesn't suffer from the process-global problem. It's the *only* injection mechanism that's structurally safe inside a host application. + +### What the SDK can and can't enforce + +- **Can enforce:** `inject_via_env` raises when the disallow-env flag is set (planned; not yet implemented). +- **Cannot enforce:** that tool authors actually use `get_secret()` instead of `os.environ[name]`. The flag will surface that mistake at runtime — the user's framework client will fail to find a key — but only if they were going to rely on tier-2 anyway. A tool that imports a library that reads env at *import* time (before the agent invocation begins) gets nothing. + +### Decision table + +| Host app | Multi-tenant? | Recommended deployment | +|---|---|---| +| Standalone Agentspan worker (no other code in the process) | n/a | tier-1 preferred, tier-2 acceptable | +| Single-user CLI tool, no concurrent users | n/a | tier-1 or tier-2; either fine | +| Django / FastAPI / Flask / Rails, single tenant | no | tier-1 only; run server separately if possible | +| Django / FastAPI / Flask / Rails, multi-tenant | **yes** | **Run server separately.** If embedding, tier-1 only + `AGENTSPAN_DISALLOW_ENV_INJECTION=1`. | +| Notebook / REPL / development | no | either fine | + +The decision pivots on "is unrelated code reading `os.environ` in the same process while agents are running?" If yes, tier-2 is unsafe. If no, tier-2 is fine. + +## 4.7 Per-language notes + +**Java is tier-1-only by language constraint.** `System.getenv()` returns an unmodifiable map at JVM start, so the SDK *cannot* implement tier-2 env injection without reflection hacks against private JDK internals. The Java SDK exposes `ToolContext.getCredential(name)` on the `ToolContext` passed to each `@Tool` method, backed by the `internal.CredentialContext` thread-local populated by the worker immediately before invocation. Tool authors read declared credentials via `ctx.getCredential(...)` and pass them explicitly to model client constructors. Framework passthrough that depends on env-var auto-discovery doesn't work in Java; users must construct framework clients with explicit `api_key` arguments. This is exactly the contract the doc recommends for new languages — Java got it for free because the language wouldn't let us cheat. + +**Java ThreadLocal does not propagate across async boundaries.** `internal.CredentialContext` (the `ThreadLocal` behind `ToolContext.getCredential`) is populated on the worker thread immediately before `@Tool` invocation and cleared immediately after. If a tool spawns an `ExecutorService.submit(...)`, `CompletableFuture.runAsync(...)`, virtual-thread `Thread.startVirtualThread(...)`, or any other handoff to a different carrier thread, the secret is **not visible** in the spawned task — `ctx.getCredential(name)` returns `null` there. This is a known limitation: tool authors who need a secret on a background thread must capture it on the calling thread (e.g. `String tok = ctx.getCredential("X"); pool.submit(() -> useToken(tok));`) rather than calling `ctx.getCredential` from inside the lambda. Reactor / RxJava / Kotlin-coroutine context propagation is the user's responsibility — there is no `InheritableThreadLocal` because it would leak across unrelated executions sharing a thread pool. See `Example16CredentialsTool` for the supported pattern. + +### Guidance for new-language SDKs + +Three rules in priority order: + +1. **Start with tier 1.** Don't ship the SDK with env injection as the only path. If you have to ship env injection, build the explicit-key API in the same PR. +2. **The agent-factory API takes a `secrets` argument from day one.** Adding it later is a breaking change. +3. **Write the deterministic concurrent test before the feature ships.** §4.5 is a hard requirement, not a nice-to-have. + +## 4.8 Scope — what the contract covers + +The contract applies everywhere an SDK injects resolved secrets into a shared mutable global for the duration of an invocation. That includes: + +1. **Native `@tool` / handler dispatch** — when a user-authored tool declares `secrets=[…]` and the SDK injects those for the tool function. Even though "Conductor workers default to `thread_count=1`" was historically used to justify skipping the lock, that's a config-dependent workaround. The fix must hold regardless of worker config. +2. **Third-party framework passthrough** — LangChain, LangGraph, OpenAI Agents, Claude Agent SDK, Google ADK, Semantic Kernel, etc. +3. **Any future code path** that mutates process environment around a callable. + +## 4.9 Where the contract is implemented + +| SDK | Helper location | Used by | +|---|---|---| +| Python | `conductor.ai.agents.runtime.secret_injection.inject_via_env` | Native `_dispatch.py` + `frameworks/langchain.py`, `langgraph.py`, `claude_agent_sdk.py` | +| .NET | `Conductor.AI.CredentialInjection.InjectViaEnvAsync` | `WorkerManager.cs` (covers native handlers + OpenAI / SemanticKernel / GoogleADK integrations) | +| TypeScript | `src/credentials.ts` (`injectSecretsForInvocation`) | `worker.ts` (covers native tools + LangChain / LangGraph serializers) | +| Java | `ToolContext.getCredential` (backed by `internal.CredentialContext` thread-local) + `internal.WorkerCredentialFetcher` (HTTP client for `/api/workers/secrets`) | `internal.WorkerManager.executeTask` (covers every `@Tool` method; tier-1 explicit-key only — env injection structurally impossible in Java) | + +--- + +# Part 5 — Secrets Management UI + +A **Secrets** management page in the Agentspan UI lets users store, view, update, and delete per-user secrets. It follows the existing React 18 + MUI 7 + React Query design language. The page is flat-name only (no bindings UI — the logical-key → store-name indirection was removed backend-side for Conductor parity, see §3.1). + +> **Naming:** the page and route are *Secrets*, matching the `/api/secrets` REST surface. (Internally the server classes are named `Credential*`; see §3.1.) + +## 5.1 Architecture + +### Page structure + +A single `/secrets` route (`SECRETS_URL.BASE`), registered as `SecretsPage` at `ui/src/pages/secrets/SecretsPage.tsx`. No sub-routes. It sits in the existing **Definitions** sidebar submenu — there is **no** `/credentials` route, no `Settings` submenu, and no `ui/src/pages/credentials/` directory. + +### State management + +React Query via `useFetchContext` + a thin `secretFetch` wrapper. Same pattern as the other definition list pages. Local `useState` for dialog visibility and toast messages. + +### Auth — none in OSS + +The standalone Agentspan server runs **anonymous**, so the Secrets page sends **no per-request token**. There is **no** `useCredentialAuth` hook, **no** `LoginDialog`, **no** `credentialFetch`/login flow, and **no** `POST /auth/login`. The page passes `{ token: null, onUnauthorized: () => {} }` to its API hooks; auth, when present, is the embedding host's concern. + +## 5.2 File structure + +| File | Responsibility | +|------|---------------| +| `ui/src/pages/secrets/SecretsPage.tsx` | Main page: table, add/edit dialog, delete confirm, toasts | +| `ui/src/pages/secrets/hooks/useSecretsApi.ts` | `secretFetch` wrapper + hooks: `useListSecrets` (`GET /api/secrets/v2`), create/update/delete | + +Route + sidebar wiring (existing files): + +| File | Entry | +|------|-------| +| `ui/src/utils/constants/route.ts` | `SECRETS_URL.BASE = "/secrets"` | +| `ui/src/routes/routes.tsx` | route for `SecretsPage` at `SECRETS_URL.BASE` | +| `ui/src/components/Sidebar/sidebarCoreItems.tsx` | `secretsItem` (title "Secrets", `linkTo: SECRETS_URL.BASE`) under the **Definitions** submenu at **position 300** (`CORE_SIDEBAR_POSITIONS.DEFINITIONS.secretsItem = 300`) | + +## 5.3 Data model + +API responses the UI consumes (served by `GET /api/secrets/v2`, see §3.3): + +```typescript +// list item — name + masked partial + timestamp (from GET /api/secrets/v2) +type SecretListItem = { + name: string; // store name, e.g. "GITHUB_TOKEN" + partial: string; // e.g. "ghp_...6789" + updated_at: string; // ISO-8601 +}; +``` + +There is no login request/response type — the OSS server is anonymous. + +## 5.4 Component details + +### SecretsPage + +- **Header**: `SectionHeader` titled "Secrets" with an `+ Add Secret` primary action. A descriptive note ("Values are encrypted at rest and never shown after creation") sits as a `Typography` subtitle below the header. No Logout control (anonymous server). +- **Search**: quick-filter `TextField` filters the list client-side by secret name. +- **Table**: MUI `Table` / `TableHead` / `TableBody`. Columns: Name | Value (partial) | Last updated | Actions. +- **Add button**: opens the add/edit dialog in "add" mode. +- **Edit icon**: opens the dialog in "edit" mode (Name read-only, Value cleared — user re-enters to update). +- **Delete icon**: `useState(null)` for `confirmDeleteName`; conditionally renders `{confirmDeleteName && }` (`ConfirmChoiceDialog` has no `open` prop and must be conditionally mounted) with `isInputConfirmation` + `valueToBeDeleted`. On confirm, calls the delete mutation. +- **Toast**: `{toastMessage && }` — guard required because `message` is a required prop. + +### Add/Edit dialog + +- **Fields**: Name (monospace, required; read-only in edit mode) + Value (password input with show/hide toggle, required in add and edit). +- **Validation**: Name must be non-empty. UPPER_SNAKE_CASE is *suggested* in helper text but not enforced — the backend supports lowercase/hyphen names too. Only blank names are rejected. +- **Submit**: `POST /api/secrets` (add) or `PUT /api/secrets/{name}` (edit) via `secretFetch`. + +### useSecretsApi + +- `secretFetch(path, ctx, options, onUnauthorized)` wraps `fetchWithContext`, inheriting the `VITE_WF_SERVER` base URL and `cleanPath` logic. In OSS it is called with `token: null` and a no-op `onUnauthorized`. +- `useListSecrets({ token, onUnauthorized })` → `GET /api/secrets/v2`, cache key `[ctx.stack, "/secrets/v2"]`, `retry: false`. +- Create / update / delete mutations accept `onSuccess` / `onError` callbacks so the page can set toast messages, and invalidate the list cache key on success. + +## 5.5 Sidebar + +The `secretsItem` lives in the existing **Definitions** submenu at position 300 — no new submenu, no renumbering: + +```typescript +// CORE_SIDEBAR_POSITIONS.DEFINITIONS +secretsItem: 300, + +// Sidebar item (under the Definitions submenu) +{ + id: "secretsItem", + title: "Secrets", + icon: null, + linkTo: SECRETS_URL.BASE, + activeRoutes: [SECRETS_URL.BASE], + position: D.secretsItem, // 300 +} +``` + +## 5.6 Error handling + +| Scenario | Behaviour | +|----------|-----------| +| 404 on delete (already gone) | Toast: "Secret not found — it may have already been deleted", severity=warning | +| 409 on create (name exists) | Form-level error on Name field: "A secret with this name already exists" | +| Network error | Toast: "Network error — please try again", severity=error | + +(There is no 401/login handling in OSS — the server is anonymous.) + +## 5.7 Testing + +- Tests live alongside the Secrets page (`ui/src/pages/secrets/`). +- **SecretsPage**: renders list from `/api/secrets/v2`; delete shows `ConfirmChoiceDialog`, typing the name enables confirm; delete success shows toast and refetches. +- **Add/Edit dialog**: blank name rejected; submit calls `POST` (add) or `PUT` (edit); show/hide toggle changes input type. + +--- + +# Part 6 — Known Gaps & Follow-ups + +- **Enterprise vault providers** — design done; implementations not yet shipped. +- **Durable token revocation** — OSS deny-list is in-memory; bounded risk because TTL ≤ execution timeout, but a server crash drops revocations. +- **Multi-threaded framework passthrough throughput** — tier-2 (env-injection) calls serialize under the shared lock; scale by adding worker processes. Tier-1 (explicit-key via `get_secret()` or factory `secrets=` arg) runs fully concurrent. +- **TypeScript SDK** — credential resolution path needs verification against the Python SDK's contract. +- **Java SDK framework passthrough** — Java SDK has runtime credential resolution (`ToolContext.getCredential` accessor, `@Tool(credentials={…})` declaration). What's NOT supported: framework integrations (LangChain4j, OpenAI-Agents) that depend on env-var auto-discovery — Java's `System.getenv()` is immutable, so users must construct those clients with explicit `api_key` arguments. +- **Credential rotation / expiry** — no first-class TTL on stored credentials; rotation is a `PUT` from the operator. +- **`AGENTSPAN_DISALLOW_ENV_INJECTION` flag** — planned hard-disable for tier-2 in embedded deployments; not yet implemented. +- **Live SSE stream masking** — `CredentialMaskingResponseAdvice` doesn't intercept `/api/agent/stream/{id}`; even with the Enterprise masker, secrets in streamed task output are not masked. +- **Code execution sandboxing** — `LocalCodeExecutor` + `CommandValidator` are not security boundaries; untrusted code requires Docker/serverless executors. Per-language Docker images and a managed serverless backend are follow-ups. + +--- + +# References + +- Sibling design docs: [agentspan-design.md](agentspan-design.md), [api-design.md](api-design.md), [sdk-design.md](sdk-design.md), [framework-integration.md](framework-integration.md). +- Server code: `server/conductor-agentspan/src/main/java/dev/agentspan/runtime/{credentials,spi,controller}/` (+ built-in store impls in the `conductor-agentspan-server` module). +- Python SDK examples: `sdk/python/examples/16_credentials_*.py` (a–k). +- Tests: `server/src/test/java/.../credentials/`, `sdk/python/tests/{unit,e2e}/test_*credential*.py`, `ui/e2e/credentials.spec.ts`. diff --git a/design/validation/README.md b/design/validation/README.md new file mode 100644 index 000000000..03ced0d42 --- /dev/null +++ b/design/validation/README.md @@ -0,0 +1,39 @@ +# Validation & E2E — Methodology + +**Status:** Consolidated 2026-06-26 + +**Scope:** How the Agentspan SDKs are validated. This is the cross-cutting overview; +each SDK has its own doc with the concrete suites, commands, and CI wiring. For how +each SDK is built, see the [reference implementations](../sdk-design/languages/); +for the SDK contract being validated, see [sdk-design.md](../sdk-design.md). + +## Two complementary approaches + +| Approach | What it proves | LLM judge? | SDKs | +|---|---|---|---| +| **Deterministic E2E suites** | The SDK compiles + executes real agents against a real server correctly | **No** — assertions are deterministic (`plan()` structure, workflow-task status, Conductor API side-effects) | Python, TypeScript, Java, C# (all 4) | +| **Examples-quality validation framework** | Ported examples behave correctly across models, and Agentspan-compiled execution matches native-framework execution | **Yes** — an LLM judge scores semantic quality | Python, TypeScript | + +The determinism rule is a project constraint (see `CLAUDE.md`): **e2e must not use +LLM-as-judge** except where the explicit purpose is judging quality/output/evals. The +examples-quality framework is the only place a judge appears. + +## Shared harness + +All e2e runs exercise the **real stack** — no mocks: +- The **Agentspan server** jar on `:6767` (`conductor-agentspan-server/build/libs/agentspan-runtime.jar`). +- **mcp-testkit** for HTTP/MCP tool endpoints (test infra only — never an SDK dependency). +- Credentials managed via the `agentspan` CLI / the server secrets API (never read from the worker env). +- Config via `AGENTSPAN_SERVER_URL`, `AGENTSPAN_CLI_PATH`, `AGENTSPAN_LLM_MODEL` (model defaults to `openai/gpt-4o-mini`). + +## Per-SDK validation docs + +| SDK | Doc | E2E suites | Quality framework | +|---|---|---|---| +| Python | [python-validation.md](python-validation.md) | ✅ `sdk/python/e2e/` | ✅ `sdk/python/validation/` | +| TypeScript | [typescript-validation.md](typescript-validation.md) | ✅ `sdk/typescript/tests/e2e/` | ✅ `sdk/typescript/validation/` | +| Java | [java-validation.md](java-validation.md) | ✅ `sdk/java/e2e/` | — | +| C# | [csharp-validation.md](csharp-validation.md) | ✅ `sdk/csharp/tests/AgentspanE2eTests/` | — | + +Each per-SDK doc is cross-linked from that SDK's +[reference implementation](../sdk-design/languages/) "Testing" section. diff --git a/design/validation/csharp-validation.md b/design/validation/csharp-validation.md new file mode 100644 index 000000000..ff043108f --- /dev/null +++ b/design/validation/csharp-validation.md @@ -0,0 +1,170 @@ +# C# SDK — Validation & E2E + +**Status:** Created 2026-06-26 + +**Scope:** How the C# SDK is validated end-to-end against a real Agentspan server. Unlike the Python and TypeScript SDKs — which add a separate *examples-quality* validation framework (model matrix runs + LLM-as-judge scoring; see [`python-validation.md`](python-validation.md) and [`typescript-validation.md`](typescript-validation.md)) — C# has **no examples-quality / LLM-judge framework**. C# validation is a single **deterministic** xUnit e2e suite (`AgentspanE2eTests`) plus one standalone guardrail-matrix example program (`90_GuardrailE2eTests`). Per `CLAUDE.md`, the e2e suite never uses an LLM to *judge* output; assertions are JSON-path / object-graph / status checks (the only LLM calls are the agent runs themselves). Related: [`README.md`](README.md) (methodology), [`../sdk-design/languages/csharp-implementation.md`](../sdk-design/languages/csharp-implementation.md) (implementation), [`../sdk-design.md`](../sdk-design.md) (cross-SDK design). + +--- + +## 1. Overview + +C# validation has two layers: + +1. **`AgentspanE2eTests`** — the xUnit e2e project at `sdk/csharp/tests/AgentspanE2eTests/`. This is the authoritative regression suite. It mirrors the Python e2e helpers and suite numbering (the source comments cross-reference Python's `_agent_def()`, `_tool_names()`, `_get_workflow()`, etc.). +2. **`90_GuardrailE2eTests`** — a standalone `Exe` example at `sdk/csharp/examples/90_GuardrailE2eTests/` that exercises the full 27-cell guardrail matrix (Position × Type × OnFail) against a live server, printing a PASS/FAIL table and exiting non-zero on any failure. It is run as a program, **not** via `dotnet test`. + +**No LLM judge.** There is no `validation/` directory, no `runs.toml`, no model-matrix orchestrator, and no scoring rubric on the C# side. Output quality is never machine-graded. This is intentional and consistent with `CLAUDE.md` rule 1 (no LLM for validation unless judging quality) — every C# assertion is deterministic. + +**Assembly name note.** The test assembly is intentionally named `AgentspanE2eTests` (not renamed to match the `Conductor.AI` namespace move). The production library `Conductor.AI` grants `[assembly: InternalsVisibleTo("AgentspanE2eTests")]` (see `sdk/csharp/src/Conductor.AI/Conductor.AI.csproj`), so renaming the assembly would break tests that inspect `internal` SDK members (tool `ToolType`/`Config`, etc.). The CI filter `FullyQualifiedName!~AgentspanE2eTests` also depends on this exact name. Namespace inside the project is `Conductor.AI.E2eTests`. + +--- + +## 2. E2E test suite + +### Layout & framework + +- Project: `sdk/csharp/tests/AgentspanE2eTests/AgentspanE2eTests.csproj` + - `net10.0`, `IsTestProject=true`, `IsPackable=false` + - Framework: **xUnit** (`xunit` 2.9.3, `xunit.runner.visualstudio`, `Microsoft.NET.Test.Sdk` 17.12.0) plus **`Xunit.SkippableFact`** 1.4.13 for server-gated skipping + - References the SDK `../../src/Conductor.AI/Conductor.AI.csproj` and compiles in the shared `../../examples/Shared/Settings.cs` +- ~26 `.cs` files; **~175** `[Fact]` / `[SkippableFact]` methods total *(approximate — grep count of `[Fact]`/`[SkippableFact]`/`[Theory]` attributes; no `[Theory]` cases present)*. + +### Server gating + +- `E2eFixture.cs` (collection fixture `[CollectionDefinition("E2e")]`) does a one-time `GET {server}/health` in `InitializeAsync`. Server base is derived from `AGENTSPAN_SERVER_URL` (default `http://localhost:6767/api`, with `/api` stripped for the health probe). +- Tests call `RequireServer()` → `Skip.IfNot(ServerAvailable, …)`. **When the server is unreachable, server-dependent tests skip rather than fail**, so CI stays green without a running server. +- `E2eFixture.FetchWorkflowAsync(executionId)` fetches `GET {server}/api/workflow/{id}?includeTasks=true` for runtime-state assertions (mirrors Python's `_get_workflow`). +- `E2eHelpers.cs` provides deterministic plan-navigation helpers: `GetAgentDef`, `AllTasksFlat` (recurses loop/decision/fork tasks), `ToolNames`, `GetTool`, `GetToolType`, `GetToolCredentials`, `GuardrailNames`, `GetGuardrail`, `SubAgentNames`. + +### Test-tier convention + +Suites split tests into two tiers, called out in file-header comments: + +- `[Fact]` — pure in-process SDK tests (no server, no LLM). Inspect `ToolDef`/`Agent`/builder properties and serialization. Run in both the unit job and the e2e job. +- `[SkippableFact]` — server tests. Compile via `PlanAsync()` and assert on the compiled plan JSON, or `RunAsync()` and assert on `result.Status` / runtime task graph. Some `RunAsync` tests do invoke an LLM, but assertions are on **status / structure / tool side-effect counters**, never on judged text. + +### Coverage (suites) + +*Counts below are approximate grep tallies of `[Fact]`/`[SkippableFact]`.* + +| File | ~Tests | Covers | +|------|-------:|--------| +| `Suite1_BasicValidation.cs` | 11 | AgentDef JSON exactness: toolType, credentials, guardrailType/position/onFail/maxRetries, strategy, model, instructions, maxTurns; worker/http/credential tool types; multi-guardrail; handoff strategy; all-strategies wire values | +| `Suite2_ToolCalling.cs` | 5 | Single/multi/async tool function-body execution; tool result in execution; runtime credential-lifecycle injection | +| `Suite3_Guardrails.cs` | 4 | Output/regex/tool guardrail function-body execution; passing guardrail → agent succeeds | +| `Suite4_Termination.cs` | 8 | maxMessage / textMention / stopMessage / composed-OR termination in plan; runtime maxTurns / textMention / maxMessage stop-early; invalid-model runtime failure | +| `Suite5_Strategies.cs` | 9 | Handoff/sequential/parallel/swarm/router compile; pipeline operator → sequential; runtime sequential/parallel/handoff execution | +| `Suite6_Callbacks.cs` | 5 | before/after model callbacks fire; both fire; callback around tool calls; callbacks present in plan | +| `Suite7_Credentials.cs` | 6 | ToolDef credential props; external ToolDef compiles; local credential tool → worker type in plan | +| `Suite8_CodingAgents.cs` | 9 | Coding swarm strategy/sub-agent count; GitHub agent tools in plan; CLI-tool-with-credentials; chained pipeline (sequential + termination); local file tools execute | +| `Suite9_McpTools.cs` | 12 | MCP/http tool ToolDef props; mcp/http types in plan; mixed agent (all three tool types); MCP credential tool | +| `Suite10_CodeExecutionAndDeploy.cs` | 12 | Docker/serverless executors compile in plan; serverless POSTs code; discovered agents compile + run-by-name; local Python runtime output; local timeout kills long-running code | +| `Suite11_CliTools.cs` | 10 | CLI ToolDef props; allowed-command execution + description blocking; worker type in plan; CLI-tool-with-credentials | +| `Suite12_HttpTools.cs` | 10 | HTTP ToolDef props; http type in plan; http-tool-with-credentials; agent with only http tools has no worker tools | +| `Suite13_StatefulDomain.cs` | 13 | `Agent.Stateful` serialization; worker tool stateful flag in plan; two stateful agents compile independently; **concurrent runs have disjoint domains**; per-tool stateful propagation + domain isolation | +| `Suite14_PdfTools.cs` | 5 | PDF tool type + default schema (markdown/filename); custom name/description round-trip; PDF generation task completes | +| `Suite15_MediaTools.cs` | 5 | image/audio/video tool types in plan; multiple distinct media types; OpenAI image generation completes | +| `Suite16_PlanExecuteRefs.cs` | 2 | `Ref` pipes whole output across plan-execute steps; two refs resolve independently | +| `Suite16_Skills.cs` | 3 | Skill loads → deterministic plans/workers; skill-as-agent-tool carries worker names for domain routing; standalone skill script runs as worker tool | +| `Suite17_SdkParity.cs` | 8 | Cross-SDK parity: handoff triggers serialize, text gate, dynamic instructions resolve fresh, lifecycle callbacks (agent/tool + composable), `[AgentDef]`+`FromInstance`, worker-tuning env vars | +| `Suite18_AgentClient.cs` | 3 | `AgentClient` (renamed from `AgentHttpClient`): control-plane-only `RunAsync` (start+poll, no local workers); `ScheduleAsync` create/list/purge lifecycle | +| `Suite19_AuthHeader.cs` | 4 | `AgentAuthHandler`: mints JWT from key+secret → sends `X-Authorization` (Orkes contract) + caches; key-only treated as token (no mint); no-creds → no header. In-memory stub handlers, no server | +| `Plans_ContextTests.cs` | 8 | `Context` dataclass + serializer (no LLM) | +| `Plans_OpTests.cs` | 5 | `Op` XOR invariant — exactly one of `Args` / `Generate`, enforced at construction time | +| `ScheduleTests.cs` | 15 | Scheduling SDK: unit tests + integration (`ScheduleIntegrationTests`) — reconcile/upsert/prune/pause/resume/empty-purge/null-preserve/delete-then-get/preview-next (integration skipped unless scheduler reachable) | +| `CredentialInjectionConcurrentTest.cs` | 3 | Secret-injection contract (per `docs/design/secret-injection-contract.md §5`): counterfactual race proof + fix-verification; deterministic via `Barrier`/`ManualResetEventSlim`, no sleeps | + +> Note: the `ci.yml` comment says "101 tests across 13 suites" — that figure is **stale**; the suite has grown to ~175 tests across the files above. Flagged for a future doc/comment sync. + +### Guardrail matrix example (`90_GuardrailE2eTests`) + +- Standalone `Exe` (`Example90GuardrailE2eTests.csproj`, `net10.0`), references `Conductor.AI` and `Shared/Settings.cs`. **Not** a `dotnet test` project — it is `dotnet run`. +- Builds 27 agents = Position (Agent-OUT / Tool-INPUT / Tool-OUTPUT) × Type (Regex / LLM / Custom) × OnFail (RETRY / RAISE / FIX), runs each against a live server, and checks each via an in-program `TestRunner.Check(...)`. +- Assertions are deterministic: `expectStatus` / `expectStatusIn` (e.g. `Status.Failed` for RAISE), `expectContains` / `expectNotContains` substring checks (e.g. output must NOT contain `SECRET42`, must contain `REDACTED` after a FIX). No LLM judging — the LLM is only the agent under test. +- Prints a 27-row PASS/FAIL/SKIP table with execution IDs and `Environment.Exit(failed > 0 ? 1 : 0)`. +- Requires `AGENTSPAN_SERVER_URL`, `AGENTSPAN_LLM_MODEL`, and `OPENAI_API_KEY` (LLM guardrail cells). + +--- + +## 3. How to run locally + +Prerequisites: .NET 10 SDK; a running Agentspan server on `:6767`; `OPENAI_API_KEY` for tests that actually run an agent. Without a reachable server, `[SkippableFact]` tests skip and `[Fact]` tests still run. + +Start the server (from a built jar): + +```bash +java -jar server/conductor-agentspan-server/build/libs/agentspan-runtime.jar --server.port=6767 +``` + +Run the full e2e suite (exact command, from `sdk/csharp/`): + +```bash +dotnet test tests/AgentspanE2eTests/AgentspanE2eTests.csproj --configuration Release +``` + +Filter to one suite (matches the `ci-csharp-sdk-e2e.yml` `suite` input): + +```bash +dotnet test tests/AgentspanE2eTests/AgentspanE2eTests.csproj \ + --configuration Release \ + --filter "FullyQualifiedName~Suite1_BasicValidation" +``` + +Run **only** the in-process (no-server) unit-tier tests — this is what the `csharp-sdk-tests` CI job does (it excludes the whole e2e assembly by name): + +```bash +dotnet test Agentspan.sln \ + --configuration Release \ + --filter "FullyQualifiedName!~AgentspanE2eTests" +``` + +Run the guardrail-matrix example (standalone program, not `dotnet test`), from `sdk/csharp/`: + +```bash +export AGENTSPAN_SERVER_URL=http://localhost:6767/api +export AGENTSPAN_LLM_MODEL=openai/gpt-4o-mini +export OPENAI_API_KEY=... +dotnet run --project examples/90_GuardrailE2eTests +``` + +Environment variables (from `examples/Shared/Settings.cs`): + +| Variable | Default | Purpose | +|----------|---------|---------| +| `AGENTSPAN_SERVER_URL` | `http://localhost:6767/api` | Server endpoint (health probe strips `/api`) | +| `AGENTSPAN_LLM_MODEL` | `openai/gpt-4o-mini` | Model for agent runs | +| `OPENAI_API_KEY` | — | Required for `RunAsync`/LLM-guardrail tests | + +--- + +## 4. CI integration + +Two GitHub Actions paths exercise C# e2e. + +### `ci.yml` (gating, on push/PR) + +- **`csharp-sdk-tests`** — builds `Agentspan.sln` (Release) and runs unit-tier tests with `--filter "FullyQualifiedName!~AgentspanE2eTests"` (excludes the e2e assembly so no live server is needed). Uploads `.trx` results. +- **`csharp-e2e`** — `needs: [build-server, csharp-sdk-tests]`, 45-min timeout. Sets up Java 21 + .NET 10, downloads the prebuilt `server-jar` artifact, starts it on `:6767` and polls `/health`, then runs: + ```bash + dotnet test tests/AgentspanE2eTests/AgentspanE2eTests.csproj \ + --configuration Release \ + --logger "console;verbosity=normal" \ + --logger "trx;LogFileName=csharp-e2e.trx" + ``` + Env: `OPENAI_API_KEY`, `ANTHROPIC_API_KEY`, `AGENTSPAN_SERVER_URL=http://localhost:6767/api`, `AGENTSPAN_LLM_MODEL=openai/gpt-4o-mini`. Mirrors the python/typescript/java e2e jobs (same `build-server` prerequisite, same `:6767` startup). Uploads `.trx` results. *(The header comment "101 tests across 13 suites" is stale; see §2.)* + +### `ci-csharp-sdk-e2e.yml` (manual) + +- `workflow_dispatch` only, with inputs `model` (default `anthropic/claude-sonnet-4-6`) and `suite` (filter, e.g. `Suite1_BasicValidation`). +- Builds the server JAR in-line (`./gradlew bootJar -x test -q` in `server/`), sets up Java 21 + .NET 10, starts `agentspan-runtime.jar --server.port=6767`, polls `/health` (30 × 2s), then runs `dotnet test tests/AgentspanE2eTests/AgentspanE2eTests.csproj --configuration Release` with an optional `--filter FullyQualifiedName~`. +- Env: `AGENTSPAN_SERVER_URL`, `AGENTSPAN_LLM_MODEL` (from input), `OPENAI_API_KEY`. Stops the server in an `always()` step. + +The guardrail-matrix example (`90_GuardrailE2eTests`) is not wired into either CI workflow — it is a manual/local diagnostic that exits non-zero on failure. + +--- + +## Uncertainties / flags + +- Test counts are **approximate** grep tallies of `[Fact]`/`[SkippableFact]` attributes; the true count of executed test cases may differ slightly (no parameterized `[Theory]` cases were found). +- `ci.yml`'s "101 tests across 13 suites" comment is out of date relative to the current ~175 tests / ~26 files. +- Confirmed absence of a C# examples-quality / LLM-judge validation framework as of this writing (no `validation/` dir, no `runs.toml` under `sdk/csharp/`). diff --git a/design/validation/java-validation.md b/design/validation/java-validation.md new file mode 100644 index 000000000..0bcc6e307 --- /dev/null +++ b/design/validation/java-validation.md @@ -0,0 +1,160 @@ +# Java SDK — Validation & E2E + +**Status:** Created 2026-06-26 + +**Scope:** How the Java SDK is validated end-to-end. Unlike the Python and TypeScript SDKs — which add a TOML-driven, LLM-judged *examples-quality* validation framework on top of their deterministic suites — **Java validation is the deterministic e2e suite only**: a set of JUnit 5 `@Tag("e2e")` test classes under `sdk/java/e2e/`, run against a live Agentspan server, locally and in CI. There is **no** separate examples-quality / LLM-judge harness for Java. Per `CLAUDE.md`, the e2e suites are deterministic: assertions are on compiled-workflow JSON structure, workflow task status, or in-process side effects (e.g. an `AtomicBoolean`/`AtomicReference` set inside a tool body) — never on LLM output text, except where the task is itself an eval. + +Related: +- Methodology / framework spec: [`README.md`](README.md) (Python-centric; the LLM-judge orchestrator described there does **not** apply to Java) +- Java implementation design: [`../sdk-design/languages/java-implementation.md`](../sdk-design/languages/java-implementation.md) +- Cross-SDK design: [`../sdk-design.md`](../sdk-design.md) + +--- + +## 1. Overview + +Java validation = **deterministic e2e suites**, run locally and in CI against a real server with real LLM calls. No mocks. The principle (shared with the other SDKs): exercise real agents through the real server, but make every *assertion* deterministic so a green run actually proves something. + +Three deterministic assertion shapes are used across the suites: +- **Plan-level (no LLM):** call `runtime.plan()` / `/agent/compile`, then assert on the compiled Conductor `workflowDef` / `agentDef` JSON. Pure structure checks (serialization round-trips, injected worker tools, sub-workflow shape). +- **Runtime side-effect:** run a real agent, but assert on a side effect captured inside the tool body (`AtomicBoolean` flag, `AtomicReference` captured argument). The LLM drives the agent; the recorded side effect is the signal. +- **Runtime workflow-status:** run a real agent and assert on server-side workflow/task state (status `COMPLETED`/`FAILED`/`TERMINATED`, task types like `SUB_WORKFLOW`/`FORK_JOIN`, terminal-fail behavior). + +Every suite is written **counterfactually** — each assertion (or a companion contrast test) is designed to fail if the feature under test is broken or silently dropped. + +There is no Java equivalent of the `e2e-orchestrator.sh` / TOML runs / LLM judge / HTML report pipeline. Java relies on Gradle's JUnit runner and Gradle's own test reports. + +--- + +## 2. E2E test suite (`sdk/java/e2e/`) + +### Layout & framework + +- **Framework:** JUnit 5 (`org.junit.jupiter:junit-jupiter`, v5.11.0 — see `sdk/java/build.gradle`). +- **No separate Gradle module.** There is **no** `sdk/java/e2e/build.gradle`. The `e2e/` directory is wired into the SDK's `test` source set: + ```gradle + sourceSets { test { java { srcDirs += file('e2e') } } } + ``` + The e2e classes live in the **default (unnamed) package**. +- **Gating by tag:** every suite class carries `@Tag("e2e")` (25 of the 26 `.java` files; `BaseTest` is the abstract base and is untagged). The default `test` task excludes tag `e2e` unless `-Pe2e` is passed: + ```gradle + test { useJUnitPlatform { if (!project.hasProperty('e2e')) excludeTags 'e2e' } } + ``` + So `./gradlew test` = unit only (fast, no server); `./gradlew test -Pe2e` = unit + e2e. +- **Parallelism:** under `-Pe2e`, `maxParallelForks = 3` (e2e is I/O-bound — LLM/docker — and suites use unique agent/task names, so they fork safely). + +### `BaseTest.java` (shared harness) + +Abstract base for all suites. Provides: +- **Server health gate (`@BeforeAll`):** GETs `BASE_URL/health`, parses `healthy`, and `assumeTrue(...)`. If the server is down/unhealthy, **all tests in the class are skipped** (not failed). Caveat: a fully-skipped run looks green but proves nothing — always confirm tests actually ran. +- **Config from env:** `AGENTSPAN_SERVER_URL` (default `http://localhost:6767/api`), `AGENTSPAN_LLM_MODEL` (default `openai/gpt-4o-mini`). `BASE_URL` is `SERVER_URL` with `/api` stripped. +- **Workflow fetch:** `getWorkflow(executionId)` → `GET BASE_URL/api/workflow/` for server-side task/status assertions. +- **Plan navigation:** `getAgentDef(CompileResponse)` walks `workflowDef → metadata → agentDef`; `allTasksFlat(workflowDef)` recursively flattens nested tasks (DO_WHILE `loopOver`, SWITCH `decisionCases`/`defaultCase`, FORK_JOIN `forkTasks`). + +### What's covered + +26 `.java` files in `sdk/java/e2e/` (25 suites + `BaseTest`), **~162 `@Test` methods total** (raw `grep -c "@Test"` across the directory — *approximate*). Note: the class-javadoc "Suite N" labels do not always match the filename numbering (they're carried over from the Python suite mapping); the table below uses the filenames. + +| File | Mode | Covers | +|------|------|--------| +| `Suite1BasicValidation` | plan | Basic `plan()` structural assertions on the compiled workflow JSON; counterfactual. | +| `Suite2ToolCalling` | runtime side-effect | Tools actually invoked during execution (asserts via `AtomicBoolean` in tool body, not LLM text / task names). | +| `Suite2ToolCallingCredentials` | runtime status | Runtime credential lifecycle: no-cred → terminal-fail; set/update via API seen at runtime via `ctx.getCredential()`; delete → terminal-fail again. Mirrors the Python/.NET/TS canonical contract. Also verifies env vars are not used as a fallback. | +| `Suite3CliTools` | plan + runtime | `CliConfig` serialization + injected `{name}_run_command` worker; local command execution and whitelist enforcement (executed by the SDK's `CliCommandExecutor`, not the server). | +| `Suite4McpTools` | plan | Server-side MCP tool (`toolType="mcp"`) serialization. | +| `Suite5HttpTools` | plan | Server-side HTTP tool (`toolType="http"`) serialization. | +| `Suite6PdfTools` | plan | Server-side PDF tool (`toolType="generate_pdf"`) serialization. | +| `Suite7MediaTools` | plan | Server-side media tools (image/audio/video/pdf) — `llmProvider`/`model`/`taskType` serialization. | +| `Suite8Guardrails` | runtime status | Guardrails fire at runtime (custom function guardrail always returns `passed=false`) → agent FAILED/TERMINATED. | +| `Suite8bGuardrailsExtended` | plan + runtime | Agent/tool-level guardrail serialization; tool body not blocked by agent OUTPUT guardrail; INPUT max-retries escalation. | +| `Suite9Handoffs` | plan + runtime | SEQUENTIAL / PARALLEL / HANDOFF / PIPE (`.then()`) — correct workflow task types (SUB_WORKFLOW, FORK_JOIN) and completion. | +| `Suite10CodeExecution` | plan + runtime | `localCodeExecution` serialization, injected `execute_code` tool, code actually runs, timeout enforced. | +| `Suite11LangChain4j` | plan + runtime | `LangChain4jAgent` bridge — detection/tagging, tool extraction (JSON Schema), compile, runtime. | +| `Suite11bOpenAIAgent` | plan + runtime | `OpenAIAgent` bridge — same shape as LangChain4j; server routes `framework="openai"` via `OpenAINormalizer`. | +| `Suite12HandoffApprove` | runtime status | HANDOFF + HITL: approval-required tool on a sub-agent → HUMAN task in the sub-execution; targeted `AgentStream.approve(event)`. | +| `Suite12TerminationGates` | runtime status | Termination conditions actually stop execution before `max_turns`. | +| `Suite13Callbacks` | plan + runtime | `CallbackHandler` positions serialize into `agentDef.callbacks`; callbacks don't break execution; multiple handlers. | +| `Suite14StatefulDomain` | plan | `Agent.stateful(...)` propagation into agentDef / tools / swarm sub-agents. | +| `Suite15Skills` | plan | `Skill.skill(path, model)` loads a `SKILL.md` directory as an agent (`framework="skill"`). | +| `Suite16Synthesize` | plan | `synthesize` flag structural effect. | +| `Suite17ConfigSerialization` | plan | Broad serialization round-trip: stateful, baseUrl, TextGate, callbacks, termination, Regex/LLM guardrails, OnCondition handoff, media/wait/human tools, `deploy()`, and parity fields (`reasoningEffort`, `contextWindowBudget`, `maskedFields`, `memory`). Largest suite (~20 tests). | +| `Suite18ToolTypes` | runtime side-effect | Tool-arg coercion pipeline (server task → SDK coerce → method invoke) for `java.time` types; captures the actual received argument via `AtomicReference`. | +| `Suite19ManualStrategy` | runtime status | MANUAL strategy end-to-end: pauses at `pick_agent` HUMAN task, responds with the second agent, asserts the selected sub-workflow ran (catches a broken name→index mapping). | +| `PlanExecuteTest` | runtime | PLAN_EXECUTE strategy end-to-end; assertions are algorithmic (file existence, word counts). | +| `SuiteHttpApi404` | runtime | Live 404 round-trip: `AgentClient` maps a server 404 to `AgentNotFoundException` (not the generic `AgentAPIException`). | + +### Server launch & known gotchas + +The suites do **not** start the server themselves — they expect one running at `AGENTSPAN_SERVER_URL` and skip if it's not. Launch it yourself (SQLite-backed, no Postgres needed; reads `OPENAI_API_KEY`/`ANTHROPIC_API_KEY` from env): + +```bash +cd server +java -jar conductor-agentspan-server/build/libs/agentspan-runtime.jar --server.port=6767 +# healthy in ~3-10s; rm agent-runtime.db* for a clean DB +``` + +> **Known gotchas (verified, local runs):** +> - **Stale-jar `NoClassDefFoundError`:** a prebuilt `agentspan-runtime.jar` can be internally inconsistent — server classes in `BOOT-INF/classes` compiled against a newer API than the bundled `BOOT-INF/lib/conductor-agentspan-*.jar`. Symptom: `/agent/start` (runtime) works but `/agent/compile` (plan tests) and human-task paths 500 with errors like `CompileResponse$CompileResponseBuilder` / `HumanTaskBuilder`. Fix: rebuild — `cd server && ./gradlew :conductor-agentspan-server:bootJar` — then restart. +> - **Default-package discovery:** plain `--tests 'Suite*'` does **not** pick up the default-package classes. Use exact names: `--tests Suite9Handoffs` (or `Suite9Handoffs.`). +> - **Long-run false timeouts:** a single-JVM full run of all suites degrades — worker polling pressure (per-`AgentRuntime` worker threads, 100ms interval) saturates the client pool over hours, so `waitForResult` can miss completions and report `Agent timed out after 600000ms` even though the workflow COMPLETED server-side. Prefer fresh-server batches over one giant run; re-running failed tests on a fresh server passes them. +> - **One Gradle daemon at a time:** concurrent `./gradlew` launches caused daemon contention serving stale results; `./gradlew --stop` to reset. + +--- + +## 3. How to run locally + +From `sdk/java/`: + +```bash +# Unit tests only (no server, e2e excluded by default): +./gradlew test + +# Unit + e2e (requires a live server at AGENTSPAN_SERVER_URL): +./gradlew test -Pe2e + +# A single suite (exact class name — default-package, so no wildcard match): +./gradlew test -Pe2e --tests Suite9Handoffs + +# A single method: +./gradlew test -Pe2e --tests Suite9Handoffs. + +# Override server / model: +./gradlew test -Pe2e \ + -DAGENTSPAN_SERVER_URL=http://localhost:6767/api \ + -DAGENTSPAN_LLM_MODEL=openai/gpt-4o-mini +``` + +Notes: +- Config is read from **env vars** by `BaseTest` (`AGENTSPAN_SERVER_URL`, `AGENTSPAN_LLM_MODEL`); the `-D` system properties above match how CI passes them in the dispatch workflow (see §4) — set them as env vars if `-D` is not picked up in your shell. +- Reports: Gradle writes HTML/XML to `sdk/java/build/reports/tests/test/`. +- Coverage: `./gradlew jacocoTestReport` aggregates whatever ran in `test` (unit-only by default, or unit+e2e under `-Pe2e`). `-PignoreTestFailures` lets a flaky e2e not abort the JaCoCo report. + +--- + +## 4. CI integration + +Two workflows run the Java e2e suites; both build/obtain the server JAR, start it on **:6767**, then run `./gradlew test -Pe2e`. + +### `ci.yml` — `java-e2e` job (primary, runs in the main pipeline) + +`needs: [build-server, java-sdk-tests]`, `timeout-minutes: 45`. Sequence: +1. **Server JAR** is built once by the shared `build-server` job (`./gradlew bootJar -PbuildUI=true -x test` in `server/`) and uploaded as the `server-jar` artifact. `java-e2e` downloads it to `server/conductor-agentspan-server/build/libs/`. +2. **mcp-testkit**: `pip install mcp-testkit`, then `mcp-testkit --transport http --port 3001 &` (test infra for the tool suites — not an SDK dependency). +3. **Start server**: `java -jar .../agentspan-runtime.jar --server.port=6767 &`, then poll `http://localhost:6767/health` (up to 30×2s). +4. **Run**: `./gradlew test -Pe2e` in `sdk/java`. +5. **Artifacts**: `sdk/java/build/reports/tests/test/` uploaded as `java-e2e-results` (always, 14-day retention). + +Env: `AGENTSPAN_SERVER_URL=http://localhost:6767/api`, `AGENTSPAN_LLM_MODEL=openai/gpt-4o-mini`, plus `OPENAI_API_KEY` / `ANTHROPIC_API_KEY` from secrets. + +The separate **`java-sdk-tests`** job (`./gradlew test :spring:test`) runs the SDK + Spring auto-config **unit** tests only — fast, no server (e2e excluded by default). + +### `ci-java-sdk-e2e.yml` — standalone, `workflow_dispatch` only + +Manual trigger with optional `model` (default `anthropic/claude-sonnet-4-6`) and `suite` filter inputs. It builds the server JAR inline (`./gradlew bootJar -x test -q` in `server/`) rather than downloading the shared artifact, starts it on :6767 with the same health poll, and runs: +```bash +./gradlew test -Pe2e \ + -DAGENTSPAN_SERVER_URL=$AGENTSPAN_SERVER_URL \ + -DAGENTSPAN_LLM_MODEL=$AGENTSPAN_LLM_MODEL \ + [--tests '*.*'] # when the suite input is set +``` +Note: this dispatch workflow does not start mcp-testkit, so tool suites that need it will skip/fail there — prefer the `ci.yml` `java-e2e` job for a full run. *(Flagged: the `--tests '*.*'` wildcard pattern here may not match default-package classes the way exact `--tests Suite9Handoffs` does — see the discovery gotcha in §2.)* diff --git a/design/validation/python-validation.md b/design/validation/python-validation.md new file mode 100644 index 000000000..a60c8875b --- /dev/null +++ b/design/validation/python-validation.md @@ -0,0 +1,269 @@ +# Python SDK — Validation & E2E + +**Status:** Refreshed 2026-06-26 + +**Scope:** Canonical reference for the two validation surfaces of the Python SDK (PyPI `conductor-agent-sdk`, import namespace `conductor.ai.agents`): (1) the **deterministic E2E suites** under `sdk/python/e2e/`, which exercise real agents against a real server + real services with deterministic assertions; and (2) the **examples-quality validation framework** under `sdk/python/validation/`, which runs every SDK example across multiple models concurrently and scores output quality with an LLM judge. Cross-links: methodology / cross-cutting harness [`README.md`](README.md); implementation [`../sdk-design/languages/python-implementation.md`](../sdk-design/languages/python-implementation.md); SDK contract (env vars, CLI binary) [`../sdk-design.md`](../sdk-design.md). + +--- + +## 1. Overview + +| Surface | Location | Purpose | Judging | +|---------|----------|---------|---------| +| Deterministic E2E suites | `sdk/python/e2e/` | Verify feature correctness end-to-end (real agents, real server, real CLI/MCP/HTTP services, no mocks) | Deterministic assertions (JSON-path / workflow-task / Conductor-API checks). LLM only where the thing under test *is* output quality (e.g. one semantic judge call in Suite 1) — see [CLAUDE.md](../../CLAUDE.md) | +| Examples-quality framework | `sdk/python/validation/` | Run every published example across multiple models, compare outputs, surface quality regressions | LLM-as-judge by design (1–5 scoring + baseline comparison) | + +The two are complementary: the E2E suites answer *"does the feature work?"* deterministically; the validation framework answers *"is the example's output good across models?"*. Both are driven by the cross-SDK orchestrator (`e2e/orchestrator.sh`) and CI (`.github/workflows/ci.yml`). + +Both surfaces honour the SDK's intentional contracts from [`../sdk-design.md`](../sdk-design.md): the `AGENTSPAN_*` environment variables, the `agentspan` CLI binary (PyPI console-script entry point `conductor.ai.cli:main`), and `AGENTSPAN_AUTO_START_SERVER=false` to keep tests pointed at the orchestrator-managed server. + +--- + +## 2. Deterministic E2E suites (`sdk/python/e2e/`) + +### 2.1 Principles + +- **No mocks** — real agents, real Conductor server, real CLI, real services (mcp-testkit for MCP/HTTP, Docker/Jupyter for code execution, provider APIs for media). +- **Deterministic assertions** — checks are made against compiled workflow JSON (`plan()`), individual workflow task status/output via the server REST API, or Conductor control-plane API state. LLM execution is used where the *behaviour* requires it (tool-calling, handoffs, guardrail runtime policies), but assertions target structural/observable facts, not free-text equality. The only LLM-as-judge call is the single semantic check in Suite 1 — consistent with the [CLAUDE.md](../../CLAUDE.md) rule (no LLM judging except when judging quality). +- **Credentials via CLI only** — managed exclusively through `agentspan credentials set/delete/list`; the SDK must never read credential values from env vars (Suite 2 explicitly asserts this isolation). + +### 2.2 Harness (`conftest.py`) + +Configuration is read from env vars exported by the orchestrator: + +| Var | Default | Purpose | +|-----|---------|---------| +| `AGENTSPAN_SERVER_URL` | `http://localhost:6767/api` | Server API URL (health, workflow inspection) | +| `AGENTSPAN_CLI_PATH` | `agentspan` | Absolute path to the built CLI binary used for credential ops | +| `MCP_TESTKIT_URL` | `http://localhost:3001` | mcp-testkit HTTP/MCP endpoint for tool suites | +| `AGENTSPAN_LLM_MODEL` | `openai/gpt-4o-mini` | Model for suites that execute agents | + +Fixtures and behaviour: + +- `verify_server` (session, autouse) — polls `BASE_URL/health`; **skips** the whole session if the server is unreachable. +- `runtime` (module) — a shared `AgentRuntime` (`from conductor.ai.agents import AgentRuntime`). +- `model`, `mcp_url`, `cli_credentials` (session) — model string, mcp-testkit URL, and a `CredentialsCLI` helper wrapping the `agentspan` binary (`set`/`delete`/`list`; tolerant of "not found" on cleanup). The helper strips the `/api` suffix because the CLI appends it internally. +- `get_workflow()` / `get_task_by_name()` — REST helpers to fetch a completed workflow and locate tasks by `referenceTaskName`, used for deterministic per-task assertions. +- `conftest.py` sets `AGENTSPAN_AUTO_START_SERVER=false` at import time to prevent the runtime from launching a second server. +- **Auto-retry**: `pytest_collection_modifyitems` attaches `flaky(reruns=2, reruns_delay=5)` to every `e2e`-marked test. These suites drive a real server + real LLM, so individual tests flake on transient latency (workflow still `RUNNING` at timeout, a tool-call batch not returning, LLM phrasing variance). Two reruns let a one-off flake recover while a genuinely broken test still fails all three attempts. No-op unless `pytest-rerunfailures` (dev extra) is installed. + +### 2.3 Suite layout + +Suites 1–16 and 20–24 (17–19 do not exist). All carry `pytest.mark.e2e`; several add `pytest.mark.timeout`, `skipif` (Docker/Jupyter/provider keys), or `xfail`. + +| Suite | File | Validates | Determinism | +|-------|------|-----------|-------------| +| 1 | `test_suite1_basic_validation.py` | `plan()` compilation — tools, guardrails, credentials, sub-agents, all 8 strategies, kitchen-sink agent reflected in workflow JSON | Deterministic (plan-only; + 1 semantic judge call) | +| 2 | `test_suite2_tool_calling.py` | Credential lifecycle: missing → env-isolation → add (CLI) → update; server injects creds into tool execution | LLM exec; deterministic task-output assertions | +| 3 | `test_suite3_cli_tools.py` | CLI tool credential isolation, command-whitelist enforcement | LLM exec; deterministic | +| 4 | `test_suite4_mcp_tools.py` | MCP tool discovery + execution, unauth → auth lifecycle (mcp-testkit) | LLM exec; deterministic tool outputs | +| 5 | `test_suite5_http_tools.py` | HTTP/OpenAPI tool discovery + execution, auth lifecycle, external-OpenAPI compile | LLM exec; deterministic tool outputs | +| 6 | `test_suite6_pdf_tools.py` | PDF generation from markdown, markitdown round-trip content survival | LLM gen; deterministic content checks | +| 7 | `test_suite7_media_tools.py` | Image (DALL·E, Gemini) + audio (TTS) generation; `skipif` no keys, one `xfail` | LLM/provider exec | +| 8 | `test_suite8_guardrails.py` | Guardrail compilation (types/positions/`on_fail`); runtime block/retry/fix/escalation | Deterministic compile + LLM runtime policies | +| 9 | `test_suite9_handoffs.py` | All 8 multi-agent strategies compile + execute; `>>` operator | LLM exec; deterministic tool outputs | +| 10 | `test_suite10_code_execution.py` | Local/Bash, timeout, language restriction, Docker isolation, Jupyter stateful | Deterministic config + LLM exec (`skipif` Docker/Jupyter) | +| 11 | `test_suite11_langgraph.py` | LangGraph detection/serialization (full/graph-structure/passthrough), schema, compile, runtime | Deterministic serialization + LLM runtime | +| 12 | `test_suite12_termination_gates.py` | `TextMentionTermination`, `MaxMessageTermination`, `TextGate` SWITCH compile, invalid-model rejection | Deterministic gate wiring + LLM runtime | +| 13 | `test_suite13_callbacks.py` | `CallbackHandler` compile (before/after tool/model/agent); runtime as worker tasks | Deterministic compile + LLM runtime | +| 14 | `test_suite14_stateful_domain.py` | Stateful agent domain propagation (tool + `stop_when` + swarm handoff + concurrent isolation); regression: non-stateful has no domain | LLM exec; deterministic via Conductor task domains | +| 15 | `test_suite15_skills.py` | Skill load/serialize, nested in `agent_tool`, script discovery, param injection, worker creation, plan compile | Deterministic load/serialize + LLM skill exec | +| 16 | `test_suite16_cli_skills.py` | CLI skill register/list/get/pull/delete; load/serve/run; script-worker polling; dep pinning | LLM run; deterministic script-worker checks | +| 20 | `test_suite20_plan_execute.py` | `PLAN_EXECUTE`: planner sub-agent, plan compile + execute, Refs across steps, PAC whitelist | LLM planner; deterministic ref/whitelist + wire-path checks | +| 21 | `test_suite21_scheduling.py` | Schedule create/reconcile/pause/resume/delete, `preview_next`, `run_now`, tri-state | Deterministic (Conductor scheduler API; no LLM) | +| 22 | `test_suite22_ocg.py` | OCG multi-instance binding isolation (per-tenant stub routing) | LLM exec; deterministic via HTTP request recording | +| 23 | `test_suite23_from_instance_and_event_hitl.py` | Event-targeted HITL (approve/reject/respond on streamed event); `Agent.from_instance` resolution/wiring | Deterministic targeting/wire + plan compile | +| 24 | `test_suite24_agent_client.py` | `AgentClient.run`/`.start`/`.join`/`.schedule` reconcile + list; surface consistency | Deterministic control-plane + LLM run on tool-less agent | + +### 2.4 Serial / stateful suites + +Suites that mutate shared server state run serially via `pytest.mark.xdist_group`, so parallel workers (`-n`) keep them on one worker: + +- `xdist_group("credentials")` — Suites 2, 3, 4, 5 (credential set/delete is global server state). +- `xdist_group("cli-skills")` — Suite 16. +- `xdist_group("ocg")` — Suite 22. + +CI runs with `--dist=loadgroup` so these groups are honoured. Suites mutating state use `try/finally` cleanup to avoid leaking state on failure. + +### 2.5 How to run + +Driven by the cross-SDK orchestrator `e2e/orchestrator.sh` (Python is the default `--sdk`). It builds the server JAR + CLI, `uv sync`s the SDK, installs mcp-testkit, starts services, health-checks, runs pytest, then renders the HTML report: + +```bash +./e2e/orchestrator.sh # build + start + run all Python suites (-j 1) +./e2e/orchestrator.sh -j 4 # 4 parallel xdist workers +./e2e/orchestrator.sh --suite suite1 # pytest -k suite1 +./e2e/orchestrator.sh --no-build --no-start # services already running +``` + +The orchestrator exports `AGENTSPAN_SERVER_URL=http://localhost:6767/api`, `AGENTSPAN_CLI_PATH=/cli/agentspan`, `MCP_TESTKIT_URL=http://localhost:3001`, `AGENTSPAN_AUTO_START_SERVER=false`, then invokes pytest from `sdk/python` and writes `e2e-results/junit.xml` + `e2e-results/report.html`. + +To run pytest directly against an already-running server (skips build/service management): + +```bash +cd sdk/python +export AGENTSPAN_SERVER_URL=http://localhost:6767/api +export AGENTSPAN_CLI_PATH="$PWD/../../cli/agentspan" +export MCP_TESTKIT_URL=http://localhost:3001 +uv run pytest e2e/ -v --tb=short -n 3 --dist=loadgroup +``` + +> Note: there is no `e2e-orchestrator.sh` at the repo root — the orchestrator lives at `e2e/orchestrator.sh`. + +### 2.6 HTML report (`report_generator.py`) + +Post-processes the pytest junit XML into a single self-contained HTML file: parses the `testsuites/testsuite` structure, groups tests by suite file, derives human-readable suite names (`test_suite1_basic_validation` → "Suite 1: Basic Validation"), and renders collapsible per-suite sections with a pass/fail/skip/error summary, color-coded statuses, and expandable error tracebacks (inline CSS, dark mode, no external deps). Invoked as `python e2e/report_generator.py `. + +### 2.7 CI + +The `python-e2e` job in `.github/workflows/ci.yml` (needs `build-server` + `python-unit-tests`, 45-min timeout): downloads the server JAR artifact, builds the CLI (`go build -o agentspan .`), `uv sync --extra dev --extra testing`, installs + starts mcp-testkit, starts the server and polls `/health`, then: + +```bash +uv run pytest e2e/ -v --tb=short \ + --junitxml=../../e2e-results/junit.xml \ + --reruns 2 --reruns-delay 5 \ + -n 3 --dist=loadgroup +``` + +It always generates and uploads `e2e-results/` (junit + HTML), retained 14 days. `OPENAI_API_KEY` / `ANTHROPIC_API_KEY` come from secrets; env vars match the harness contract (`AGENTSPAN_SERVER_URL`, `AGENTSPAN_CLI_PATH`, `AGENTSPAN_AUTO_START_SERVER=false`). + +--- + +## 3. Examples-quality validation framework (`sdk/python/validation/`) + +Runs every published SDK example across one-or-more models concurrently, parses each example's output, and (optionally) scores quality with an LLM judge against a baseline model. Install with `uv sync --extra validation`. + +### 3.1 Architecture + +TOML config defines named runs (one model each). Runs execute concurrently via the orchestrator. A multi-run LLM judge compares outputs against a baseline. + +``` +runs.toml → load_toml_config() → resolve_runs() → run_all() + │ + ThreadPoolExecutor + ┌───────┼───────┐ + ▼ ▼ ▼ + run_single run_single run_single + (sub-dir) (sub-dir) (sub-dir) + │ │ │ + └───────┼───────┘ + ▼ + judge_across_runs() + (judge/ sub-dir) +``` + +### 3.2 Config structure + +```toml +[defaults] +timeout = 300 # per-example timeout (seconds) +parallel = true # run examples within a run concurrently +max_workers = 8 # max concurrent examples per run +retries = 0 +server_url = "http://localhost:6767/api" + +[env] # global env, applied to all runs (shell wins via setdefault) +# AGENTSPAN_AUTH_KEY = "" + +[judge] +baseline_run = "openai" +model = "anthropic/claude-sonnet-4-6" +max_output_chars = 3000 +max_tokens = 300 +rate_limit = 0.5 # seconds between judge calls +max_calls = 0 # 0 = unlimited + +[runs.openai] +group = "OPENAI_EXAMPLES" +model = "openai/gpt-4o" + +[runs.anthropic] +group = "OPENAI_EXAMPLES" +model = "anthropic/claude-sonnet-4-20250514" + +[runs.anthropic.env] # per-run env, never touches os.environ +# ANTHROPIC_API_KEY = "" +``` + +`[defaults]` values merge into every `[runs.*]` (run-level overrides win). Live config is `validation/runs.toml` (gitignored); template is `validation/runs.toml.example`. Run-config keys: `name` (auto), `group`, `model`, `secondary_model`, `parallel`, `max_workers`, `timeout`, `retries`, `server_url`. + +### 3.3 Example groups + +Groups are defined in `validation/groups.py` and selected per run via `group = "NAME"`. Notable groups: `PASSING_EXAMPLES` (the core SDK examples), `SMOKE_TEST` (small fast subset), `OPENAI_EXAMPLES` / `ADK_EXAMPLES` / `LANGGRAPH_EXAMPLES` / `LANGCHAIN_EXAMPLES` (per-framework, gated on dep availability via `SUBDIRS` in `config.py`), `HITL_EXAMPLES` (driven by the `HITL_STDIN` map), `SLOW_EXAMPLES`, and `KNOWN_FAILURES`. Discovery scans `examples/` plus framework subdirs and skips subdirs whose framework dependency is not installed. `--list-groups` prints all groups. + +### 3.4 Execution + +Each `run_single()`: +1. Discovers examples for the run's group. +2. Starts the server pool. +3. Calls `run_examples()` — single model, concurrent examples via `ThreadPoolExecutor` (`max_workers`). +4. Writes `run_results.json`, `outputs/`, `meta.json`, `report.json` into the run sub-dir. + +`run_example()` runs each example as a subprocess (`python