diff --git a/plugin/README.md b/plugin/README.md index b8111e5..174eb1a 100644 --- a/plugin/README.md +++ b/plugin/README.md @@ -13,6 +13,7 @@ The plugin and the [DAB template](../template/) share the same scaffold contract | **`agentops-stacks`** | `scaffold a new agentops project` | Scaffolds a new multi-agent LangGraph project (per-agent Databricks Apps, shared components, UC schema and volume, MLflow experiments, CI/CD wiring). One-time use at project start. | | **`agentops-lifecycle`** | `walk me through the agentops lifecycle` | Guides an existing scaffold through the complete Single-Account Single-Agent lifecycle — data prep, agent dev, eval gate, SME calibration, CI/CD promotion, batch eval baseline, and production monitoring. 10 steps across dev → staging → prod. | | **`add-agent`** | `add agent`, `new agent`, `create another agent` | Adds a new agent to an existing project — copies an existing agent as a template and wires it into `databricks.yml` and the manifest. | +| **`add-supervisor`** | `add supervisor`, `add a router`, `orchestrate my agents` | Adds a supervisor that routes across the project's agents. Runs a Selection Matrix to pick the best-fit pattern — custom LangGraph (GA, default), Supervisor API (Beta), or Agent Bricks MAS (managed tile) — then scaffolds it into `databricks.yml` and the manifest. | ### Commands @@ -21,6 +22,7 @@ The plugin and the [DAB template](../template/) share the same scaffold contract | `/init-agentops-stacks` | `agentops-stacks` | | `/agentops-lifecycle` | `agentops-lifecycle` | | `/add-agent` | `add-agent` | +| `/add-supervisor` | `add-supervisor` | ### Installers @@ -98,17 +100,24 @@ plugin/ ├── commands/ │ ├── init-agentops-stacks.md # scaffold command │ ├── agentops-lifecycle.md # lifecycle command +│ ├── add-supervisor.md # add-supervisor command │ └── common-issues.md # troubleshooting reference └── skills/ ├── install_skills.sh # local + Genie upload installer ├── install_genie_code_skills.py # in-workspace notebook installer ├── agentops-stacks/ │ ├── SKILL.md # scaffold skill (5-phase input collection) - │ └── reference/ # post-scaffold, Genie Code, common issues docs + │ ├── reference/ # post-scaffold, Genie Code, common issues docs + │ └── scripts/ + │ ├── add_agent.py # /add-agent engine + │ ├── add_supervisor.py # /add-supervisor engine (3 patterns) + │ └── supervisor_templates/ # supervisor graph/job/notebook templates ├── agentops-lifecycle/ │ └── SKILL.md # lifecycle skill (10-step dev→prod guide) - └── add-agent/ - └── SKILL.md # add-agent skill (wires new agent into existing project) + ├── add-agent/ + │ └── SKILL.md # add-agent skill (wires new agent into existing project) + └── add-supervisor/ + └── SKILL.md # add-supervisor skill (routes across agents; best-fit pattern) ``` Each skill is a single `SKILL.md`. There's no Python renderer and no vendored diff --git a/plugin/commands/add-supervisor.md b/plugin/commands/add-supervisor.md new file mode 100644 index 0000000..d1eb889 --- /dev/null +++ b/plugin/commands/add-supervisor.md @@ -0,0 +1,28 @@ +--- +description: > + Add a supervisor agent that routes across your project's agents. Picks the + best-fit pattern — custom LangGraph, Supervisor API, or Agent Bricks MAS — via + a decision matrix, then scaffolds it into databricks.yml and the manifest. +--- + +Use the `add-supervisor` skill. + +This command adds a supervisor to an existing AgentOps Stacks project. A +supervisor routes user queries across your agents (and managed sub-agents like +Genie spaces or Knowledge Assistants). + +**Prerequisite:** `.agentops-stacks/manifest.yml` and at least one agent under +`src/agents/` must exist. Run `/add-agent` first if you only have one agent and +want the supervisor to route between several. + +The skill runs a Selection Matrix to choose among three patterns: + +- **custom** (GA, default) — a hand-written LangGraph supervisor, served as a + Databricks App, fully declared in `databricks.yml`, gated by the CI eval loop. +- **supervisor_api** (Beta) — the Databricks-managed loop wrapped in a + declarable App; minimal code, per-request model choice. +- **agent_bricks_mas** (managed tile) — no-code; not a DAB resource, so it's + provisioned by a bundle-declared bootstrap job and consumed as an endpoint. + +Defer to the skill's SKILL.md for the full decision matrix, per-pattern +scaffolding behavior, security posture, and next steps. diff --git a/plugin/skills/add-supervisor/SKILL.md b/plugin/skills/add-supervisor/SKILL.md new file mode 100644 index 0000000..c23b720 --- /dev/null +++ b/plugin/skills/add-supervisor/SKILL.md @@ -0,0 +1,157 @@ +--- +name: add-supervisor +description: Add a supervisor agent that routes across an AgentOps Stacks project's agents. Selects the best-fit supervisor pattern — custom LangGraph, Supervisor API, or Agent Bricks MAS — using a decision matrix, then scaffolds it into databricks.yml and the manifest. Triggers on "add supervisor", "add a router", "orchestrate my agents", "multi-agent supervisor", "route between agents". +--- + +# add-supervisor — Add a Supervisor to a Project + +Adds a supervisor agent to an existing AgentOps Stacks project. A supervisor +routes user queries across the project's agents (and other managed sub-agents +like Genie spaces or Knowledge Assistants). + +There are three supervisor patterns. They are **not interchangeable** — they +differ in who owns the routing loop and what lands in the bundle. This skill's +job is to pick the best-fit pattern for the user's needs, then scaffold it as a +minimal, one-PR addition that rides the same DAB + eval-gate + dev/staging/prod +lifecycle as every other agent. + +## When to use + +- The project already has **≥1 agent** (`databricks.yml` + `.agentops-stacks/manifest.yml` exist). +- The user wants a single entry point that routes to multiple specialists. +- Usually invoked *after* `/add-agent` has produced a second agent. + +This is a **post-scaffold pattern**, applied as the project matures — like eval +gates, governance, and monitoring. It is not part of `bundle init`. + +## The three patterns + +| | **Custom LangGraph** | **Supervisor API** | **Agent Bricks MAS** | +|---|---|---|---| +| Routing loop owned by | Your code | Databricks (managed) | Databricks (managed tile) | +| Artifact in the bundle | A real agent App under `src/agents/` | A thin wrapper App under `src/agents/` | **None** — a managed endpoint the bundle *consumes* | +| Declarable in `databricks.yml`? | **Yes, natively** | **Yes, as a wrapper App** | **No** — provisioned by an imperative bootstrap job | +| MLflow eval gate in CI? | Yes (standard traces) | Yes (UC OTel + MLflow tracing) | Verify per-workspace; managed | +| Status | GA | Beta (AI Gateway + OTel preview) | UI GA; SDK Beta (admin-gated) | +| Best when | Max control, portability, guardrails, HITL, custom state | Managed loop, minimal code, per-request model choice | No-code, SME-iterable, broadest managed tool coverage | + +**Default is `custom`** — it is the only pattern that is GA, fully DAB-declarable, +and gated on real MLflow eval end-to-end. Deviate only when the matrix says so. + +## Selection Matrix — run these gates in order, take the first that fires + +Ask the user only what you can't already infer from the project and their +description. Show your reasoning. + +**D1 — Lifecycle parity is non-negotiable?** +> "The supervisor must ride the same DAB + eval gate + dev/staging/prod +> promotion as every other agent, on GA, fully in databricks.yml." +→ **custom**. (This is the default; stop here unless a later gate is explicitly required.) + +**D2 — Orchestration control needed?** +Custom state, deterministic/conditional routing, input/output guardrails, HITL +interrupts, a shared Lakebase checkpointer, or tool-level retry policy? +→ **custom**. +Otherwise the job is "fan out to the right specialist over managed sub-agents" → D3. + +**D3 — Build modality?** +- No-code, business-SME-iterable, broadest managed tool coverage (dashboards, AI + Search, nested supervisors, web search), and the user accepts UI-first + Beta + SDK + out-of-band provisioning → **agent_bricks_mas**. +- Code-first, wants the managed loop *without* writing a graph, per-request model + choice (Haiku→Opus, GPT-5), minimal orchestration code but still packaged as a + bundle App → **supervisor_api**. + +**D4 — Compliance / Beta tolerance (override):** +- HIPAA / enhanced-security workspace, or the user needs GA + certainty that + promotion gates on real MLflow traces → **custom** (overrides D3). +- >50 sub-agents, or not every end user will have access to every sub-agent → + rules out **agent_bricks_mas** (hard limits); fall back to **custom** or **supervisor_api**. +- Beta-tolerant and an admin can enable AI Gateway + the UC OTel-traces preview → + keep the D3 answer. + +### Fast discriminators + +- Sub-agents are things you'd code anyway (LLM nodes, `GenieAgent`, endpoints) in one graph → **custom** +- Sub-agents already exist as managed tiles/endpoints, just need routing → **agent_bricks_mas** (no-code) or **supervisor_api** (code) +- "Simplest, fastest demo, a non-engineer maintains it" → **agent_bricks_mas** +- "Lightest code, managed loop, but must live in my repo + CI" → **supervisor_api** +- "Maximum control, portable, GA, gated" → **custom** + +## Workflow + +1. **Locate the project** — find `databricks.yml` in the current dir or a parent. +2. **List existing agents** — show what's under `src/agents/` (these are the + candidate routes). +3. **Run the Selection Matrix** — infer what you can, ask only what's ambiguous, + and state which pattern you chose and why. Confirm with the user. +4. **Gather inputs:** + - **Supervisor name** — must match `^[a-z][a-z0-9_]{2,}$` (and not collide with an existing agent). + - **Routes** — comma-separated sub-agent names. Local agents are validated; + names that aren't local agents are assumed managed sub-agents (Genie/KA/endpoint). + - **Source agent** (custom/supervisor_api only) — which existing agent's App + shape to base on (default: first found). +5. **Run the script:** + ```bash + python plugin/skills/agentops-stacks/scripts/add_supervisor.py \ + --name \ + --type \ + --routes \ + [--from ] \ + --project-dir + ``` +6. **Guide customization** (differs by pattern — see below) and relay the + script's next-steps output unchanged. + +## What the script does, per pattern + +**custom** / **supervisor_api** (both are agent Apps): +1. Copies a source agent as the App shape, renaming references (parity with `add_agent.py`). +2. Overwrites `graph.py` with the supervisor variant and `tools.py` with a supervisor stub. +3. Adds the pattern's dependency to `pyproject.toml` (`langgraph-supervisor` or `databricks-openai`). +4. Appends an experiment + app resource to `databricks.yml`. +5. Records the supervisor in `.agentops-stacks/manifest.yml`. +→ CI's `detect_patterns → eval_gate` picks it up automatically (it has `eval/gates.yml`). + +**agent_bricks_mas** (managed tile, not a DAB resource): +1. Scaffolds `notebooks/bootstrap_supervisor_.py` — imperative Beta-SDK provisioning. +2. Scaffolds `resources/supervisor__bootstrap.yml` — a bundle-declared job that runs it. +3. Includes that resource in `databricks.yml`. +4. Records the supervisor in the manifest with an `endpoint:` field to fill after provisioning. +→ The tile is created out-of-band by running the job; the bundle then *consumes* the endpoint. + +## After adding + +- **custom** — edit `graph.py` to point each route at its real backend + (`GenieAgent`, remote endpoint, or a ReAct sub-agent); add a routing-accuracy + scorer to `eval/gates.yml`; `uv sync`; `bundle validate/deploy -t dev`. +- **supervisor_api** — enable AI Gateway + the UC OTel-traces preview; set each + route's `_ENDPOINT`/Genie id in `graph.py`; `uv sync`; deploy. +- **agent_bricks_mas** — deploy the bundle (deploys the *job*), run the job to + create the tile, record the endpoint in the manifest, then optionally wire it + as a consumable `serving_endpoint` for other agents. + +## Error handling + +- Name collides with an existing agent → abort. +- Invalid name format → abort with the pattern hint. +- No `databricks.yml` found → abort (not an AgentOps Stacks project). +- `custom`/`supervisor_api` with no existing agents to base on → abort (scaffold an agent first). +- Manifest already has a `supervisor:` block → leave it; tell the user to edit by hand. + +## Security posture (state this to the user for the chosen pattern) + +- **custom** — you own guardrails; scope each sub-agent's auth via MLflow + `resources=[...]`; least-privilege per endpoint. Fully in workspace boundary. +- **supervisor_api** — authorization respects the **caller's** UC permissions; + traces to UC tables (OTel preview); `databricks_web_search` unavailable under HIPAA; Beta. +- **agent_bricks_mas** — the end user must have access to **every** sub-agent; + web search blocked in HIPAA/enhanced-security; the provisioning identity ≠ the + bundle identity, so document the service-principal grants. + +## Reference + +- Pattern deep-dive: `docs/supervisor-patterns.md` in the rendered project (scaffolded by the template). +- Agent Bricks Supervisor: https://docs.databricks.com/aws/en/generative-ai/agent-bricks/multi-agent-supervisor +- Supervisor API: https://docs.databricks.com/aws/en/agents/agent-bricks/supervisor-api +- Custom multi-agent apps: https://docs.databricks.com/aws/en/agents/agent-framework/multi-agent-apps diff --git a/plugin/skills/agentops-lifecycle/SKILL.md b/plugin/skills/agentops-lifecycle/SKILL.md index 76b9e9b..919f83c 100644 --- a/plugin/skills/agentops-lifecycle/SKILL.md +++ b/plugin/skills/agentops-lifecycle/SKILL.md @@ -295,6 +295,40 @@ endpoint or UC model registration required. --- +## Step 3.5 — Add a Supervisor (only when you have >1 agent) + +If the project has more than one agent, add a supervisor to route user queries +across them. This is a post-scaffold pattern — use the `/add-supervisor` skill. +Skip if you have a single agent. + +The supervisor is chosen by best fit across three patterns (Selection Matrix in +the `add-supervisor` skill), summarized: + +| Pattern | Loop owner | In the bundle | DAB-declarable | Status | +|---|---|---|---|---| +| **custom** (default) | your code | agent App under `src/agents/` | yes, natively | GA | +| **supervisor_api** | Databricks | wrapper App under `src/agents/` | yes, as an App | Beta | +| **agent_bricks_mas** | Databricks tile | none — a consumed endpoint | no (bootstrap job) | UI GA / SDK Beta | + +- Default to **custom** — it's GA, fully declarable, and gated by the same eval + loop as every other agent (a supervisor scaffolded as an agent App gets its + own `eval/gates.yml`, which CI's `detect_patterns → eval_gate` picks up with + no workflow change). +- Choose **supervisor_api** for a managed loop with minimal code; **agent_bricks_mas** + for a no-code, SME-iterable tile. See `docs/supervisor-patterns.md`. + +```bash +# In your coding assistant: +/add-supervisor +# or: "add a supervisor that routes between my rag and support agents" +``` + +The supervisor is recorded in `.agentops-stacks/manifest.yml` under +`supervisor:`. From here, the rest of the lifecycle (eval gate, CI, staging, +prod) applies to the supervisor agent exactly as it does to any agent. + +--- + ## Step 4 — Offline Evaluation & Eval Gate Setup Build the evaluation framework **before** any code leaves dev. The scaffold diff --git a/plugin/skills/agentops-stacks/scripts/add_supervisor.py b/plugin/skills/agentops-stacks/scripts/add_supervisor.py new file mode 100644 index 0000000..9a603ac --- /dev/null +++ b/plugin/skills/agentops-stacks/scripts/add_supervisor.py @@ -0,0 +1,384 @@ +#!/usr/bin/env python3 +"""Add a supervisor agent to an existing AgentOps Stacks project. + +A supervisor routes user queries across the project's existing agents (and +other managed sub-agents such as Genie spaces or Knowledge Assistants). This +script supports the three supervisor patterns AgentOps Stacks recognizes, which +differ in *who owns the routing loop* and *what artifact lands in the bundle*: + + custom A hand-written LangGraph supervisor graph. It is just another + agent under src/agents// — served as a Databricks App + via MLflow AgentServer, fully declared in databricks.yml, and + gated by the same CI eval loop as every other agent. GA. + + supervisor_api The Databricks-managed supervisor loop (Responses API), + packaged as a thin wrapper App. Same declarable App shape as + `custom`; Databricks owns the routing loop. Beta — requires + AI Gateway + the UC OTel-traces preview enabled. + + agent_bricks_mas The Agent Bricks Supervisor tile. This is NOT a DAB resource + and CANNOT be created by `databricks bundle deploy`. The script + scaffolds a bundle-declared *bootstrap job* (imperative, via the + Beta SDK / manage_mas) plus a consumable endpoint reference. The + tile itself is provisioned out-of-band and merely referenced. + +For `custom` and `supervisor_api` the supervisor is created by cloning the agent +scaffold shape (mirroring add_agent.py) and swapping in a supervisor graph, so +CI's `detect_patterns -> eval_gate` picks it up with zero workflow changes. + +Usage: + python add_supervisor.py --name router --type custom --routes rag,support + python add_supervisor.py --name router --type supervisor_api --routes rag,support + python add_supervisor.py --name ops_mas --type agent_bricks_mas --routes rag,support + +Recommended: use the /add-supervisor skill, which runs the Selection Matrix +conversationally and then calls this script. +""" + +import argparse +import re +import shutil +import sys +from pathlib import Path + +NAME_RE = re.compile(r"^[a-z][a-z0-9_]{2,}$") +VALID_TYPES = {"custom", "supervisor_api", "agent_bricks_mas"} + +# Templates live next to this script so they ship with the plugin and don't +# collide with the DAB `template/` tree (which is Go-templated by bundle init). +TEMPLATE_DIR = Path(__file__).resolve().parent / "supervisor_templates" + + +# --------------------------------------------------------------------------- # +# Project discovery +# --------------------------------------------------------------------------- # + +def find_project_root(start: Path) -> Path: + """Walk up from start to find the project root (has databricks.yml).""" + current = start.resolve() + while current != current.parent: + if (current / "databricks.yml").exists(): + return current + current = current.parent + raise FileNotFoundError("Could not find databricks.yml in any parent directory") + + +def find_existing_agents(project_root: Path) -> list[str]: + """Return list of existing agent names (dirs under src/agents with agent.py).""" + agents_dir = project_root / "src" / "agents" + if not agents_dir.exists(): + return [] + return sorted( + d.name for d in agents_dir.iterdir() + if d.is_dir() and (d / "agent.py").exists() + ) + + +def get_project_name(project_root: Path) -> str: + """Extract the bundle name from databricks.yml.""" + content = (project_root / "databricks.yml").read_text() + match = re.search(r"^\s*name:\s*(.+)$", content, re.MULTILINE) + return match.group(1).strip() if match else "unknown" + + +def hyphenate(name: str) -> str: + return name.replace("_", "-") + + +# --------------------------------------------------------------------------- # +# Template rendering — a tiny {{ placeholder }} substituter (NOT Go templates, +# to avoid any interaction with `databricks bundle init`). +# --------------------------------------------------------------------------- # + +def render(template_name: str, subs: dict) -> str: + raw = (TEMPLATE_DIR / template_name).read_text() + for key, val in subs.items(): + raw = raw.replace("{{" + key + "}}", val) + return raw + + +# --------------------------------------------------------------------------- # +# custom / supervisor_api — scaffold the supervisor as an agent App +# --------------------------------------------------------------------------- # + +def scaffold_supervisor_agent(project_root: Path, name: str, sup_type: str, + source: str, routes: list[str]): + """Copy an existing agent as the base, then overwrite graph/tools/deps with + the supervisor variant. Keeps the eval harness, app server, and app.yaml so + the CI eval gate and Databricks App serving work unchanged.""" + agents_dir = project_root / "src" / "agents" + source_dir = agents_dir / source + new_dir = agents_dir / name + + if new_dir.exists(): + sys.exit(f"ERROR: Agent directory already exists: {new_dir}") + if not source_dir.exists(): + sys.exit(f"ERROR: Source agent not found: {source_dir}") + + shutil.copytree(source_dir, new_dir) + + # Rename all references to the source agent -> supervisor name (mirrors + # add_agent.py so app/start_server.py, eval experiment names, etc. line up). + for filepath in new_dir.rglob("*"): + if not filepath.is_file(): + continue + try: + content = filepath.read_text() + except UnicodeDecodeError: + continue + updated = content.replace(source, name) + if updated != content: + filepath.write_text(updated) + + subs = { + "SUPERVISOR_NAME": name, + "PROJECT_NAME": get_project_name(project_root), + "ROUTES_PY_LIST": repr(routes), + "ROUTES_COMMENT": ", ".join(routes) if routes else "(none yet — edit graph.py)", + } + + graph_tmpl = ("graph_custom.py.tmpl" if sup_type == "custom" + else "graph_supervisor_api.py.tmpl") + (new_dir / "graph.py").write_text(render(graph_tmpl, subs)) + (new_dir / "tools.py").write_text(render("tools_supervisor.py.tmpl", subs)) + # Overwrite agent.py with the supervisor handler. The source agent's agent.py + # may import graph symbols (e.g. get_async_checkpointer when the base agent + # had Lakebase memory) that the supervisor graph.py doesn't define — which + # would break server startup. The supervisor uses a stateless handler. + (new_dir / "agent.py").write_text(render("agent_supervisor.py.tmpl", subs)) + + # Merge extra deps into the supervisor's pyproject.toml. + _add_supervisor_deps(new_dir / "pyproject.toml", sup_type) + + print(f" Created: src/agents/{name}/ (supervisor, type={sup_type})") + print(f" graph.py routes to: {subs['ROUTES_COMMENT']}") + + +def _add_supervisor_deps(pyproject: Path, sup_type: str): + """Add the supervisor's runtime dependency to pyproject.toml if missing.""" + if not pyproject.exists(): + return + content = pyproject.read_text() + dep = (' "langgraph-supervisor>=0.0.5",' + if sup_type == "custom" + else ' "databricks-openai>=0.4.0",') + marker = dep.strip().split(">=")[0].strip('"') + if marker in content: + return + # Insert right after the langgraph pin, which every agent has. + content = re.sub( + r'(\n\s*"langgraph>=[^"]+",)', + r"\1\n" + dep, + content, + count=1, + ) + pyproject.write_text(content) + + +def append_agent_resources_to_databricks_yml(project_root: Path, name: str): + """Append experiment + app resource for the supervisor agent (identical + wiring to add_agent.py so the App deploys like any other agent).""" + yml_path = project_root / "databricks.yml" + content = yml_path.read_text() + project_name = get_project_name(project_root) + + experiment_block = f""" + {name}_experiment: + name: /Shared/${{bundle.name}}_{name}_${{bundle.target}} + artifact_location: dbfs:/Volumes/${{var.catalog}}/${{var.schema}}/artifacts""" + + app_block = f""" + {name}: + name: "{hyphenate(project_name)}-{hyphenate(name)}" + description: "{name} supervisor — {project_name}" + source_code_path: ./src/agents/{name} + config: + command: ["uv", "run", "python", "app/start_server.py"] + resources: + - name: "experiment" + experiment: + experiment_id: ${{resources.experiments.{name}_experiment.id}} + permission: "CAN_MANAGE\"""" + + if "experiments:" in content: + content = content.replace("\n apps:", f"{experiment_block}\n\n apps:") + else: + content = content.replace( + "resources:\n apps:", + f"resources:\n experiments:{experiment_block}\n\n apps:", + ) + + content = content.replace("\nsync:", f"{app_block}\n\nsync:") + yml_path.write_text(content) + print(f" Updated: databricks.yml (added experiment + app for {name})") + + +# --------------------------------------------------------------------------- # +# agent_bricks_mas — scaffold a bootstrap job (imperative), NOT a DAB resource +# --------------------------------------------------------------------------- # + +def scaffold_mas_bootstrap(project_root: Path, name: str, routes: list[str]): + """Agent Bricks Supervisor tiles are not DAB resources and cannot be created + by `bundle deploy`. We scaffold a bundle-declared job that runs a notebook + calling the Beta manage_mas SDK, plus a notebook stub. The resulting endpoint + is recorded in the manifest and can then be *consumed* by other agents.""" + subs = { + "SUPERVISOR_NAME": name, + "PROJECT_NAME": get_project_name(project_root), + "ROUTES_PY_LIST": repr(routes), + } + + # Bootstrap notebook (imperative provisioning). + nb_dir = project_root / "notebooks" + nb_dir.mkdir(exist_ok=True) + nb_path = nb_dir / f"bootstrap_supervisor_{name}.py" + if nb_path.exists(): + sys.exit(f"ERROR: Bootstrap notebook already exists: {nb_path}") + nb_path.write_text(render("mas_bootstrap_notebook.py.tmpl", subs)) + print(f" Created: notebooks/bootstrap_supervisor_{name}.py (imperative, Beta SDK)") + + # DAB-declared job that runs the bootstrap notebook. This IS declarative — + # what's non-declarative is the tile the notebook creates, which we document. + res_dir = project_root / "resources" + res_dir.mkdir(exist_ok=True) + job_path = res_dir / f"supervisor_{name}_bootstrap.yml" + if job_path.exists(): + sys.exit(f"ERROR: Bootstrap job resource already exists: {job_path}") + job_path.write_text(render("mas_bootstrap_job.yml.tmpl", subs)) + print(f" Created: resources/supervisor_{name}_bootstrap.yml (bundle-declared job)") + + # Wire the include into databricks.yml. + yml_path = project_root / "databricks.yml" + content = yml_path.read_text() + include_line = f" - ./resources/supervisor_{name}_bootstrap.yml\n" + if include_line not in content: + content = re.sub( + r"(include:\n)", + r"\1" + include_line, + content, + count=1, + ) + yml_path.write_text(content) + print(f" Updated: databricks.yml (included bootstrap job resource)") + + +# --------------------------------------------------------------------------- # +# Manifest +# --------------------------------------------------------------------------- # + +def update_manifest(project_root: Path, name: str, sup_type: str, routes: list[str]): + """Record the supervisor in the scaffold contract. CI and tooling read this + without needing to know how the supervisor was created.""" + manifest_path = project_root / ".agentops-stacks" / "manifest.yml" + if not manifest_path.exists(): + print(f" WARN: No manifest at {manifest_path} — skipping manifest update") + return + + content = manifest_path.read_text().rstrip("\n") + if "\nsupervisor:" in content or content.startswith("supervisor:"): + print(" WARN: manifest already has a `supervisor:` block — leaving it " + "in place. Edit .agentops-stacks/manifest.yml by hand if needed.") + return + + routes_yaml = "".join(f"\n - {r}" for r in routes) if routes else " []" + block = f""" + +# Supervisor agent (added via /add-supervisor) +# type: custom | supervisor_api | agent_bricks_mas +supervisor: + type: {sup_type} + name: {name} + routes:{routes_yaml}""" + if sup_type == "agent_bricks_mas": + block += f""" + # Agent Bricks MAS tiles are provisioned imperatively (not by bundle deploy). + # After running the bootstrap job, record the managed endpoint name here: + endpoint: "" # TODO: set after notebooks/bootstrap_supervisor_{name}.py runs""" + + manifest_path.write_text(content + block + "\n") + print(" Updated: .agentops-stacks/manifest.yml (supervisor block)") + + +# --------------------------------------------------------------------------- # +# Main +# --------------------------------------------------------------------------- # + +def main(): + parser = argparse.ArgumentParser( + description="Add a supervisor agent to an AgentOps Stacks project" + ) + parser.add_argument("--name", required=True, + help="Supervisor agent name (lowercase, underscores, min 3 chars)") + parser.add_argument("--type", required=True, choices=sorted(VALID_TYPES), + help="Supervisor pattern (see the Selection Matrix in SKILL.md)") + parser.add_argument("--routes", default="", + help="Comma-separated sub-agent names this supervisor routes to") + parser.add_argument("--from", dest="source", default=None, + help="Existing agent to base the App shape on " + "(custom/supervisor_api only; default: first found)") + parser.add_argument("--project-dir", default=".", help="Project root (default: cwd)") + args = parser.parse_args() + + if not NAME_RE.match(args.name): + sys.exit("ERROR: Supervisor name must start with a lowercase letter and " + "contain only lowercase letters, digits, and underscores (min 3 chars).") + + routes = [r.strip() for r in args.routes.split(",") if r.strip()] + + project_root = find_project_root(Path(args.project_dir)) + print(f"Project root: {project_root}") + + existing = find_existing_agents(project_root) + if args.name in existing: + sys.exit(f"ERROR: An agent named '{args.name}' already exists.") + + # Validate that routes reference real agents (warn, don't hard-fail — a route + # may point at a Genie space or managed endpoint rather than a local agent). + unknown = [r for r in routes if r not in existing] + if unknown: + print(f" NOTE: routes not matching a local agent (assumed managed " + f"sub-agents — Genie/KA/endpoint): {unknown}") + + if args.type in ("custom", "supervisor_api"): + if not existing: + sys.exit("ERROR: No existing agents to base the supervisor App on. " + "Scaffold at least one agent first.") + source = args.source or existing[0] + if source not in existing: + sys.exit(f"ERROR: Source agent '{source}' not found. Available: {existing}") + print(f"Adding {args.type} supervisor '{args.name}' (App shape from '{source}')\n") + scaffold_supervisor_agent(project_root, args.name, args.type, source, routes) + append_agent_resources_to_databricks_yml(project_root, args.name) + else: # agent_bricks_mas + print(f"Adding agent_bricks_mas supervisor '{args.name}' " + f"(bootstrap job + endpoint reference)\n") + scaffold_mas_bootstrap(project_root, args.name, routes) + + update_manifest(project_root, args.name, args.type, routes) + + print(f"\nDone. Supervisor '{args.name}' ({args.type}) added.") + _print_next_steps(args.name, args.type) + + +def _print_next_steps(name: str, sup_type: str): + print("\nNext steps:") + if sup_type == "custom": + print(f" 1. cd src/agents/{name} && uv sync # picks up langgraph-supervisor") + print(f" 2. Edit graph.py — confirm each route's sub-agent endpoint/Genie id") + print(f" 3. Edit eval/gates.yml — add a routing-accuracy scorer for the supervisor") + print(f" 4. databricks bundle validate -t dev && databricks bundle deploy -t dev") + elif sup_type == "supervisor_api": + print(f" 1. cd src/agents/{name} && uv sync # picks up databricks-openai") + print(f" 2. Ensure AI Gateway + the UC OTel-traces preview are enabled (Beta)") + print(f" 3. Edit graph.py — set the sub-agent tool references (genie_space/serving_endpoint)") + print(f" 4. databricks bundle validate -t dev && databricks bundle deploy -t dev") + else: + print(f" 1. Agent Bricks MAS is NOT created by bundle deploy (not a DAB resource).") + print(f" 2. databricks bundle deploy -t dev # deploys the bootstrap JOB") + print(f" 3. Run the job (or notebooks/bootstrap_supervisor_{name}.py) to create the tile.") + print(f" 4. Record the resulting endpoint in .agentops-stacks/manifest.yml (supervisor.endpoint).") + print(f" 5. Other agents can then consume it as a serving_endpoint resource.") + + +if __name__ == "__main__": + main() diff --git a/plugin/skills/agentops-stacks/scripts/supervisor_templates/agent_supervisor.py.tmpl b/plugin/skills/agentops-stacks/scripts/supervisor_templates/agent_supervisor.py.tmpl new file mode 100644 index 0000000..5784be5 --- /dev/null +++ b/plugin/skills/agentops-stacks/scripts/supervisor_templates/agent_supervisor.py.tmpl @@ -0,0 +1,64 @@ +"""{{SUPERVISOR_NAME}} — MLflow AgentServer handlers (supervisor). + +@invoke/@stream entry points. Routing/graph assembly lives in graph.py. + +This is the stateless supervisor handler: it compiles the routing graph without +a checkpointer. If you want conversation memory at the supervisor level, add a +Lakebase checkpointer here the same way the base agent template does (compile +per-request with an async checkpointer keyed on the session/thread id). +""" + +import logging +from typing import AsyncGenerator + +import mlflow +from mlflow.genai.agent_server import invoke, stream +from mlflow.types.responses import ( + ResponsesAgentRequest, + ResponsesAgentResponse, + ResponsesAgentStreamEvent, + to_chat_completions_input, +) + +from graph import graph +from app.utils import ( + get_session_id, + process_agent_astream_events, +) + +logger = logging.getLogger(__name__) +mlflow.langchain.autolog() + + +@invoke() +async def invoke_handler(request: ResponsesAgentRequest) -> ResponsesAgentResponse: + """Handle synchronous invocation requests.""" + outputs = [ + event.item + async for event in stream_handler(request) + if event.type == "response.output_item.done" + ] + return ResponsesAgentResponse(output=outputs) + + +@stream() +async def stream_handler( + request: ResponsesAgentRequest, +) -> AsyncGenerator[ResponsesAgentStreamEvent, None]: + """Handle streaming requests.""" + if not isinstance(request, ResponsesAgentRequest): + request = ResponsesAgentRequest(**request) + + session_id = get_session_id(request) + if session_id: + mlflow.update_current_trace(metadata={"mlflow.trace.session": session_id}) + + messages = {"messages": to_chat_completions_input([i.model_dump() for i in request.input])} + + compiled = graph + config = {} + + async for event in process_agent_astream_events( + compiled.astream(input=messages, config=config, stream_mode=["updates", "messages"]) + ): + yield event diff --git a/plugin/skills/agentops-stacks/scripts/supervisor_templates/graph_custom.py.tmpl b/plugin/skills/agentops-stacks/scripts/supervisor_templates/graph_custom.py.tmpl new file mode 100644 index 0000000..09ed018 --- /dev/null +++ b/plugin/skills/agentops-stacks/scripts/supervisor_templates/graph_custom.py.tmpl @@ -0,0 +1,98 @@ +"""{{SUPERVISOR_NAME}} — custom LangGraph supervisor. + +Routes across sub-agents: {{ROUTES_COMMENT}} + +This is a hand-written supervisor: your code owns the routing loop, so you +control state, guardrails, retries, and human-in-the-loop. It is served as a +Databricks App via MLflow AgentServer exactly like any other agent, is declared +in databricks.yml, and is gated by the same CI eval loop. + +Sub-agents can be: + - Local agents in this project, reached as remote serving endpoints/Apps once + deployed (wire their endpoint names below). + - Genie spaces, via databricks_langchain.GenieAgent. + - Any Databricks model serving endpoint. + +Edit `build_sub_agents()` to point each route at its real backend. +""" + +import os +import logging + +from langgraph.graph import MessagesState +from langgraph.prebuilt import create_react_agent +from langgraph_supervisor import create_supervisor +from databricks_langchain import ChatDatabricks + +logger = logging.getLogger(__name__) + +LLM_ENDPOINT = os.environ.get("LLM_ENDPOINT", "databricks-claude-sonnet-4") + +# Sub-agent routes recorded at scaffold time. Each name becomes a managed +# member of the supervisor. Confirm/adjust the backing for each below. +ROUTES = {{ROUTES_PY_LIST}} + +# Routing instructions steer the supervisor's handoff decisions. Make each +# sub-agent's responsibility specific and non-overlapping. +SUPERVISOR_PROMPT = ( + "You are a supervisor routing user requests to the most appropriate " + "specialist agent. Analyze the request, hand off to exactly one specialist, " + "and return its answer. If the request spans domains, chain specialists: " + "gather information first, then act. If no specialist fits, answer directly " + "and say so." +) + + +def _sub_agent_for(route: str): + """Build a LangGraph sub-agent for a route. + + Default: a ReAct agent backed by the Foundation Model API, named after the + route. Replace a branch with one of these to reach real backends: + + # Genie space (SQL analytics): + from databricks_langchain.genie import GenieAgent + return GenieAgent(genie_space_id=os.environ["_GENIE_SPACE_ID"], + genie_agent_name=route) + + # Remote serving endpoint / deployed sibling agent: + from databricks_langchain import ChatDatabricks + llm = ChatDatabricks(endpoint=os.environ["_ENDPOINT"]) + return create_react_agent(llm, tools=[], name=route) + """ + llm = ChatDatabricks(endpoint=LLM_ENDPOINT) + return create_react_agent( + llm, + tools=[], # TODO: give each specialist its own tools + prompt=f"You are the '{route}' specialist. Handle only {route}-related requests.", + name=route, + ) + + +def build_sub_agents() -> list: + """Return the list of compiled sub-agents the supervisor can route to.""" + if not ROUTES: + logger.warning("No routes configured for supervisor — add sub-agent names " + "to ROUTES and wire their backends in _sub_agent_for().") + return [_sub_agent_for(route) for route in ROUTES] + + +def build_graph(): + """Assemble the supervisor over its sub-agents.""" + sub_agents = build_sub_agents() + supervisor_llm = ChatDatabricks(endpoint=LLM_ENDPOINT) + # create_supervisor returns an uncompiled StateGraph; compile at export. + return create_supervisor( + sub_agents, + model=supervisor_llm, + prompt=SUPERVISOR_PROMPT, + output_mode="full_history", + state_schema=MessagesState, + ) + + +# Export builder for per-request compilation (parity with the agent template, +# so a Lakebase checkpointer can be attached the same way if memory is enabled). +graph_builder = build_graph() + +# Compiled graph used at startup / as a fallback. +graph = graph_builder.compile() diff --git a/plugin/skills/agentops-stacks/scripts/supervisor_templates/graph_supervisor_api.py.tmpl b/plugin/skills/agentops-stacks/scripts/supervisor_templates/graph_supervisor_api.py.tmpl new file mode 100644 index 0000000..f74be30 --- /dev/null +++ b/plugin/skills/agentops-stacks/scripts/supervisor_templates/graph_supervisor_api.py.tmpl @@ -0,0 +1,99 @@ +"""{{SUPERVISOR_NAME}} — Supervisor API (managed loop) wrapper. + +Routes across sub-agents: {{ROUTES_COMMENT}} + +Databricks owns the routing loop here: the managed Supervisor API decides which +sub-agent/tool to call, executes it, feeds results back, and repeats until a +final answer. You write no routing graph — you declare the sub-agents as tools +and let the AI Gateway run the loop. + +We wrap that single managed call in a one-node LangGraph so this supervisor +serves through the same MLflow AgentServer / Databricks App path as every other +agent in the bundle, and streams through the same app/utils.py handlers. + +Beta requirements (enable before deploying to staging/prod): + - AI Gateway enabled on the workspace. + - "Store OpenTelemetry traces in Unity Catalog" preview enabled (for tracing). + - Caller identity has UC access to every referenced sub-agent. + +Docs: https://docs.databricks.com/aws/en/agents/agent-bricks/supervisor-api +""" + +import os +import logging + +from langchain_core.messages import AIMessage +from langgraph.graph import START, END, StateGraph, MessagesState + +logger = logging.getLogger(__name__) + +SUPERVISOR_MODEL = os.environ.get("SUPERVISOR_MODEL", "databricks-claude-sonnet-4") + +# Sub-agent routes recorded at scaffold time. Each becomes a tool the managed +# supervisor may call. Fill in the concrete reference for each route below. +ROUTES = {{ROUTES_PY_LIST}} + + +def _supervisor_tools() -> list: + """Build the tool list passed to the managed supervisor loop. + + Each entry references a sub-agent by its Databricks-native type. Replace the + placeholders with real ids/names. Supported types include: + {"type": "genie_space", "genie_space": {"space_id": "..."}} + {"type": "serving_endpoint", "serving_endpoint": {"name": "..."}} + {"type": "knowledge_assistant", "knowledge_assistant": {"endpoint_name": "..."}} + {"type": "uc_function", "uc_function": {"name": "catalog.schema.fn"}} + """ + tools = [] + for route in ROUTES: + # Default assumption: each route is a deployed serving endpoint whose + # name is provided via env var _ENDPOINT. Adjust per route. + endpoint = os.environ.get(f"{route.upper()}_ENDPOINT") + if endpoint: + tools.append({"type": "serving_endpoint", + "serving_endpoint": {"name": endpoint}}) + else: + logger.warning("Route '%s' has no _ENDPOINT set — edit " + "_supervisor_tools() to reference its backend.", route) + return tools + + +def _call_supervisor(messages: list[dict]) -> str: + """Invoke the managed Supervisor API and return the final text answer.""" + from databricks_openai import DatabricksOpenAI + + client = DatabricksOpenAI(use_ai_gateway=True) + resp = client.responses.create( + model=SUPERVISOR_MODEL, + input=messages, + tools=_supervisor_tools(), + ) + # The Responses API exposes a convenience aggregate of the final output. + return getattr(resp, "output_text", None) or str(resp) + + +# LangChain message .type -> Responses API role. +_ROLE_MAP = {"human": "user", "ai": "assistant", "system": "system", "tool": "tool"} + + +def supervisor_node(state: MessagesState) -> dict: + """Single node: hand the conversation to the managed supervisor loop.""" + messages = [ + {"role": _ROLE_MAP.get(getattr(m, "type", "human"), "user"), + "content": m.content} + for m in state["messages"] + ] + answer = _call_supervisor(messages) + return {"messages": [AIMessage(content=answer)]} + + +def build_graph(): + builder = StateGraph(MessagesState) + builder.add_node("supervisor", supervisor_node) + builder.add_edge(START, "supervisor") + builder.add_edge("supervisor", END) + return builder + + +graph_builder = build_graph() +graph = graph_builder.compile() diff --git a/plugin/skills/agentops-stacks/scripts/supervisor_templates/mas_bootstrap_job.yml.tmpl b/plugin/skills/agentops-stacks/scripts/supervisor_templates/mas_bootstrap_job.yml.tmpl new file mode 100644 index 0000000..53cd89a --- /dev/null +++ b/plugin/skills/agentops-stacks/scripts/supervisor_templates/mas_bootstrap_job.yml.tmpl @@ -0,0 +1,18 @@ +# Bundle-declared bootstrap job for the {{SUPERVISOR_NAME}} Agent Bricks Supervisor. +# +# The Supervisor *tile* itself is NOT a DAB resource and cannot be created by +# `databricks bundle deploy` — this job runs the imperative bootstrap notebook +# that provisions it via the Beta SDK. The job is fully declarative; the tile it +# creates is provisioned out-of-band. Migrate to a native resource when Agent +# Bricks Supervisors become a first-class DAB resource type. +resources: + jobs: + supervisor_{{SUPERVISOR_NAME}}_bootstrap: + name: "${bundle.name}_{{SUPERVISOR_NAME}}_supervisor_bootstrap" + tasks: + - task_key: provision_supervisor + notebook_task: + notebook_path: ../notebooks/bootstrap_supervisor_{{SUPERVISOR_NAME}}.py + base_parameters: + catalog: "${var.catalog}" + schema: "${var.schema}" diff --git a/plugin/skills/agentops-stacks/scripts/supervisor_templates/mas_bootstrap_notebook.py.tmpl b/plugin/skills/agentops-stacks/scripts/supervisor_templates/mas_bootstrap_notebook.py.tmpl new file mode 100644 index 0000000..e04c0c8 --- /dev/null +++ b/plugin/skills/agentops-stacks/scripts/supervisor_templates/mas_bootstrap_notebook.py.tmpl @@ -0,0 +1,107 @@ +# Databricks notebook source + +# MAGIC %md +# MAGIC # Bootstrap Agent Bricks Supervisor — {{SUPERVISOR_NAME}} +# MAGIC +# MAGIC **Why a notebook and not `databricks.yml`?** An Agent Bricks Supervisor +# MAGIC (MAS) tile is **not a Declarative Automation Bundle resource type**. It +# MAGIC cannot be created by `databricks bundle deploy`. This notebook provisions +# MAGIC it imperatively via the Beta SDK, and the surrounding bundle-declared job +# MAGIC (`resources/supervisor_{{SUPERVISOR_NAME}}_bootstrap.yml`) makes running it +# MAGIC repeatable and CI-triggerable. +# MAGIC +# MAGIC Standing requirement honored: *"Notebooks only create resources DAB +# MAGIC doesn't yet support. Document why so it can be migrated when support lands."* +# MAGIC +# MAGIC **Beta / prereqs:** +# MAGIC - Agent Bricks Supervisor is Beta; the management SDK requires +# MAGIC account-admin approval to enable. +# MAGIC - Each sub-agent must already exist (Genie space, KA endpoint, serving +# MAGIC endpoint, UC function, or MCP connection). +# MAGIC - The end user querying the supervisor must have access to **every** +# MAGIC sub-agent, or the supervisor cannot answer. +# MAGIC +# MAGIC Docs: https://docs.databricks.com/aws/en/generative-ai/agent-bricks/multi-agent-supervisor + +# COMMAND ---------- + +dbutils.widgets.text("catalog", "", "Unity Catalog (for endpoint reference)") +dbutils.widgets.text("schema", "", "Schema") + +catalog = dbutils.widgets.get("catalog") +schema = dbutils.widgets.get("schema") + +# Sub-agent routes recorded at scaffold time. +ROUTES = {{ROUTES_PY_LIST}} + +# COMMAND ---------- + +# MAGIC %md +# MAGIC ## Configure sub-agents +# MAGIC +# MAGIC Provide exactly one backing reference per agent: `genie_space_id`, +# MAGIC `ka_tile_id`, `endpoint_name`, `uc_function_name`, or `connection_name`. +# MAGIC The `description` is critical — it drives the supervisor's routing. + +# COMMAND ---------- + +# TODO: map each route to its real backend. Example shapes: +AGENTS = [ + # {"name": "rag", "endpoint_name": "my-rag-endpoint", + # "description": "Answers product questions from indexed docs."}, + # {"name": "analytics", "genie_space_id": "01abc...", + # "description": "SQL analytics on usage metrics and trends."}, +] + +INSTRUCTIONS = ( + "Route each request to the single most appropriate specialist based on its " + "description. If the request spans domains, gather information first, then act." +) + +assert AGENTS, ( + "No sub-agents configured. Map each route in ROUTES to a backend in AGENTS " + f"before running. Routes recorded at scaffold time: {ROUTES}" +) + +# COMMAND ---------- + +# MAGIC %md +# MAGIC ## Create or update the Supervisor tile (Beta SDK) +# MAGIC +# MAGIC This mirrors the `manage_mas` MCP tool. If you have the agentops MCP / +# MAGIC ai-dev-kit tooling available in your assistant, you can instead call +# MAGIC `manage_mas(action="create_or_update", ...)` and skip the raw SDK call. + +# COMMAND ---------- + +# The exact Beta SDK import path is version-dependent. Prefer the manage_mas +# MCP tool where available. Raw-SDK sketch: +# +# from databricks.agents.supervisor import create_or_update_supervisor # Beta +# result = create_or_update_supervisor( +# name="{{SUPERVISOR_NAME}}", +# agents=AGENTS, +# description="{{SUPERVISOR_NAME}} supervisor — {{PROJECT_NAME}}", +# instructions=INSTRUCTIONS, +# ) +# endpoint_name = result.endpoint_name +# print(f"Supervisor endpoint: {endpoint_name}") +# print("Record this in .agentops-stacks/manifest.yml under supervisor.endpoint") + +raise NotImplementedError( + "Fill in AGENTS above and enable the Beta Supervisor SDK (or use the " + "manage_mas MCP tool), then remove this guard. Provisioning is imperative " + "and Beta — this is intentionally not silent." +) + +# COMMAND ---------- + +# MAGIC %md +# MAGIC ## After provisioning +# MAGIC +# MAGIC 1. Wait for endpoint status `ONLINE` (2–5 min). +# MAGIC 2. Record the endpoint name in `.agentops-stacks/manifest.yml` +# MAGIC (`supervisor.endpoint`). +# MAGIC 3. To let other agents in this bundle *consume* it, add a +# MAGIC `serving_endpoint` resource referencing it and grant the app SP query +# MAGIC access. The bundle consumes it; it does not create it. diff --git a/plugin/skills/agentops-stacks/scripts/supervisor_templates/tools_supervisor.py.tmpl b/plugin/skills/agentops-stacks/scripts/supervisor_templates/tools_supervisor.py.tmpl new file mode 100644 index 0000000..f22309d --- /dev/null +++ b/plugin/skills/agentops-stacks/scripts/supervisor_templates/tools_supervisor.py.tmpl @@ -0,0 +1,15 @@ +"""Tool selection for the {{SUPERVISOR_NAME}} supervisor. + +A supervisor's "tools" are its sub-agents — those are wired in graph.py +(create_supervisor members for `custom`, or the managed tool list for +`supervisor_api`), not here. + +This file exists for parity with the agent scaffold and for any *extra* +supervisor-level tools you want available alongside routing (e.g. a shared +lookup UC function the supervisor itself may call before deciding a route). +""" + + +def get_tools() -> list: + """Return supervisor-level tools (routing sub-agents are wired in graph.py).""" + return [] diff --git a/plugin/skills/install_skills.sh b/plugin/skills/install_skills.sh index 990dbcc..ec021e4 100755 --- a/plugin/skills/install_skills.sh +++ b/plugin/skills/install_skills.sh @@ -22,7 +22,7 @@ YELLOW='\033[1;33m' BLUE='\033[0;34m' NC='\033[0m' -SKILL_NAMES=("agentops-stacks" "agentops-lifecycle" "add-agent" "vector-search-ops" "lakebase-ops" "uc-functions-ops") +SKILL_NAMES=("agentops-stacks" "agentops-lifecycle" "add-agent" "add-supervisor" "vector-search-ops" "lakebase-ops" "uc-functions-ops") SKILLS_DIR=".claude/skills" INSTALL_TO_GENIE=false DB_PROFILE="${DATABRICKS_CONFIG_PROFILE:-DEFAULT}" @@ -50,6 +50,7 @@ show_help() { echo " - agentops-stacks: Scaffold a new DAB project with CI/CD and UC conventions" echo " - agentops-lifecycle: Guide an existing scaffold through the 10-step dev→prod lifecycle" echo " - add-agent: Add a second agent to an existing scaffold" + echo " - add-supervisor: Add a supervisor that routes across agents (best-fit pattern)" echo " - vector-search-ops: Operate and troubleshoot the Vector Search component" echo " - lakebase-ops: Operate and troubleshoot the Lakebase memory component" echo " - uc-functions-ops: Register, grant, and manage UC function tools" @@ -68,6 +69,10 @@ list_skills() { echo -e " ${GREEN}add-agent${NC}" echo " Add a second agent to an existing scaffold" echo "" + echo -e " ${GREEN}add-supervisor${NC}" + echo " Add a supervisor that routes across agents — picks the best-fit pattern" + echo " (custom LangGraph / Supervisor API / Agent Bricks MAS)" + echo "" echo -e " ${GREEN}vector-search-ops${NC}" echo " Check index status, trigger sync, test retriever, update DLT pipeline" echo "" diff --git a/template/{{.input_root_dir}}/.agentops-stacks/manifest.yml.tmpl b/template/{{.input_root_dir}}/.agentops-stacks/manifest.yml.tmpl index c593430..38cc615 100644 --- a/template/{{.input_root_dir}}/.agentops-stacks/manifest.yml.tmpl +++ b/template/{{.input_root_dir}}/.agentops-stacks/manifest.yml.tmpl @@ -19,3 +19,12 @@ components: # Registered agents (add new agents here or via /add-agent skill) agents: - name: {{ .input_initial_agent_name }} + +# Supervisor agent (added via /add-supervisor once you have >1 agent). +# Routes user queries across the agents above. Three patterns are supported — +# see docs/supervisor-patterns.md. None is configured until you add one. +# supervisor: +# type: custom | supervisor_api | agent_bricks_mas +# name: +# routes: [, ] +# endpoint: "" # agent_bricks_mas only — the provisioned managed endpoint diff --git a/template/{{.input_root_dir}}/docs/supervisor-patterns.md.tmpl b/template/{{.input_root_dir}}/docs/supervisor-patterns.md.tmpl new file mode 100644 index 0000000..b8b215b --- /dev/null +++ b/template/{{.input_root_dir}}/docs/supervisor-patterns.md.tmpl @@ -0,0 +1,110 @@ +# Supervisor Patterns + +A supervisor agent routes user queries across your project's agents. When you +have more than one agent, add a supervisor with the `/add-supervisor` skill (or +`plugin/skills/agentops-stacks/scripts/add_supervisor.py`). + +There are three supervisor patterns. They differ in **who owns the routing +loop** and **what artifact lands in this bundle** — pick by best fit, not habit. + +## Choosing a pattern + +| | **Custom LangGraph** | **Supervisor API** | **Agent Bricks MAS** | +|---|---|---|---| +| Routing loop owned by | Your code | Databricks (managed) | Databricks (managed tile) | +| Artifact in the bundle | Agent App under `src/agents/` | Wrapper App under `src/agents/` | None — a consumed endpoint | +| Declarable in `databricks.yml`? | Yes, natively | Yes, as a wrapper App | No — imperative bootstrap job | +| Gated by the CI eval gate? | Yes | Yes | Verify per-workspace | +| Status | GA | Beta | UI GA; SDK Beta | + +**Default: Custom LangGraph.** It is the only pattern that is GA, fully +DAB-declarable, and gated on real MLflow eval end-to-end. Choose another only +when its specific advantage is required. + +### Decision matrix (first gate that fires wins) + +1. **Lifecycle parity non-negotiable** (same DAB + eval gate + promotion, GA, + fully in `databricks.yml`) → **Custom**. +2. **Orchestration control** (custom state, conditional routing, guardrails, + HITL, Lakebase checkpointer, retry policy) → **Custom**. +3. **Build modality** (managed loop, no custom graph): + - No-code, SME-iterable, broadest tool coverage → **Agent Bricks MAS**. + - Code-first, managed loop, per-request model choice → **Supervisor API**. +4. **Overrides**: HIPAA / enhanced-security or GA-certainty → **Custom**. + >50 sub-agents or users lacking access to some sub-agents → not **MAS**. + +## Pattern 1 — Custom LangGraph supervisor (GA, default) + +A hand-written supervisor graph (via `langgraph-supervisor`'s `create_supervisor` +or a raw `StateGraph` router). It is **just another agent**: served as a +Databricks App via MLflow AgentServer, declared in `databricks.yml`, and gated +by the same CI eval loop. + +- **You control** routing logic, state, guardrails, retries, HITL. +- **Sub-agents** can be `databricks_langchain.GenieAgent`, remote serving + endpoints (your deployed sibling agents), or in-process ReAct agents. +- **Deploy**: `databricks bundle deploy -t dev` — no extra steps. + +Scaffolded files: `src/agents//graph.py` (supervisor), `tools.py`, +`eval/` (inherited), plus an app + experiment in `databricks.yml`. + +## Pattern 2 — Supervisor API (Beta) + +The Databricks-managed supervisor loop (`databricks-openai` Responses API), +wrapped in a one-node LangGraph so it serves through the same App path. You +declare sub-agents as tools; the AI Gateway runs the loop and picks the model +per request. + +- **You write no routing graph.** Databricks owns the loop. +- **Prereqs**: AI Gateway enabled; "Store OpenTelemetry traces in Unity Catalog" + preview enabled; caller has UC access to every sub-agent. +- **Limits**: 30-min background cap; no `stream=True`+`background=True`; no + durable recovery; web search restricted under HIPAA. +- **Deploy**: `databricks bundle deploy -t dev` (it's still a wrapper App). + +## Pattern 3 — Agent Bricks Supervisor / MAS (managed tile) + +The no-code Supervisor tile in Agent Bricks. **It is not a DAB resource and +cannot be created by `bundle deploy`.** This project scaffolds: + +- `notebooks/bootstrap_supervisor_.py` — imperative Beta-SDK provisioning. +- `resources/supervisor__bootstrap.yml` — a bundle-declared job that runs it. + +Run the job (or the notebook) to create the tile out-of-band, then record the +resulting endpoint in `.agentops-stacks/manifest.yml` (`supervisor.endpoint`). +Other agents can then **consume** that endpoint as a `serving_endpoint`. + +- **Best for** no-code assembly, SME iteration, broadest managed tool coverage + (Genie, KA, endpoints, UC functions, dashboards, AI Search, MCP, nested + supervisors, web search). +- **Limits**: max 50 sub-agents; every end user must have access to every + sub-agent; AI Search = Delta Sync only; web search restricted under HIPAA. + +> **Why a notebook here and not `databricks.yml`?** Per this project's standing +> requirement, notebooks only create resources DAB doesn't yet support, and we +> document why. Migrate the MAS tile to a native resource when one exists. + +## Manifest contract + +Whichever pattern you pick, it is recorded in `.agentops-stacks/manifest.yml`: + +```yaml +supervisor: + type: custom | supervisor_api | agent_bricks_mas + name: + routes: [agent_a, agent_b] + endpoint: "" # agent_bricks_mas only — the provisioned managed endpoint +``` + +CI and tooling read this contract without needing to know how the supervisor +was created. + +## Security posture + +- **Custom** — you own guardrails; scope sub-agent auth via MLflow `resources`; + least-privilege per endpoint. In workspace boundary. +- **Supervisor API** — authorization respects the caller's UC permissions; + traces to UC (OTel preview); web search restricted under HIPAA; Beta. +- **Agent Bricks MAS** — end user needs access to every sub-agent; web search + restricted under HIPAA/enhanced-security; provisioning SP ≠ bundle SP — + document grants. diff --git a/tests/test_add_supervisor.py b/tests/test_add_supervisor.py new file mode 100644 index 0000000..8464a8e --- /dev/null +++ b/tests/test_add_supervisor.py @@ -0,0 +1,246 @@ +"""Tests for add_supervisor.py — the /add-supervisor engine. + +These build a minimal fake project tree (no Databricks CLI needed) and run the +script's functions directly, asserting the three supervisor patterns wire into +databricks.yml, the manifest, and the agent/App or bootstrap-job layout as +designed. Complements test_create_project.py (which covers `bundle init`). +""" + +import importlib.util +import sys +import textwrap +from pathlib import Path + +import pytest + +SCRIPT = ( + Path(__file__).parent.parent + / "plugin" / "skills" / "agentops-stacks" / "scripts" / "add_supervisor.py" +) + + +def _load_module(): + spec = importlib.util.spec_from_file_location("add_supervisor", SCRIPT) + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) + return mod + + +addsup = _load_module() + + +# --------------------------------------------------------------------------- # +# Fixtures — a minimal AgentOps Stacks project with one agent. +# --------------------------------------------------------------------------- # + +DATABRICKS_YML = """\ +bundle: + name: my_proj + engine: direct + +include: + - ./resources/experiment.yml + +resources: + apps: + rag: + name: "my-proj-rag" + source_code_path: ./src/agents/rag + +sync: + include: + - src/agents/** + +targets: + dev: + default: true +""" + +MANIFEST_YML = """\ +contract_version: 5 +project_name: my_proj +agents: + - name: rag +""" + +PYPROJECT = """\ +[project] +name = "my_proj" +dependencies = [ + "mlflow>=3.10.0", + "langgraph>=1.1.0", + "python-dotenv>=1.2.1", +] +""" + + +@pytest.fixture +def project(tmp_path): + root = tmp_path / "my_proj" + agent = root / "src" / "agents" / "rag" + (agent / "app").mkdir(parents=True) + (agent / "eval").mkdir(parents=True) + (root / ".agentops-stacks").mkdir(parents=True) + (root / "resources").mkdir(parents=True) + + (root / "databricks.yml").write_text(DATABRICKS_YML) + (root / ".agentops-stacks" / "manifest.yml").write_text(MANIFEST_YML) + + # Minimal source agent files that add_supervisor copies/overwrites. + (agent / "agent.py").write_text("# rag agent\nfrom graph import graph\n") + (agent / "graph.py").write_text("# rag graph\n") + (agent / "tools.py").write_text("def get_tools():\n return []\n") + (agent / "pyproject.toml").write_text(PYPROJECT) + (agent / "app.yaml").write_text("command: []\n") + (agent / "app" / "start_server.py").write_text('AgentServer("rag")\n') + (agent / "eval" / "gates.yml").write_text("block:\n - safety:\n floor: 4.0\n") + return root + + +def read(root, rel): + return (Path(root) / rel).read_text() + + +def exists(root, rel): + return (Path(root) / rel).exists() + + +# --------------------------------------------------------------------------- # +# Discovery helpers +# --------------------------------------------------------------------------- # + +def test_find_project_root(project): + nested = project / "src" / "agents" / "rag" + assert addsup.find_project_root(nested) == project.resolve() + + +def test_find_existing_agents(project): + assert addsup.find_existing_agents(project) == ["rag"] + + +def test_get_project_name(project): + assert addsup.get_project_name(project) == "my_proj" + + +# --------------------------------------------------------------------------- # +# custom supervisor +# --------------------------------------------------------------------------- # + +def test_custom_scaffolds_agent_app(project): + addsup.scaffold_supervisor_agent(project, "router", "custom", "rag", ["rag", "support"]) + # New agent dir with the supervisor graph + tools. + assert exists(project, "src/agents/router/graph.py") + assert exists(project, "src/agents/router/eval/gates.yml") # inherited → CI gates it + graph = read(project, "src/agents/router/graph.py") + assert "create_supervisor" in graph + assert "'rag'" in graph and "'support'" in graph + # Dependency added. + assert "langgraph-supervisor" in read(project, "src/agents/router/pyproject.toml") + + +def test_custom_overwrites_agent_py(project): + # Base agent.py imports `graph` symbols; the supervisor must overwrite it + # with its own handler so startup imports resolve against the new graph.py. + addsup.scaffold_supervisor_agent(project, "router", "custom", "rag", ["rag"]) + agent_py = read(project, "src/agents/router/agent.py") + assert "AgentServer handlers (supervisor)" in agent_py + assert "get_async_checkpointer" not in agent_py # not carried over from a lakebase base + assert "from graph import graph" in agent_py + + +def test_custom_overwrites_agent_py_even_with_lakebase_base(project): + # Simulate a base agent that had Lakebase memory: its agent.py imports + # get_async_checkpointer. The supervisor overwrite must drop that import. + base = project / "src" / "agents" / "rag" / "agent.py" + base.write_text("from graph import graph_builder, get_async_checkpointer\n") + addsup.scaffold_supervisor_agent(project, "router", "custom", "rag", ["rag"]) + assert "get_async_checkpointer" not in read(project, "src/agents/router/agent.py") + + +def test_custom_appends_databricks_yml(project): + addsup.scaffold_supervisor_agent(project, "router", "custom", "rag", ["rag"]) + addsup.append_agent_resources_to_databricks_yml(project, "router") + yml = read(project, "databricks.yml") + assert "router_experiment:" in yml + assert "source_code_path: ./src/agents/router" in yml + assert "my-proj-router" in yml + # App block inserted before sync:, experiment created a new experiments block. + assert yml.index("router:") < yml.index("sync:") + + +def test_manifest_records_custom_supervisor(project): + addsup.update_manifest(project, "router", "custom", ["rag", "support"]) + m = read(project, ".agentops-stacks/manifest.yml") + assert "supervisor:" in m + assert "type: custom" in m + assert "name: router" in m + assert "- rag" in m and "- support" in m + assert "endpoint:" not in m # only MAS gets an endpoint field + + +# --------------------------------------------------------------------------- # +# supervisor_api +# --------------------------------------------------------------------------- # + +def test_supervisor_api_graph_and_dep(project): + addsup.scaffold_supervisor_agent(project, "router", "supervisor_api", "rag", ["rag"]) + graph = read(project, "src/agents/router/graph.py") + assert "DatabricksOpenAI" in graph + assert "responses.create" in graph + assert "databricks-openai" in read(project, "src/agents/router/pyproject.toml") + + +def test_manifest_records_supervisor_api(project): + addsup.update_manifest(project, "router", "supervisor_api", ["rag"]) + m = read(project, ".agentops-stacks/manifest.yml") + assert "type: supervisor_api" in m + + +# --------------------------------------------------------------------------- # +# agent_bricks_mas — NOT a DAB resource; bootstrap job + notebook only. +# --------------------------------------------------------------------------- # + +def test_mas_scaffolds_bootstrap_job_and_notebook(project): + addsup.scaffold_mas_bootstrap(project, "ops_mas", ["rag", "support"]) + assert exists(project, "notebooks/bootstrap_supervisor_ops_mas.py") + assert exists(project, "resources/supervisor_ops_mas_bootstrap.yml") + # Bootstrap job is included in databricks.yml. + yml = read(project, "databricks.yml") + assert "./resources/supervisor_ops_mas_bootstrap.yml" in yml + # The job is a real DAB resource; the tile it creates is documented as not one. + job = read(project, "resources/supervisor_ops_mas_bootstrap.yml") + assert "jobs:" in job + assert "notebook_task" in job + nb = read(project, "notebooks/bootstrap_supervisor_ops_mas.py") + assert "not a Declarative Automation Bundle resource" in nb.lower() or \ + "not a declarative automation bundle resource" in nb.lower() + + +def test_mas_does_not_create_agent_app(project): + addsup.scaffold_mas_bootstrap(project, "ops_mas", ["rag"]) + # MAS must NOT create an src/agents App — it's a managed tile. + assert not exists(project, "src/agents/ops_mas") + + +def test_manifest_records_mas_with_endpoint_field(project): + addsup.update_manifest(project, "ops_mas", "agent_bricks_mas", ["rag"]) + m = read(project, ".agentops-stacks/manifest.yml") + assert "type: agent_bricks_mas" in m + assert "endpoint:" in m # placeholder to fill after provisioning + + +# --------------------------------------------------------------------------- # +# Guards +# --------------------------------------------------------------------------- # + +def test_manifest_not_double_written(project): + addsup.update_manifest(project, "router", "custom", ["rag"]) + addsup.update_manifest(project, "router2", "custom", ["rag"]) + m = read(project, ".agentops-stacks/manifest.yml") + assert m.count("supervisor:") == 1 # second call is a no-op + + +def test_dep_not_duplicated(project): + addsup.scaffold_supervisor_agent(project, "router", "custom", "rag", ["rag"]) + py = read(project, "src/agents/router/pyproject.toml") + assert py.count("langgraph-supervisor") == 1 diff --git a/workflows/single-account-single-agent.json b/workflows/single-account-single-agent.json index 28e3d07..457f51e 100644 --- a/workflows/single-account-single-agent.json +++ b/workflows/single-account-single-agent.json @@ -332,6 +332,25 @@ "git_flow": "dev branch → PR to main (CI gate) → merge to main → merge to release (CD to prod)", "unity_catalog": "Prod catalog is READ-ONLY from dev. Agent reads from dev catalog in dev/staging; prod agent uses prod catalog exclusively.", "mlflow_spine": "One MLflow tracking server per workspace. Experiment per agent per environment. @champion alias governs what runs in production.", - "escalation_path": "Step fails after max_retries → Slack DM to team lead → human reviews MLflow trace → documents root cause → unblocks workflow" + "escalation_path": "Step fails after max_retries → Slack DM to team lead → human reviews MLflow trace → documents root cause → unblocks workflow", + "supervisor_selection": { + "when": "Add a supervisor once the project has >1 agent (post-scaffold pattern, via /add-supervisor). A supervisor routes user queries across agents and other managed sub-agents.", + "default": "custom", + "decision_matrix": [ + {"gate": "D1", "test": "Lifecycle parity non-negotiable — same DAB + eval gate + dev/staging/prod promotion, GA, fully in databricks.yml", "choose": "custom"}, + {"gate": "D2", "test": "Orchestration control needed — custom state, conditional routing, guardrails, HITL, Lakebase checkpointer, retry policy", "choose": "custom"}, + {"gate": "D3a", "test": "Managed loop, no-code, SME-iterable, broadest managed tool coverage (dashboards, AI Search, nested supervisors, web search); accepts UI-first + Beta SDK + out-of-band provisioning", "choose": "agent_bricks_mas"}, + {"gate": "D3b", "test": "Managed loop, code-first, no custom graph, per-request model choice, packaged as a bundle App", "choose": "supervisor_api"}, + {"gate": "D4", "test": "HIPAA/enhanced-security or GA-certainty required (override)", "choose": "custom"}, + {"gate": "D4", "test": ">50 sub-agents or users lacking access to some sub-agents (override — rules out MAS)", "choose": "custom_or_supervisor_api"} + ], + "patterns": { + "custom": {"status": "GA", "loop_owner": "your code", "artifact": "agent App under src/agents/", "dab_declarable": true, "eval_gated": true}, + "supervisor_api": {"status": "Beta", "loop_owner": "Databricks (managed)", "artifact": "wrapper App under src/agents/", "dab_declarable": true, "eval_gated": true, "prereqs": "AI Gateway + UC OTel-traces preview"}, + "agent_bricks_mas": {"status": "UI GA; SDK Beta", "loop_owner": "Databricks (managed tile)", "artifact": "none — consumed endpoint; provisioned by an imperative bootstrap job", "dab_declarable": false, "eval_gated": "verify per-workspace", "limits": "max 50 sub-agents; end user must have access to every sub-agent"} + }, + "manifest_contract": "Recorded in .agentops-stacks/manifest.yml under `supervisor:` with type, name, routes, and (MAS only) endpoint. CI/tooling read this without knowing how the supervisor was created.", + "docs": "docs/supervisor-patterns.md (rendered project)" + } } }