diff --git a/.github/workflows/expert-e2e.yml b/.github/workflows/expert-e2e.yml index 45592c1..a3fe65b 100644 --- a/.github/workflows/expert-e2e.yml +++ b/.github/workflows/expert-e2e.yml @@ -25,6 +25,13 @@ on: required: false type: string default: agent_schema.yaml + agent: + description: >- + Agent name (as declared in expert.toml or the name of a sibling + dir containing agent_schema.yaml). Omit for single-agent repos. + required: false + type: string + default: "" suite: description: 'Robot suite stem to run (e.g. "05_ask_latency"). Omit for all.' required: false @@ -102,6 +109,9 @@ jobs: id: test run: | args=() + if [ -n "${{ inputs.agent }}" ]; then + args+=(--agent "${{ inputs.agent }}") + fi if [ -n "${{ inputs.suite }}" ]; then args+=(--suite "${{ inputs.suite }}") fi diff --git a/README.md b/README.md index 17e9896..2616a79 100644 --- a/README.md +++ b/README.md @@ -223,10 +223,59 @@ expert sync Push docs + rebuild Context Cache expert ask "" Stream answer from a deployed agent expert sessions list/delete Manage user sessions (LGPD) expert test Run the packaged E2E Robot Framework kit +expert agents List agents known to this workspace +expert use Pin an agent as the active one for this workspace +expert which Print the agent a bare command would resolve to ``` Every command supports `--help` for full options. +### Multi-agent workspaces + +Private repositories often host several specialists side-by-side. `expert` +auto-detects a multi-agent layout and lets you target any agent with three +equivalent syntaxes: + +```bash +# One-off shortcut — great for quick hops between agents. +expert @ecg ask "What does lead V1 tell us?" +expert @derm ask "Differential for an acral nodule?" + +# Explicit flag — CI-friendly, unambiguous. +expert ask --agent ecg "..." + +# Pin for the current workspace (stored in .expert/state.json). +expert use ecg +expert ask "..." # now routes to ecg +expert use --clear # undo +``` + +Discovery order (first match wins): + +1. `expert.toml` at the workspace root (authoritative — aliases, endpoints, defaults). +2. Any sibling directory containing `agent_schema.yaml` (auto-discovery). +3. Fallback: single-agent mode against `./agent_schema.yaml`. + +An `expert.toml` looks like this: + +```toml +[defaults] +agent = "ecg" # used when no flag / pin / env var is set + +[agents.ecg] +schema = "ecg-expert/agent_schema.yaml" +endpoint = "https://ecg-agent.example.com" +api_key_env = "ECG_ADMIN_KEY" # reads from env at runtime +description = "12-lead ECG specialist" + +[agents.derm] +schema = "derm-expert/agent_schema.yaml" +``` + +Prefix matching is supported: `expert @ec ask "..."` resolves to `ecg` if +no other agent name starts with `ec`. Ambiguous prefixes raise a friendly +error listing all candidates. + --- ## End-to-end testing diff --git a/cli/expert/commands/agents.py b/cli/expert/commands/agents.py new file mode 100644 index 0000000..406b539 --- /dev/null +++ b/cli/expert/commands/agents.py @@ -0,0 +1,131 @@ +"""Workspace-aware agent management commands: ``agents``, ``use``, ``which``. + +These are the only commands that *never* need to resolve to a single agent — +they inspect, select, or describe the workspace itself. +""" + +from __future__ import annotations + +from typing import Annotated + +import typer +from rich.table import Table + +from ..ui import console, print_error, print_info, print_success +from ..workspace import ( + AgentNotFoundError, + AmbiguousAgentError, + Workspace, + WorkspaceError, +) + + +def agents_cmd( + verbose: Annotated[ + bool, + typer.Option("--verbose", "-v", help="Show schema paths and endpoints."), + ] = False, +) -> None: + """List every agent known to this workspace.""" + ws = Workspace.discover() + agents = ws.agents() + if not agents: + print_info( + "No agents found. Scaffold one with `expert init ` or create " + "an `expert.toml` workspace file." + ) + return + + active = ws.active() + table = Table(title=f"Agents — workspace: {ws.root}") + table.add_column("Active", width=6, justify="center") + table.add_column("Name", style="bold") + table.add_column("Source", style="dim") + if verbose: + table.add_column("Schema") + table.add_column("Endpoint") + table.add_column("Description", overflow="fold") + + for info in agents: + is_active = "✓" if info.name == active else "" + row = [ + is_active, + info.name, + info.source, + ] + if verbose: + try: + schema_rel = str(info.schema_path.relative_to(ws.root)) + except ValueError: + schema_rel = str(info.schema_path) + row.extend([schema_rel, info.endpoint or "—"]) + row.append(info.description or "") + table.add_row(*row) + + console.print(table) + if ws.default_agent: + print_info(f"default (expert.toml): [cyan]{ws.default_agent}[/cyan]") + if active: + print_info(f"active (expert use): [cyan]{active}[/cyan]") + + +def use_cmd( + name: Annotated[ + str | None, + typer.Argument( + help="Agent name to pin as active. Omit to clear the pin.", + ), + ] = None, + clear: Annotated[ + bool, + typer.Option("--clear", help="Remove the active-agent pointer."), + ] = False, +) -> None: + """Pin an agent as the active one for this workspace (stored locally).""" + ws = Workspace.discover() + + if clear or (name is None): + if ws.state_file.is_file(): + ws.clear_active() + print_success("Cleared active agent pointer.") + else: + print_info("No active agent set.") + return + + try: + # Re-use matcher so `expert use derm` works when `derm-expert` is declared. + canonical = ws._match(name) + ws.set_active(canonical) + except (AgentNotFoundError, AmbiguousAgentError, WorkspaceError) as exc: + print_error(str(exc)) + raise typer.Exit(code=1) from exc + + print_success(f"Active agent set to [cyan]{canonical}[/cyan].") + print_info(f"State stored in {ws.state_file}") + + +def which_cmd( + agent: Annotated[ + str | None, + typer.Option( + "--agent", + "-a", + help="Preview resolution for the given selector without running anything.", + ), + ] = None, +) -> None: + """Print the agent a bare command (no --agent, no @alias) would resolve to.""" + ws = Workspace.discover() + try: + ctx = ws.resolve(selector=agent) + except (AgentNotFoundError, AmbiguousAgentError, WorkspaceError) as exc: + print_error(str(exc)) + raise typer.Exit(code=1) from exc + + print_info(f"Active agent: [bold cyan]{ctx.name}[/bold cyan] (source: {ctx.selector_source})") + print_info(f" schema: {ctx.schema_path}") + print_info(f" endpoint: {ctx.endpoint or '—'}") + print_info(f" api key: {'set' if ctx.api_key else '—'}") + + +__all__ = ["agents_cmd", "use_cmd", "which_cmd"] diff --git a/cli/expert/commands/ask.py b/cli/expert/commands/ask.py index b204443..0d18356 100644 --- a/cli/expert/commands/ask.py +++ b/cli/expert/commands/ask.py @@ -44,6 +44,7 @@ from rich.text import Text from ..config import make_http_client +from ..context import resolve as resolve_context from ..ui import console, print_error, print_info, print_success _USER_ID = "cli" @@ -56,22 +57,26 @@ def cmd( question: Annotated[str, typer.Argument(help="Question to send to the agent.")], + agent: Annotated[ + str | None, + typer.Option("--agent", "-a", help="Agent name from the workspace."), + ] = None, endpoint: Annotated[ - str, + str | None, typer.Option( "--endpoint", envvar="EXPERT_AGENT_ENDPOINT", - help="Base URL of the running agent.", + help="Override the agent's endpoint.", ), - ], + ] = None, api_key: Annotated[ - str, + str | None, typer.Option( "--api-key", envvar="EXPERT_AGENT_API_KEY", - help="Admin bearer token.", + help="Override the agent's admin bearer token.", ), - ], + ] = None, session: Annotated[ str | None, typer.Option( @@ -88,6 +93,15 @@ def cmd( ] = True, ) -> None: """Ask the agent a question.""" + ctx = resolve_context( + agent=agent, + endpoint=endpoint, + api_key=api_key, + require_remote=True, + ) + endpoint, api_key = ctx.require_remote() + if ctx.selector_source not in ("single", "schema-flag"): + print_info(f"→ [cyan]{ctx.name}[/cyan] ({ctx.selector_source})") if session is None: session = str(uuid.uuid4()) print_info(f"Starting new session [cyan]{session}[/cyan].") diff --git a/cli/expert/commands/count_tokens.py b/cli/expert/commands/count_tokens.py index b71b1e8..1889a82 100644 --- a/cli/expert/commands/count_tokens.py +++ b/cli/expert/commands/count_tokens.py @@ -15,6 +15,7 @@ import typer from app.schema import AgentSchema +from ..context import resolve as resolve_context from ..ui import console, print_error, print_info, print_success, print_warning if TYPE_CHECKING: @@ -140,20 +141,31 @@ def cmd( help="API key for google-genai token counting.", ), ], + agent: Annotated[ + str | None, + typer.Option("--agent", "-a", help="Agent name from the workspace."), + ] = None, schema_path: Annotated[ - Path, - typer.Option("--schema", "-s", help="Path to agent_schema.yaml."), - ] = Path("./agent_schema.yaml"), + Path | None, + typer.Option( + "--schema", + "-s", + help="Explicit path to agent_schema.yaml (bypasses workspace resolution).", + ), + ] = None, model: Annotated[ str, typer.Option("--model", help="Model used for the count_tokens API call."), ] = "gemini-2.0-flash-exp", ) -> None: """Walk the knowledge base and sum the estimated token count per file.""" - schema_path = schema_path.resolve() + ctx = resolve_context(agent=agent, schema=schema_path) + schema_path = ctx.schema_path if not schema_path.is_file(): print_error(f"schema file not found: {schema_path}") raise typer.Exit(code=1) + if ctx.selector_source not in ("single", "schema-flag"): + print_info(f"agent [cyan]{ctx.name}[/cyan] ({ctx.selector_source})") try: schema = AgentSchema.from_yaml(schema_path) diff --git a/cli/expert/commands/init.py b/cli/expert/commands/init.py index 600ab68..aea98fd 100644 --- a/cli/expert/commands/init.py +++ b/cli/expert/commands/init.py @@ -217,5 +217,24 @@ def cmd( raise typer.Exit(code=1) from exc print_success(f"Created new agent at [cyan]{path}[/cyan].") + _print_workspace_hint(path, name) print_info("Next step: [bold]expert validate --schema ./agent_schema.yaml[/bold]") console.print() + + +def _print_workspace_hint(path: Path, name: str) -> None: + """If the new agent lives inside a multi-agent workspace, nudge the user.""" + from ..workspace import Workspace + + parent = path.parent + try: + ws = Workspace.discover(cwd=parent) + except Exception: # pragma: no cover - discovery is best-effort here + return + + # Only hint when there's >1 agent (either discovered or declared). + if len(ws.agents_by_name) >= 2: + print_info( + f"Detected multi-agent workspace at [cyan]{ws.root}[/cyan]. " + f"Use [bold]expert agents[/bold] to list, or [bold]expert @{name} [/bold]." + ) diff --git a/cli/expert/commands/sessions.py b/cli/expert/commands/sessions.py index 6f37c58..74b231a 100644 --- a/cli/expert/commands/sessions.py +++ b/cli/expert/commands/sessions.py @@ -11,6 +11,7 @@ from rich.table import Table from ..config import make_http_client +from ..context import resolve as resolve_context from ..ui import console, print_error, print_info, print_success, print_warning app = typer.Typer( @@ -20,6 +21,23 @@ ) +def _remote( + agent: str | None, + endpoint_override: str | None, + api_key_override: str | None, +) -> tuple[str, str]: + """Resolve (endpoint, api_key) for every session command via the workspace.""" + ctx = resolve_context( + agent=agent, + endpoint=endpoint_override, + api_key=api_key_override, + require_remote=True, + ) + if ctx.selector_source not in ("single", "schema-flag"): + print_info(f"→ [cyan]{ctx.name}[/cyan] ({ctx.selector_source})") + return ctx.require_remote() + + async def _get_json(endpoint: str, api_key: str, path: str) -> Any: async with make_http_client(endpoint=endpoint, api_key=api_key) as client: response = await client.get(path) @@ -51,30 +69,44 @@ def _run(coro: Any) -> Any: raise typer.Exit(code=2) from exc +_AgentOpt = Annotated[ + str | None, + typer.Option("--agent", "-a", help="Agent name from the workspace."), +] _EndpointOpt = Annotated[ - str, - typer.Option("--endpoint", envvar="EXPERT_AGENT_ENDPOINT", help="Base URL of the agent."), + str | None, + typer.Option( + "--endpoint", + envvar="EXPERT_AGENT_ENDPOINT", + help="Override the agent's endpoint.", + ), ] _ApiKeyOpt = Annotated[ - str, - typer.Option("--api-key", envvar="EXPERT_AGENT_API_KEY", help="Admin bearer token."), + str | None, + typer.Option( + "--api-key", + envvar="EXPERT_AGENT_API_KEY", + help="Override the agent's admin bearer token.", + ), ] @app.command("list") def list_cmd( - endpoint: _EndpointOpt, - api_key: _ApiKeyOpt, + agent: _AgentOpt = None, + endpoint: _EndpointOpt = None, + api_key: _ApiKeyOpt = None, user: Annotated[ str | None, typer.Option("--user", help="Filter sessions by user_id."), ] = None, ) -> None: """List active sessions.""" + endpoint_resolved, api_key_resolved = _remote(agent, endpoint, api_key) path = "/sessions" if user: path = f"/sessions?user_id={user}" - body = _run(_get_json(endpoint.rstrip("/"), api_key, path)) + body = _run(_get_json(endpoint_resolved, api_key_resolved, path)) items: list[dict[str, Any]] if isinstance(body, list): items = [x for x in body if isinstance(x, dict)] @@ -105,11 +137,13 @@ def list_cmd( @app.command("show") def show_cmd( session_id: Annotated[str, typer.Argument(help="Session ID.")], - endpoint: _EndpointOpt, - api_key: _ApiKeyOpt, + agent: _AgentOpt = None, + endpoint: _EndpointOpt = None, + api_key: _ApiKeyOpt = None, ) -> None: """Show the message history of a single session.""" - body = _run(_get_json(endpoint.rstrip("/"), api_key, f"/sessions/{session_id}")) + endpoint_resolved, api_key_resolved = _remote(agent, endpoint, api_key) + body = _run(_get_json(endpoint_resolved, api_key_resolved, f"/sessions/{session_id}")) if not isinstance(body, dict): print_error("unexpected response shape.") raise typer.Exit(code=2) @@ -132,14 +166,16 @@ def show_cmd( @app.command("delete") def delete_cmd( session_id: Annotated[str, typer.Argument(help="Session ID to delete.")], - endpoint: _EndpointOpt, - api_key: _ApiKeyOpt, + agent: _AgentOpt = None, + endpoint: _EndpointOpt = None, + api_key: _ApiKeyOpt = None, yes: Annotated[ bool, typer.Option("--yes", "-y", help="Skip the confirmation prompt."), ] = False, ) -> None: """Delete a session and its message history (LGPD right-to-erasure).""" + endpoint_resolved, api_key_resolved = _remote(agent, endpoint, api_key) if not yes: confirmed = typer.confirm( f"Delete session {session_id}? This action is irreversible.", @@ -149,5 +185,5 @@ def delete_cmd( print_warning("Aborted.") raise typer.Exit(code=0) - _run(_delete(endpoint.rstrip("/"), api_key, f"/sessions/{session_id}")) + _run(_delete(endpoint_resolved, api_key_resolved, f"/sessions/{session_id}")) print_success(f"Session [cyan]{session_id}[/cyan] deleted.") diff --git a/cli/expert/commands/sync.py b/cli/expert/commands/sync.py index 92bbcdb..0396323 100644 --- a/cli/expert/commands/sync.py +++ b/cli/expert/commands/sync.py @@ -17,6 +17,7 @@ from app.schema import AgentSchema from ..config import make_http_client +from ..context import resolve as resolve_context from ..ui import console, print_diff_table, print_error, print_info, print_success @@ -85,32 +86,46 @@ async def _post_sync( def cmd( + agent: Annotated[ + str | None, + typer.Option("--agent", "-a", help="Agent name from the workspace."), + ] = None, endpoint: Annotated[ - str, + str | None, typer.Option( "--endpoint", envvar="EXPERT_AGENT_ENDPOINT", - help="Base URL of the running agent.", + help="Override the agent's endpoint (defaults to workspace/env value).", ), - ], + ] = None, api_key: Annotated[ - str, + str | None, typer.Option( "--api-key", envvar="EXPERT_AGENT_API_KEY", - help="Admin bearer token.", + help="Override the agent's admin bearer token.", ), - ], + ] = None, schema_path: Annotated[ - Path, - typer.Option("--schema", "-s", help="Path to agent_schema.yaml."), - ] = Path("./agent_schema.yaml"), + Path | None, + typer.Option("--schema", "-s", help="Explicit path to agent_schema.yaml."), + ] = None, ) -> None: """Upload the local knowledge base and trigger a Context Cache rebuild.""" - schema_path = schema_path.resolve() + ctx = resolve_context( + agent=agent, + schema=schema_path, + endpoint=endpoint, + api_key=api_key, + require_remote=True, + ) + schema_path = ctx.schema_path + endpoint, api_key = ctx.require_remote() if not schema_path.is_file(): print_error(f"schema file not found: {schema_path}") raise typer.Exit(code=1) + if ctx.selector_source not in ("single", "schema-flag"): + print_info(f"agent [cyan]{ctx.name}[/cyan] ({ctx.selector_source})") try: schema = AgentSchema.from_yaml(schema_path) diff --git a/cli/expert/commands/test.py b/cli/expert/commands/test.py index fbc84d6..d305e2c 100644 --- a/cli/expert/commands/test.py +++ b/cli/expert/commands/test.py @@ -18,6 +18,7 @@ import typer +from ..context import resolve as resolve_context from ..ui import console, print_error, print_info, print_success # Canonical order of the packaged suites. The numeric prefixes keep `robot` @@ -33,11 +34,19 @@ def cmd( + agent: Annotated[ + str | None, + typer.Option( + "--agent", + "-a", + help="Agent name from the workspace. Resolved via `expert agents`.", + ), + ] = None, suite: Annotated[ list[str] | None, typer.Option( "--suite", - "-s", + "-S", help=( "Run only the given suite(s) by stem (e.g. '05_ask_latency'). " "Can be passed multiple times. Default: all." @@ -72,16 +81,17 @@ def cmd( Path | None, typer.Option( "--schema", - help="Path to agent_schema.yaml (defaults to env EXPERT_AGENT_SCHEMA).", + "-s", + help="Explicit path to agent_schema.yaml (bypasses workspace resolution).", ), ] = None, endpoint: Annotated[ str | None, - typer.Option("--endpoint", help="Override EXPERT_AGENT_ENDPOINT."), + typer.Option("--endpoint", help="Override the agent's endpoint."), ] = None, api_key: Annotated[ str | None, - typer.Option("--api-key", help="Override EXPERT_AGENT_API_KEY."), + typer.Option("--api-key", help="Override the agent's admin bearer token."), ] = None, dry_run: Annotated[ bool, @@ -118,15 +128,25 @@ def cmd( print_error(f"No suites matched selection {suite!r}. Available: {available}") raise typer.Exit(code=2) - # Propagate overrides to the environment so ExpertLibrary's defaults pick - # them up without needing --var boilerplate in simple cases. - env_overrides: dict[str, str] = {} - if endpoint: - env_overrides["EXPERT_AGENT_ENDPOINT"] = endpoint - if api_key: - env_overrides["EXPERT_AGENT_API_KEY"] = api_key - if schema: - env_overrides["EXPERT_AGENT_SCHEMA"] = str(schema) + # Resolve the agent context (supports --agent / @alias / `expert use`) + # so that the packaged Robot suites see fully-populated env vars even + # in multi-agent workspaces without requiring --var or --endpoint. + # We fall back to a bare resolve (schema-only) so that the offline + # suites still work when endpoint/api_key are not configured. + ctx = resolve_context( + agent=agent, + schema=schema, + endpoint=endpoint, + api_key=api_key, + ) + if ctx.selector_source not in ("single", "schema-flag"): + print_info(f"→ [cyan]{ctx.name}[/cyan] ({ctx.selector_source})") + + env_overrides: dict[str, str] = {"EXPERT_AGENT_SCHEMA": str(ctx.schema_path)} + if ctx.endpoint: + env_overrides["EXPERT_AGENT_ENDPOINT"] = ctx.endpoint + if ctx.api_key: + env_overrides["EXPERT_AGENT_API_KEY"] = ctx.api_key for key, value in env_overrides.items(): os.environ[key] = value diff --git a/cli/expert/commands/validate.py b/cli/expert/commands/validate.py index 7f97500..216f5f6 100644 --- a/cli/expert/commands/validate.py +++ b/cli/expert/commands/validate.py @@ -9,7 +9,8 @@ from app.schema import AgentSchema from pydantic import ValidationError -from ..ui import print_error, print_schema, print_success, print_warning +from ..context import resolve as resolve_context +from ..ui import print_error, print_info, print_schema, print_success, print_warning def _iter_matching_files( @@ -30,16 +31,31 @@ def _iter_matching_files( def cmd( + agent: Annotated[ + str | None, + typer.Option( + "--agent", + "-a", + help="Agent name (from expert.toml or sibling dirs). See `expert agents`.", + ), + ] = None, schema_path: Annotated[ - Path, - typer.Option("--schema", "-s", help="Path to agent_schema.yaml."), - ] = Path("./agent_schema.yaml"), + Path | None, + typer.Option( + "--schema", + "-s", + help="Explicit path to agent_schema.yaml (bypasses workspace resolution).", + ), + ] = None, ) -> None: """Validate an agent schema and its referenced filesystem layout.""" - schema_path = schema_path.resolve() + ctx = resolve_context(agent=agent, schema=schema_path) + schema_path = ctx.schema_path if not schema_path.is_file(): print_error(f"schema file not found: {schema_path}") raise typer.Exit(code=1) + if ctx.selector_source not in ("single", "schema-flag"): + print_info(f"agent [cyan]{ctx.name}[/cyan] ({ctx.selector_source})") try: schema = AgentSchema.from_yaml(schema_path) diff --git a/cli/expert/context.py b/cli/expert/context.py new file mode 100644 index 0000000..ce56787 --- /dev/null +++ b/cli/expert/context.py @@ -0,0 +1,70 @@ +"""Shared helpers used by every command that needs an :class:`AgentContext`. + +Commands should call :func:`resolve` at their very top, forward flag-overrides +in, and then read ``ctx.schema_path`` / ``ctx.endpoint`` / ``ctx.api_key``. + +This keeps the multi-agent resolution logic in one place — if we ever change +precedence rules, every command picks it up automatically. +""" + +from __future__ import annotations + +from dataclasses import replace +from pathlib import Path + +import typer + +from .ui import print_error +from .workspace import ( + AgentContext, + AgentNotFoundError, + AmbiguousAgentError, + Workspace, + WorkspaceError, +) + + +def resolve( + *, + agent: str | None = None, + schema: Path | None = None, + endpoint: str | None = None, + api_key: str | None = None, + require_remote: bool = False, +) -> AgentContext: + """Resolve an :class:`AgentContext` or abort the CLI with a helpful message. + + Flag-level overrides take priority over workspace-derived values so that + scripts / CI can still force an endpoint or API key on a single run + without editing ``expert.toml``. + + When ``require_remote`` is set, missing ``endpoint`` / ``api_key`` turn + into a non-zero exit instead of being silently ``None``. + """ + ws = Workspace.discover() + try: + ctx = ws.resolve(selector=agent, schema_override=schema) + except (AgentNotFoundError, AmbiguousAgentError, WorkspaceError) as exc: + print_error(str(exc)) + raise typer.Exit(code=1) from exc + + # Flag overrides from the caller take precedence over anything the + # workspace resolver produced. + if endpoint or api_key: + ctx = replace( + ctx, + endpoint=endpoint or ctx.endpoint, + api_key=api_key or ctx.api_key, + ) + + if require_remote: + try: + ctx.require_remote() + except WorkspaceError as exc: + print_error(str(exc)) + raise typer.Exit(code=2) from exc + + return ctx + + +__all__ = ["resolve"] diff --git a/cli/expert/main.py b/cli/expert/main.py index dc1779d..d0b4638 100644 --- a/cli/expert/main.py +++ b/cli/expert/main.py @@ -1,12 +1,28 @@ -"""Top-level `typer` app for `expert`.""" +"""Top-level `typer` app for `expert`. + +The CLI is aware of *multi-agent workspaces*: a repo can host several +`agent_schema.yaml` files and the user can target them individually via: + +- Explicit flag: ``expert ask --agent derm "..."`` +- Active pointer: ``expert use derm`` then ``expert ask "..."`` +- Positional shortcut: ``expert @derm ask "..."`` + +The ``@alias`` form is handled **here** in the entrypoint via a small +argv rewriter that runs before Typer parses its arguments. The rewriter +turns ``expert @ ...`` into +``expert --agent ...`` so downstream commands just need +to accept the standard ``--agent`` flag. +""" from __future__ import annotations +import sys from typing import Annotated import typer from . import __version__ +from .commands import agents as agents_commands from .commands import ask, count_tokens, init, sessions, sync, test, validate from .ui import console @@ -41,6 +57,55 @@ def _root( _ = version +# Subcommands that accept `--agent`. Used by the @alias rewriter so that +# nonsense like `expert @derm use ecg` falls through to a useful error +# instead of silently rewriting into `expert use ecg --agent derm`. +_AGENT_AWARE: frozenset[str] = frozenset( + {"ask", "validate", "count-tokens", "sync", "test", "sessions", "which"} +) + + +def _rewrite_at_alias(argv: list[str]) -> list[str]: + """Expand a leading ``@`` token into ``--agent `` further right. + + Examples:: + + expert @ecg ask "hi" → expert ask "hi" --agent ecg + expert @derm sessions list → expert sessions list --agent derm + expert @ecg → expert agents --agent ecg (listing mode) + + Safe no-ops: + + - ``@`` in argv[1] that isn't the immediate prefix to a known + agent-aware subcommand is left alone (so ``expert @derm use foo`` + is *not* silently rewritten). + - Options like ``--foo=@bar`` are never touched because we only look at + ``argv[1]``. + """ + if len(argv) < 2 or not argv[1].startswith("@") or len(argv[1]) < 2: + return argv + if argv[1] in ("@-", "@"): + return argv + alias = argv[1][1:] + rest = argv[2:] + + subcommand_idx: int | None = None + for idx, token in enumerate(rest): + if not token.startswith("-"): + subcommand_idx = idx + break + if subcommand_idx is None or rest[subcommand_idx] not in _AGENT_AWARE: + # No agent-aware subcommand present: leave argv alone so Typer can + # render a useful error instead of rewriting into a wrong shape. + return argv + + # Append `--agent ` at the end so it flows through regardless of + # whether the subcommand is a leaf (`ask`) or a sub-Typer (`sessions + # list`). Typer happily routes the flag to the deepest command that + # declares it. + return [argv[0], *rest, "--agent", alias] + + app.command(name="init", help="Scaffold a new agent project.")(init.cmd) app.command(name="validate", help="Validate an agent_schema.yaml locally.")(validate.cmd) app.command( @@ -54,7 +119,25 @@ def _root( name="test", help="Run the packaged Robot Framework E2E kit against the current agent.", )(test.cmd) +app.command( + name="agents", + help="List agents known to this workspace.", +)(agents_commands.agents_cmd) +app.command( + name="use", + help="Pin an agent as the active one for this workspace.", +)(agents_commands.use_cmd) +app.command( + name="which", + help="Print which agent a bare command would resolve to.", +)(agents_commands.which_cmd) -if __name__ == "__main__": +def main() -> None: + """Entry point that runs the ``@alias`` rewriter before dispatching.""" + sys.argv = _rewrite_at_alias(sys.argv) app() + + +if __name__ == "__main__": + main() diff --git a/cli/expert/workspace.py b/cli/expert/workspace.py new file mode 100644 index 0000000..50508da --- /dev/null +++ b/cli/expert/workspace.py @@ -0,0 +1,460 @@ +"""Multi-agent workspace: discovery, `expert.toml`, and active-agent state. + +A *workspace* is the repository (or subtree) that hosts one or more agent +schemas. The CLI supports three equivalent ways of pointing a command at a +specific agent inside a multi-agent workspace: + +1. **Explicit flag** — ``expert ask --agent derm "hi"``. +2. **Positional `@alias`** — ``expert @derm ask "hi"`` (intercepted in + ``main.py`` and rewritten into the flag above, transparently). +3. **Active pointer** — ``expert use derm`` persists a pointer in + ``.expert/state.json`` so subsequent ``expert ask "..."`` calls in that + cwd stay on ``derm`` until the user runs ``expert use`` again. + +When none of these disambiguate an unambiguous single agent, commands raise +:class:`AmbiguousAgentError` with a helpful message listing the candidates. + +## Discovery + +Workspace detection walks up from ``cwd`` looking for the first parent that +contains **any** of these markers: + +- ``expert.toml`` (explicit, strongest signal — anchors the workspace). +- ``.expert/state.json`` (previously `expert use`-d directory). +- a sibling pattern of ``*/agent_schema.yaml`` (multi-agent repo by + convention). + +If none is found the workspace defaults to a *single-agent* mode rooted at +cwd, preserving the historical behaviour (``./agent_schema.yaml``). + +## ``expert.toml`` schema + +```toml +# Optional per-workspace defaults. +[defaults] +agent = "ecg" # Default agent when no flag / active pointer is set. + +# One section per agent. The key becomes the canonical name. +[agents.ecg] +schema = "ecg-expert/agent_schema.yaml" # Required. Relative to this file. +endpoint = "https://ecg-xxx.a.run.app" # Optional override. +api_key_env = "ECG_ADMIN_KEY" # Optional. Takes precedence over api_key. +api_key = "..." # Optional, discouraged (use env). +description = "ECG-specialist clinical agent." # Optional free-form. + +[agents.derm] +schema = "derm-expert/agent_schema.yaml" +``` + +Any agent that is **auto-discovered** via ``*/agent_schema.yaml`` but not +explicitly declared in ``expert.toml`` is still selectable by its directory +name, and inherits endpoint/api_key from the global ``EXPERT_AGENT_*`` env +vars. +""" + +from __future__ import annotations + +import json +import os +import tomllib +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any + +_STATE_DIR = ".expert" +_STATE_FILE = "state.json" +_WORKSPACE_FILE = "expert.toml" +_SCHEMA_FILENAME = "agent_schema.yaml" +_DISCOVERY_MAX_DEPTH = 3 +_ENV_ACTIVE_AGENT = "EXPERT_AGENT" + + +class WorkspaceError(RuntimeError): + """Base for workspace-related errors. Carries an exit-code hint.""" + + exit_code: int = 1 + + +class AgentNotFoundError(WorkspaceError): + """Raised when the caller names an agent that doesn't exist in the workspace.""" + + +class AmbiguousAgentError(WorkspaceError): + """Raised when a selector matches zero, or more than one, agents. + + ``candidates`` holds every known agent so callers can render a helpful + prompt/error with the available options. + """ + + def __init__(self, message: str, *, candidates: list[AgentInfo]) -> None: + super().__init__(message) + self.candidates = candidates + + +@dataclass(frozen=True) +class AgentInfo: + """Metadata about an agent known to the workspace (pre-resolution).""" + + name: str + schema_path: Path + endpoint: str | None = None + api_key: str | None = None + description: str | None = None + # "toml" — declared in expert.toml; "auto" — discovered by convention. + source: str = "auto" + + +@dataclass(frozen=True) +class AgentContext: + """Fully-resolved agent context a command can rely on. + + ``api_key`` / ``endpoint`` may still be ``None`` if the agent is offline + (e.g. for ``expert validate`` which only needs the schema). Commands that + require remote access should call :meth:`require_remote` instead of + reading the fields directly. + """ + + name: str + schema_path: Path + endpoint: str | None + api_key: str | None + description: str | None + selector_source: str # "flag", "@alias", "active", "env", "default", "auto", "single" + + def require_remote(self) -> tuple[str, str]: + """Return ``(endpoint, api_key)``, raising a user-friendly error if missing.""" + if not self.endpoint or not self.api_key: + raise WorkspaceError( + f"Agent '{self.name}' has no endpoint/api_key configured. " + "Set EXPERT_AGENT_ENDPOINT + EXPERT_AGENT_API_KEY, or declare " + "them in expert.toml under [agents." + f"{self.name}]." + ) + return self.endpoint.rstrip("/"), self.api_key + + +@dataclass +class Workspace: + """Discovered multi-agent workspace rooted at ``root``.""" + + root: Path + agents_by_name: dict[str, AgentInfo] = field(default_factory=dict) + default_agent: str | None = None + # True when no expert.toml AND no sibling schemas found — legacy single-agent mode. + single_agent_mode: bool = False + + @classmethod + def discover(cls, *, cwd: Path | None = None) -> Workspace: + """Discover the workspace rooted at (or above) ``cwd``.""" + start = (cwd or Path.cwd()).resolve() + root, toml_path = _find_workspace_root(start) + ws = cls(root=root) + + if toml_path is not None: + ws._load_toml(toml_path) + + # Auto-discover siblings regardless of whether a TOML exists — the TOML + # only adds aliases/metadata, it doesn't preclude extra agents shipped + # in sibling dirs. + ws._discover_siblings() + + if not ws.agents_by_name: + # Legacy single-agent mode: one schema next to the user's cwd. + local = start / _SCHEMA_FILENAME + if local.is_file(): + ws.agents_by_name["."] = AgentInfo( + name=".", + schema_path=local, + source="single", + ) + ws.single_agent_mode = True + + return ws + + # --------------------------- TOML loading --------------------------- # + + def _load_toml(self, path: Path) -> None: + try: + with path.open("rb") as fh: + raw = tomllib.load(fh) + except (OSError, tomllib.TOMLDecodeError) as exc: # pragma: no cover - rare + raise WorkspaceError(f"failed to parse {path}: {exc}") from exc + + defaults = raw.get("defaults") if isinstance(raw.get("defaults"), dict) else {} + default_name = defaults.get("agent") if isinstance(defaults, dict) else None + if isinstance(default_name, str): + self.default_agent = default_name + + agents_section = raw.get("agents") if isinstance(raw.get("agents"), dict) else {} + if not isinstance(agents_section, dict): + return + + for name, body in agents_section.items(): + if not isinstance(name, str) or not isinstance(body, dict): + continue + schema_rel = body.get("schema") + if not isinstance(schema_rel, str) or not schema_rel: + raise WorkspaceError(f"expert.toml: agent '{name}' is missing a 'schema' field.") + schema_abs = (path.parent / schema_rel).resolve() + api_key = _resolve_api_key(body) + self.agents_by_name[name] = AgentInfo( + name=name, + schema_path=schema_abs, + endpoint=_opt_str(body.get("endpoint")), + api_key=api_key, + description=_opt_str(body.get("description")), + source="toml", + ) + + # --------------------------- Auto-discovery ------------------------- # + + def _discover_siblings(self) -> None: + """Walk immediate children of ``root`` for ``*/agent_schema.yaml``.""" + if not self.root.is_dir(): + return + for child in sorted(self.root.iterdir()): + if not child.is_dir() or child.name.startswith("."): + continue + schema = child / _SCHEMA_FILENAME + if not schema.is_file(): + continue + # Skip if already declared via TOML under a different key — the + # TOML entry is authoritative for that schema. + if any(info.schema_path == schema for info in self.agents_by_name.values()): + continue + # Skip if the directory name collides with a declared TOML name; + # declared ones win. + if child.name in self.agents_by_name: + continue + self.agents_by_name[child.name] = AgentInfo( + name=child.name, + schema_path=schema, + source="auto", + ) + + # --------------------------- State file ----------------------------- # + + @property + def state_file(self) -> Path: + return self.root / _STATE_DIR / _STATE_FILE + + def active(self) -> str | None: + """Return the agent name pinned via ``expert use``, if any.""" + path = self.state_file + if not path.is_file(): + return None + try: + data = json.loads(path.read_text()) + except (OSError, json.JSONDecodeError): + return None + name = data.get("agent") if isinstance(data, dict) else None + return name if isinstance(name, str) else None + + def set_active(self, name: str) -> None: + if name not in self.agents_by_name: + raise AgentNotFoundError( + f"Unknown agent '{name}'. Run `expert agents` to list candidates." + ) + path = self.state_file + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps({"agent": name}, indent=2) + "\n") + + def clear_active(self) -> None: + path = self.state_file + if path.is_file(): + path.unlink() + + # --------------------------- Listing -------------------------------- # + + def agents(self) -> list[AgentInfo]: + return sorted(self.agents_by_name.values(), key=lambda a: a.name) + + # --------------------------- Resolution ----------------------------- # + + def resolve( + self, + selector: str | None = None, + *, + env: dict[str, str] | None = None, + schema_override: Path | None = None, + ) -> AgentContext: + """Return a fully-resolved :class:`AgentContext`. + + Resolution order (first match wins): + + 1. Explicit ``selector`` (from ``--agent`` or ``@alias``). + 2. ``EXPERT_AGENT`` env var. + 3. ``.expert/state.json`` (set by ``expert use``). + 4. ``[defaults] agent = "..."`` in ``expert.toml``. + 5. Exactly-one-agent short-circuit. + 6. ``schema_override`` (``--schema`` flag, purely file-based fallback). + + Fails with :class:`AmbiguousAgentError` otherwise. + """ + env = env if env is not None else dict(os.environ) + source: str + name: str | None = None + + # An explicit --schema path short-circuits resolution entirely: + # the caller is telling us "use this file, don't touch the + # workspace". This mirrors the pre-multi-agent CLI behaviour. + if schema_override is not None and selector is None: + return AgentContext( + name=schema_override.parent.name or ".", + schema_path=schema_override.resolve(), + endpoint=env.get("EXPERT_AGENT_ENDPOINT"), + api_key=env.get("EXPERT_AGENT_API_KEY"), + description=None, + selector_source="schema-flag", + ) + + if selector: + name, source = self._match(selector), "flag" + elif env.get(_ENV_ACTIVE_AGENT): + name, source = self._match(env[_ENV_ACTIVE_AGENT]), "env" + elif (pinned := self.active()) is not None: + name, source = self._match(pinned), "active" + elif self.default_agent is not None: + name, source = self._match(self.default_agent), "default" + elif len(self.agents_by_name) == 1: + name, source = ( + next(iter(self.agents_by_name)), + ("single" if self.single_agent_mode else "auto"), + ) + + if name is None: + raise AmbiguousAgentError( + self._ambiguity_message(selector, env), + candidates=self.agents(), + ) + + info = self.agents_by_name[name] + endpoint = info.endpoint or env.get("EXPERT_AGENT_ENDPOINT") + api_key = info.api_key or env.get("EXPERT_AGENT_API_KEY") + schema_path = schema_override.resolve() if schema_override else info.schema_path + return AgentContext( + name=info.name, + schema_path=schema_path, + endpoint=endpoint, + api_key=api_key, + description=info.description, + selector_source=source, + ) + + # --------------------------- Internals ------------------------------ # + + def _match(self, selector: str) -> str: + """Resolve an agent selector (exact name or unique prefix). + + Accepts and strips a leading ``@`` so that the same helper can back + both ``--agent derm`` and ``@derm`` transparently. + """ + if not selector: + raise AgentNotFoundError("empty agent selector") + needle = selector.lstrip("@") + if needle in self.agents_by_name: + return needle + matches = [n for n in self.agents_by_name if n.startswith(needle)] + if len(matches) == 1: + return matches[0] + if not matches: + raise AgentNotFoundError( + f"No agent named '{needle}'. " + f"Available: {', '.join(sorted(self.agents_by_name)) or '(none)'}." + ) + raise AmbiguousAgentError( + f"Prefix '{needle}' is ambiguous — matches: {', '.join(sorted(matches))}. " + "Use the full name or a longer prefix.", + candidates=[self.agents_by_name[m] for m in matches], + ) + + def _ambiguity_message(self, selector: str | None, env: dict[str, str]) -> str: + if not self.agents_by_name: + return ( + "No agent_schema.yaml found in this workspace. " + "Run `expert init ` to scaffold one, or pass " + "--schema explicitly." + ) + lines = [ + "Multiple agents found in this workspace and no selector was given.", + "", + "Candidates:", + ] + for info in self.agents(): + rel = _safe_relpath(info.schema_path, self.root) + badge = "[toml]" if info.source == "toml" else "[auto]" + lines.append(f" • {info.name:<20} {rel} {badge}") + lines.extend( + [ + "", + "Pick one, in order of preference:", + " expert @ # one-off shortcut", + " expert --agent # explicit flag (CI-friendly)", + " expert use # pin for this workspace", + ] + ) + _ = selector, env + return "\n".join(lines) + + +# ------------------------------------------------------------------------- # +# Helpers +# ------------------------------------------------------------------------- # + + +def _find_workspace_root(start: Path) -> tuple[Path, Path | None]: + """Walk up from ``start`` to find a workspace root + optional TOML path. + + Returns ``(root, toml_path)`` where ``toml_path`` may be ``None``. The + ``root`` is: + + - The first ancestor containing ``expert.toml`` (authoritative marker). + - Else the first ancestor containing ``.expert/state.json`` (previously + pinned via ``expert use``). + - Else ``start`` itself. Sibling-schema discovery is always rooted at + ``start`` — we never silently promote an unrelated ancestor to + ``root`` just because it happens to have other agent directories + lying around. + """ + current = start + for _ in range(_DISCOVERY_MAX_DEPTH + 1): + toml = current / _WORKSPACE_FILE + if toml.is_file(): + return current, toml + if (current / _STATE_DIR / _STATE_FILE).is_file(): + return current, None + if current.parent == current: + break + current = current.parent + return start, None + + +def _opt_str(value: Any) -> str | None: + return value if isinstance(value, str) and value else None + + +def _resolve_api_key(body: dict[str, Any]) -> str | None: + env_var = body.get("api_key_env") + if isinstance(env_var, str) and env_var: + env_value = os.environ.get(env_var) + if env_value: + return env_value + raw = body.get("api_key") + return raw if isinstance(raw, str) and raw else None + + +def _safe_relpath(path: Path, base: Path) -> str: + try: + return str(path.relative_to(base)) + except ValueError: + return str(path) + + +__all__ = [ + "AgentContext", + "AgentInfo", + "AgentNotFoundError", + "AmbiguousAgentError", + "Workspace", + "WorkspaceError", +] diff --git a/cli/tests/test_main_alias.py b/cli/tests/test_main_alias.py new file mode 100644 index 0000000..3e2b998 --- /dev/null +++ b/cli/tests/test_main_alias.py @@ -0,0 +1,183 @@ +"""Tests for the `@alias` argv rewriter and workspace-aware commands.""" + +from __future__ import annotations + +from pathlib import Path + +import pytest +from expert.main import _rewrite_at_alias, app +from typer.testing import CliRunner + + +def test_rewrite_at_alias_for_agent_aware_subcommand() -> None: + argv = ["expert", "@ecg", "ask", "hi", "--no-stream"] + assert _rewrite_at_alias(argv) == [ + "expert", + "ask", + "hi", + "--no-stream", + "--agent", + "ecg", + ] + + +def test_rewrite_at_alias_preserves_nested_subcommand() -> None: + argv = ["expert", "@derm", "sessions", "list"] + assert _rewrite_at_alias(argv) == [ + "expert", + "sessions", + "list", + "--agent", + "derm", + ] + + +def test_rewrite_at_alias_no_rewrite_for_non_agent_command() -> None: + """`use`/`agents` aren't in the allow-list; argv is returned unchanged.""" + argv = ["expert", "@ecg", "use", "ecg"] + assert _rewrite_at_alias(argv) == argv + + +def test_rewrite_at_alias_no_arg_after() -> None: + """`expert @ecg` with nothing else is left alone (Typer will show help).""" + argv = ["expert", "@ecg"] + assert _rewrite_at_alias(argv) == argv + + +def test_rewrite_ignores_dashed_tokens_between_alias_and_subcommand() -> None: + argv = ["expert", "@ecg", "--verbose", "validate"] + assert _rewrite_at_alias(argv) == [ + "expert", + "--verbose", + "validate", + "--agent", + "ecg", + ] + + +def test_rewrite_appends_agent_at_end_for_sessions_list() -> None: + """Appending at the end routes the flag to the deepest sub-Typer.""" + argv = ["expert", "@derm", "sessions", "list", "--user", "u1"] + assert _rewrite_at_alias(argv) == [ + "expert", + "sessions", + "list", + "--user", + "u1", + "--agent", + "derm", + ] + + +def test_rewrite_handles_empty_alias() -> None: + argv = ["expert", "@", "ask", "hi"] + # Too short — should no-op rather than misinterpret. + assert _rewrite_at_alias(argv) == argv + + +# ------------------------------------------------------------------------- # +# Integration: workspace-aware commands +# ------------------------------------------------------------------------- # + + +def _seed(tmp_path: Path) -> Path: + (tmp_path / "ecg").mkdir() + (tmp_path / "derm").mkdir() + (tmp_path / "ecg" / "agent_schema.yaml").write_text("x") + (tmp_path / "derm" / "agent_schema.yaml").write_text("x") + (tmp_path / "expert.toml").write_text( + '[defaults]\nagent = "ecg"\n\n' + '[agents.ecg]\nschema = "ecg/agent_schema.yaml"\n' + 'endpoint = "https://ecg.example"\napi_key = "sk-ecg"\n\n' + '[agents.derm]\nschema = "derm/agent_schema.yaml"\n', + ) + return tmp_path + + +def test_agents_command_lists(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + _seed(tmp_path) + monkeypatch.chdir(tmp_path) + runner = CliRunner() + result = runner.invoke(app, ["agents"]) + assert result.exit_code == 0, result.output + assert "ecg" in result.output + assert "derm" in result.output + + +def test_which_uses_toml_default(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + _seed(tmp_path) + monkeypatch.chdir(tmp_path) + monkeypatch.delenv("EXPERT_AGENT", raising=False) + monkeypatch.delenv("EXPERT_AGENT_ENDPOINT", raising=False) + runner = CliRunner() + result = runner.invoke(app, ["which"]) + assert result.exit_code == 0, result.output + assert "ecg" in result.output + assert "default" in result.output + + +def test_use_then_which(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + _seed(tmp_path) + monkeypatch.chdir(tmp_path) + monkeypatch.delenv("EXPERT_AGENT", raising=False) + runner = CliRunner() + + res = runner.invoke(app, ["use", "derm"]) + assert res.exit_code == 0, res.output + + res = runner.invoke(app, ["which"]) + assert res.exit_code == 0, res.output + assert "derm" in res.output + assert "active" in res.output + + +def test_which_with_agent_flag_overrides_pin( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + _seed(tmp_path) + monkeypatch.chdir(tmp_path) + monkeypatch.delenv("EXPERT_AGENT", raising=False) + runner = CliRunner() + runner.invoke(app, ["use", "derm"]) + res = runner.invoke(app, ["which", "--agent", "ecg"]) + assert res.exit_code == 0, res.output + assert "ecg" in res.output + assert "flag" in res.output + + +def test_use_clear(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + _seed(tmp_path) + monkeypatch.chdir(tmp_path) + runner = CliRunner() + runner.invoke(app, ["use", "derm"]) + assert (tmp_path / ".expert" / "state.json").is_file() + + res = runner.invoke(app, ["use", "--clear"]) + assert res.exit_code == 0, res.output + assert not (tmp_path / ".expert" / "state.json").is_file() + + +def test_use_unknown_agent(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + _seed(tmp_path) + monkeypatch.chdir(tmp_path) + runner = CliRunner() + res = runner.invoke(app, ["use", "does-not-exist"]) + assert res.exit_code != 0 + + +def test_ambiguous_workspace_shows_helpful_error( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """Two auto-discovered agents, no selector → helpful multi-line error.""" + (tmp_path / "a").mkdir() + (tmp_path / "b").mkdir() + (tmp_path / "a" / "agent_schema.yaml").write_text("x") + (tmp_path / "b" / "agent_schema.yaml").write_text("x") + monkeypatch.chdir(tmp_path) + monkeypatch.delenv("EXPERT_AGENT", raising=False) + + runner = CliRunner() + res = runner.invoke(app, ["which"]) + assert res.exit_code != 0 + assert "expert @" in res.output or "@" in res.output + assert "--agent" in res.output diff --git a/cli/tests/test_workspace.py b/cli/tests/test_workspace.py new file mode 100644 index 0000000..6aacd4d --- /dev/null +++ b/cli/tests/test_workspace.py @@ -0,0 +1,262 @@ +"""Tests for multi-agent workspace discovery and resolution.""" + +from __future__ import annotations + +import json +from pathlib import Path + +import pytest +from expert.workspace import ( + AgentNotFoundError, + AmbiguousAgentError, + Workspace, + WorkspaceError, +) + +# ------------------------------------------------------------------------- # +# Fixtures +# ------------------------------------------------------------------------- # + + +def _mk_schema(dir_: Path, name: str = "a") -> Path: + dir_.mkdir(parents=True, exist_ok=True) + f = dir_ / "agent_schema.yaml" + f.write_text(f"# dummy schema for {name}\n") + return f + + +def _mk_workspace( + root: Path, + *, + agents: dict[str, dict[str, object]] | None = None, + default: str | None = None, +) -> None: + """Create a workspace directory with optional expert.toml. + + ``agents`` maps canonical names to dicts of ``schema``/``endpoint``/etc. + Schemas are materialised on disk relative to ``root``. + """ + if agents is None: + return + lines: list[str] = [] + if default: + lines.extend(["[defaults]", f'agent = "{default}"', ""]) + for name, body in agents.items(): + schema_rel = body.get("schema") or f"{name}/agent_schema.yaml" + assert isinstance(schema_rel, str) + _mk_schema(root / Path(schema_rel).parent, name=name) + lines.append(f"[agents.{name}]") + lines.append(f'schema = "{schema_rel}"') + for key in ("endpoint", "api_key", "api_key_env", "description"): + value = body.get(key) + if isinstance(value, str): + lines.append(f'{key} = "{value}"') + lines.append("") + (root / "expert.toml").write_text("\n".join(lines)) + + +# ------------------------------------------------------------------------- # +# Discovery +# ------------------------------------------------------------------------- # + + +def test_single_agent_mode(tmp_path: Path) -> None: + _mk_schema(tmp_path) + ws = Workspace.discover(cwd=tmp_path) + assert ws.single_agent_mode is True + assert list(ws.agents_by_name) == ["."] + + ctx = ws.resolve() + assert ctx.name == "." + assert ctx.selector_source == "single" + + +def test_auto_discover_siblings(tmp_path: Path) -> None: + _mk_schema(tmp_path / "ecg") + _mk_schema(tmp_path / "derm") + ws = Workspace.discover(cwd=tmp_path) + assert ws.single_agent_mode is False + assert set(ws.agents_by_name) == {"ecg", "derm"} + assert all(info.source == "auto" for info in ws.agents()) + + +def test_toml_overrides_auto(tmp_path: Path) -> None: + _mk_workspace( + tmp_path, + agents={ + "ecg": {"schema": "ecg/agent_schema.yaml", "endpoint": "https://ecg"}, + "derm": {"schema": "derm/agent_schema.yaml"}, + }, + default="ecg", + ) + ws = Workspace.discover(cwd=tmp_path) + assert ws.default_agent == "ecg" + assert ws.agents_by_name["ecg"].source == "toml" + assert ws.agents_by_name["ecg"].endpoint == "https://ecg" + + +def test_toml_plus_sibling_not_declared(tmp_path: Path) -> None: + """Declared agents + undeclared siblings should coexist.""" + _mk_workspace( + tmp_path, + agents={"ecg": {"schema": "ecg/agent_schema.yaml"}}, + ) + _mk_schema(tmp_path / "derm") + ws = Workspace.discover(cwd=tmp_path) + assert set(ws.agents_by_name) == {"ecg", "derm"} + assert ws.agents_by_name["ecg"].source == "toml" + assert ws.agents_by_name["derm"].source == "auto" + + +# ------------------------------------------------------------------------- # +# Resolution precedence +# ------------------------------------------------------------------------- # + + +def test_resolve_explicit_selector_wins(tmp_path: Path) -> None: + _mk_workspace( + tmp_path, + agents={"ecg": {}, "derm": {}}, + default="ecg", + ) + ws = Workspace.discover(cwd=tmp_path) + ws.set_active("ecg") + ctx = ws.resolve(selector="derm") + assert ctx.name == "derm" + assert ctx.selector_source == "flag" + + +def test_resolve_env_var(tmp_path: Path) -> None: + _mk_workspace(tmp_path, agents={"ecg": {}, "derm": {}}) + ws = Workspace.discover(cwd=tmp_path) + ctx = ws.resolve(env={"EXPERT_AGENT": "derm"}) + assert ctx.name == "derm" + assert ctx.selector_source == "env" + + +def test_resolve_active_pin(tmp_path: Path) -> None: + _mk_workspace(tmp_path, agents={"ecg": {}, "derm": {}}) + ws = Workspace.discover(cwd=tmp_path) + ws.set_active("derm") + ctx = ws.resolve(env={}) + assert ctx.name == "derm" + assert ctx.selector_source == "active" + + +def test_resolve_default_from_toml(tmp_path: Path) -> None: + _mk_workspace(tmp_path, agents={"ecg": {}, "derm": {}}, default="ecg") + ws = Workspace.discover(cwd=tmp_path) + ctx = ws.resolve(env={}) + assert ctx.name == "ecg" + assert ctx.selector_source == "default" + + +def test_resolve_ambiguous(tmp_path: Path) -> None: + _mk_workspace(tmp_path, agents={"ecg": {}, "derm": {}}) + ws = Workspace.discover(cwd=tmp_path) + with pytest.raises(AmbiguousAgentError) as exc_info: + ws.resolve(env={}) + assert "Multiple agents" in str(exc_info.value) + names = {c.name for c in exc_info.value.candidates} + assert names == {"ecg", "derm"} + + +def test_resolve_unique_prefix(tmp_path: Path) -> None: + _mk_workspace(tmp_path, agents={"ecg-expert": {}, "derm": {}}) + ws = Workspace.discover(cwd=tmp_path) + ctx = ws.resolve(selector="ecg") + assert ctx.name == "ecg-expert" + + +def test_resolve_ambiguous_prefix(tmp_path: Path) -> None: + _mk_workspace(tmp_path, agents={"ecg-expert": {}, "ecg-trainer": {}}) + ws = Workspace.discover(cwd=tmp_path) + with pytest.raises(AmbiguousAgentError): + ws.resolve(selector="ecg") + + +def test_resolve_unknown_selector(tmp_path: Path) -> None: + _mk_workspace(tmp_path, agents={"ecg": {}}) + ws = Workspace.discover(cwd=tmp_path) + with pytest.raises(AgentNotFoundError): + ws.resolve(selector="nope") + + +def test_resolve_at_alias_prefix_strip(tmp_path: Path) -> None: + _mk_workspace(tmp_path, agents={"ecg": {}, "derm": {}}) + ws = Workspace.discover(cwd=tmp_path) + ctx = ws.resolve(selector="@ecg") + assert ctx.name == "ecg" + + +def test_resolve_schema_override_bypasses_workspace(tmp_path: Path) -> None: + standalone = tmp_path / "orphan" + schema = _mk_schema(standalone) + # No workspace here — ensure the flag-based fallback works. + ws = Workspace.discover(cwd=tmp_path) + ctx = ws.resolve(schema_override=schema, env={}) + assert ctx.schema_path == schema + assert ctx.selector_source == "schema-flag" + + +# ------------------------------------------------------------------------- # +# API key resolution +# ------------------------------------------------------------------------- # + + +def test_api_key_from_env_via_api_key_env( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setenv("MY_ECG_KEY", "sk-from-env") + _mk_workspace( + tmp_path, + agents={"ecg": {"schema": "ecg/agent_schema.yaml", "api_key_env": "MY_ECG_KEY"}}, + ) + ws = Workspace.discover(cwd=tmp_path) + assert ws.agents_by_name["ecg"].api_key == "sk-from-env" + + +def test_env_endpoint_fills_when_toml_missing(tmp_path: Path) -> None: + _mk_workspace(tmp_path, agents={"ecg": {}}) + ws = Workspace.discover(cwd=tmp_path) + ctx = ws.resolve(env={"EXPERT_AGENT_ENDPOINT": "https://x", "EXPERT_AGENT_API_KEY": "k"}) + assert ctx.endpoint == "https://x" + assert ctx.api_key == "k" + + +def test_require_remote_raises_when_incomplete(tmp_path: Path) -> None: + _mk_workspace(tmp_path, agents={"ecg": {}}) + ws = Workspace.discover(cwd=tmp_path) + ctx = ws.resolve(env={}) + with pytest.raises(WorkspaceError): + ctx.require_remote() + + +# ------------------------------------------------------------------------- # +# Pin state file +# ------------------------------------------------------------------------- # + + +def test_set_active_writes_state(tmp_path: Path) -> None: + _mk_workspace(tmp_path, agents={"ecg": {}, "derm": {}}) + ws = Workspace.discover(cwd=tmp_path) + ws.set_active("derm") + state = json.loads(ws.state_file.read_text()) + assert state == {"agent": "derm"} + assert ws.active() == "derm" + + +def test_clear_active(tmp_path: Path) -> None: + _mk_workspace(tmp_path, agents={"ecg": {}}) + ws = Workspace.discover(cwd=tmp_path) + ws.set_active("ecg") + ws.clear_active() + assert ws.active() is None + + +def test_set_active_rejects_unknown(tmp_path: Path) -> None: + _mk_workspace(tmp_path, agents={"ecg": {}}) + ws = Workspace.discover(cwd=tmp_path) + with pytest.raises(AgentNotFoundError): + ws.set_active("nope") diff --git a/docs/AGENT_E2E_SETUP.md b/docs/AGENT_E2E_SETUP.md index e161815..a39e1fa 100644 --- a/docs/AGENT_E2E_SETUP.md +++ b/docs/AGENT_E2E_SETUP.md @@ -45,8 +45,16 @@ first or warn the user. - [ ] Repo settings → *Actions → General → Workflow permissions* allow reading from public actions (default). -If the repo hosts **multiple agents** (a monorepo), each agent gets its own -workflow file pointing at its own schema. +If the repo hosts **multiple agents** (a monorepo), you have two options: + +1. **One workflow per agent** — each file pins a different `schema:` and a + different set of secrets. Recommended when the agents are owned by + different teams or deployed to different projects. +2. **One workflow, matrix-over-agents** — declare an `expert.toml` at the + repo root and let `expert test` resolve each agent by name. See the + "matrix" snippet in [§6. Customising for your agent](#6-customising-for-your-agent). + +Both integrations share the same reusable workflow; only the caller changes. --- @@ -209,6 +217,42 @@ You almost never need to fork the suites. Knobs available out of the box: | Run only one suite | Trigger with the `suite:` choice input (`gh workflow run … -f suite=05_ask_latency`). | | Pin to a stable upstream version | Replace `@main` with `@v0.1.1` everywhere (both `uses:` and `cli-ref:`). | | Add a per-deploy smoke check | Call the reusable workflow from your `deploy.yml` after the Cloud Run rollout finishes. | +| Test N agents in one monorepo | See "matrix" snippet below, or keep one workflow-per-agent for clearer blame. | + +### Matrix over agents (monorepo) + +If `expert.toml` at the repo root declares several agents, the CLI already +understands `expert test --agent `. You can call the reusable workflow +once per agent via a matrix: + +```yaml +jobs: + e2e: + strategy: + fail-fast: false + matrix: + agent: + - { name: ecg, schema: ecg-expert/agent_schema.yaml, endpoint_secret: ECG_ENDPOINT, key_secret: ECG_API_KEY } + - { name: derm, schema: derm-expert/agent_schema.yaml, endpoint_secret: DERM_ENDPOINT, key_secret: DERM_API_KEY } + uses: feliperbroering/expert-agent/.github/workflows/expert-e2e.yml@<> + with: + schema: ${{ matrix.agent.schema }} + sample-question: "ping" + cli-ref: <> + secrets: + endpoint: ${{ secrets[matrix.agent.endpoint_secret] }} + api-key: ${{ secrets[matrix.agent.key_secret] }} +``` + +Locally, the same layout lets you do: + +```bash +expert agents # list all known agents +expert use ecg # pin ecg for this shell +expert ask "..." # routes to ecg +expert @derm ask "..." # one-off hop to derm +expert test --agent derm # run the packaged E2E kit against derm +``` If you genuinely need a *new* assertion the upstream suites don't cover, contribute it back to `expert-agent` rather than vendoring locally — the kit diff --git a/pyproject.toml b/pyproject.toml index f90db24..31d8cff 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -70,7 +70,7 @@ test = [ ] [project.scripts] -expert = "expert.main:app" +expert = "expert.main:main" expert-agent-backend = "app.main:run" [project.urls] diff --git a/uv.lock b/uv.lock index 1f039f9..5b248d7 100644 --- a/uv.lock +++ b/uv.lock @@ -544,7 +544,7 @@ wheels = [ [[package]] name = "expert-agent" -version = "0.1.0" +version = "0.1.1" source = { editable = "." } dependencies = [ { name = "bcrypt" },