From 56625c0be2e2ef48dc07f3f2c03fc877df819326 Mon Sep 17 00:00:00 2001 From: Oleksandr Gubarets Date: Mon, 7 Sep 2026 14:48:09 +0300 Subject: [PATCH 01/10] add design description --- docs/designs/dynamic_tool_discovery.md | 241 +++++++++++++++++++++++++ 1 file changed, 241 insertions(+) create mode 100644 docs/designs/dynamic_tool_discovery.md diff --git a/docs/designs/dynamic_tool_discovery.md b/docs/designs/dynamic_tool_discovery.md new file mode 100644 index 00000000..a330cc14 --- /dev/null +++ b/docs/designs/dynamic_tool_discovery.md @@ -0,0 +1,241 @@ +# Design: Dynamic Tool Discovery + +- **Status:** Draft +- **Issue:** [#430](https://github.com/epam/ai-dial-quickapps-backend/issues/430) + +## Problem Statement + +All tool definitions (REST API, MCP, DIAL deployment) are merged into one flat list and sent +verbatim to the LLM on every request: + +```python +# _chat_completion_config_builder.py +payload["tools"] = self.__tools # every tool, fully expanded, every call +``` + +With five or more MCP servers or large REST APIs this can consume ~55 K tokens upfront per turn, +regardless of which tools the model will actually use. This dilutes model attention and degrades +tool-selection accuracy. + +**Root cause:** `AgentModule.provide_openai_tools` assembles a single flat `list[OpenAiToolConfigDict]` +at DI-wiring time. There is no mechanism to defer, filter, or paginate that list at request time. + +--- + +## Goals + +- Reduce upfront token cost for applications with many tools by deferring full definitions until + the model indicates it needs them. +- Non-breaking: existing applications that do not opt in behave identically. +- Model-agnostic: must work for any LLM accessible via DIAL without provider-specific extensions. +- Shaped for future configurability (per-tool granularity, semantic search, result caching) + without implementing those knobs in MVP. + +--- + +## Codebase Anchors + +| What | File | +|---|---| +| LLM payload assembly | `src/quickapp/core/agent/_chat_completion_config_builder.py` | +| Tool list DI wiring | `src/quickapp/core/agent/agent_module.py` (`provide_openai_tools`) | +| MCP schema loading | `src/quickapp/mcp_tooling/_mcp_tool_initializer.py` | +| Existing lazy-injection precedent | `src/quickapp/orchestrator_attachment_strategies/lazy_on_demand/` | +| Toolset config root | `src/quickapp/config/toolsets/toolset.py`, `BaseToolSet` | +| Internal tool base | `src/quickapp/common/staged_base_tool.py` (`StagedBaseTool`) | + +The only existing "on-demand" pattern in the codebase is `LazyOnDemandStrategyModule`, which +injects the `internal_attachments_get_content` tool per-request via the same DI `@multiprovider` +extension point. Dynamic tool discovery should follow the same pattern. + +--- + +## Decision Points + +Three orthogonal decisions drive the design space: + +| # | Decision | Choices | +|---|---|---| +| A | **Deferral granularity** | Per-toolset vs per-tool | +| B | **Discovery surface** | Keyword search · Compact manifest · Semantic search | +| C | **How full definitions reach the LLM** | As tool-call result text · Injected into `payload["tools"]` | + +--- + +## Options + +### Option 1 — Per-toolset deferred flag + keyword-search discovery tool + +**Mechanism:** +Add `deferred: bool = False` to `BaseToolSet`. When `deferred=true`, the toolset does not +contribute its tool schemas to `payload["tools"]`. Instead it contributes a single internal +discovery tool: + +``` +{toolset_name}_discover(query: str) → list[{name, description, parameters}] +``` + +The tool performs a keyword match on tool name + description and returns full definitions as +JSON text in the tool-call result. The LLM reads the definitions from context and then makes +the actual tool call. + +**Round-trip cost:** +1 before first tool use (discover → read result → call tool). + +**MCP init:** schemas still fetched at startup (no change); they are just withheld from +`payload["tools"]` until discovered. + +**Config example:** +```json +{ + "name": "my-mcp-server", + "type": "mcp", + "deferred": true, + "server": { "url": "..." } +} +``` + +**Pros:** +- Follows `LazyOnDemandStrategyModule` pattern exactly — no orchestrator changes. +- Non-breaking opt-in. +- Token savings are immediate (entire toolset suppressed). +- Simple to implement and test. + +**Cons:** +- Model must understand and follow the discovery protocol → system-prompt engineering required. +- Two round-trips before a tool can be used for the first time. +- Full definition returned as text; the model cannot use the schema for structured argument + generation on the same call. + +--- + +### Option 2 — Compact manifest upfront + single `get_tool_definition` tool + +**Mechanism:** +All tool names + one-line descriptions (no `parameters`) are sent upfront, either as a +minimalist `tools` array (empty parameters) or as a structured section in the system prompt. +A single internal tool `get_tool_definition(tool_name: str)` returns the full `OpenAiToolConfig` +JSON for any tool on demand. + +**Round-trip cost:** +1 before first use of any previously-unseen tool. + +**Config:** opt-in flag at `ApplicationConfig` (global) or per-toolset. + +**Pros:** +- Model always has full name-space visibility (all names + descriptions visible). +- Single shared discovery tool regardless of toolset count. +- One fewer round-trip than Option 1 when the model knows which tool it wants. + +**Cons:** +- Manifest can still be substantial for 100+ tools. +- Requires careful framing: empty-parameter tool entries may confuse some models. +- Full definition is still returned as text (same as Option 1); native schema generation + not available on the definition-fetch call. + +--- + +### Option 3 — Orchestrator-level dynamic tool injection + +**Mechanism:** +A discovery tool (keyword or semantic) returns tool names. The orchestrator intercepts the +discovery tool result and **injects the corresponding full schemas into `payload["tools"]` +on the next LLM call** — not into the message history. The model then uses the tool natively +with proper schema-based argument generation. + +**Changes required:** +- Orchestrator maintains `_discovered_tool_names: set[str]` state across iterations. +- `_ChatCompletionConfigBuilder` accepts a per-request "additional tools" override. +- Discovery tool result is processed as a side-effect by the orchestrator before the next + iteration, not treated as a regular TOOL message. +- Discovered names must optionally be persisted in conversation state for multi-turn continuity. + +**Round-trip cost:** +1 (discover call → definitions in tools array → native call). + +**Pros:** +- LLM interacts with discovered tools natively (proper argument schema, no prompt workarounds). +- Cleanest UX: from the model's perspective, discovered tools behave identically to pre-loaded ones. + +**Cons:** +- Significant orchestrator changes. +- Discovered-tool state must survive across iterations and potentially across conversation turns + (state serialisation). +- More complex error paths (discovery result processed as side-effect, not normal tool result). + +--- + +### Option 4 — Semantic search (enhancement layer) + +Replaces keyword matching in Options 1–3 with vector-similarity search on tool descriptions, +using a DIAL embedding deployment. + +**Pros:** better recall for natural-language queries; fewer false positives in large catalogs. +**Cons:** extra DIAL deployment dependency; per-discovery-call latency; adds significant +complexity for marginal MVP gain. + +Treat as a **post-MVP strategy swap** on top of Options 1 or 3. + +--- + +## Comparison + +| | Option 1 | Option 2 | Option 3 | +|---|---|---|---| +| Orchestrator changes | None | None | Significant | +| Round-trips before first use | +1 | +1 | +1 | +| Native schema on discovery turn | No | No | Yes | +| Full name-space always visible | No | Yes | No (configurable) | +| Implementation complexity | Low | Low–Medium | High | +| Follows existing lazy pattern | Yes | Partial | No | + +--- + +## Recommendation + +**MVP: Option 1** (per-toolset `deferred` flag + keyword-search discovery tool). + +Rationale: +- Directly mirrors `LazyOnDemandStrategyModule` — the implementation pattern is already proven. +- No orchestrator changes keeps the blast radius small. +- Delivers meaningful token savings (entire toolset suppressed) with an opt-in default. +- Leaves room to upgrade to Option 3 once discovery behaviour is validated in production. + +**Follow-on: Option 3** once the MVP is validated. It removes the prompt-engineering dependency +and makes discovered tools first-class. + +**Option 4** (semantic) can be offered as an alternative search strategy within either Option 1 +or 3 as a configuration toggle. + +--- + +## Open Questions + +1. **Granularity:** Should `deferred` live on `BaseToolSet` (per-toolset) or also be settable + per-tool inside a toolset? Per-toolset is simpler and covers the main use case (defer an + entire MCP server); per-tool is more surgical for mixed sets. + +2. **Discovery tool naming:** One tool per deferred toolset (`{toolset}_discover`) or a single + global `discover_tools(toolset?: str, query: str)`? A single tool reduces clutter in the + `tools` array but couples discovery to a specific naming convention. + +3. **Search quality:** Is substring keyword match on name + description sufficient for MVP, or + should we index descriptions (e.g. TF-IDF) for better recall? Keyword match is simple and + deterministic; TF-IDF adds a build-time index step. + +4. **System-prompt alignment:** Should deferred toolset descriptions be injected into the + system prompt (to help the model decide when to search) or omitted entirely? Including a + brief catalog summary reduces false-negative discovery calls. + +5. **Conversation-turn persistence (Option 3):** Should discovered tool names persist across + turns (stored in conversation state) so the model does not need to rediscover on every turn? + This trades discovery latency for state-size growth. + +--- + +## Out of Scope (MVP) + +| Item | Reason | +|---|---| +| Semantic / embedding-based search | Post-MVP strategy swap | +| Per-tool granularity within a toolset | Per-toolset is sufficient for initial use case | +| Dynamic schema injection into `payload["tools"]` | Option 3 — follow-on | +| Result caching across turns | State management complexity; defer | +| Provider-specific tool-pagination APIs | Not available via DIAL today | From 532acff911c7139e2efbb206e8baaad76f14904f Mon Sep 17 00:00:00 2001 From: Oleksandr Gubarets Date: Mon, 7 Sep 2026 17:51:41 +0300 Subject: [PATCH 02/10] improve design --- docs/designs/dynamic_tool_discovery.md | 471 +++++++++++++++++++++---- 1 file changed, 397 insertions(+), 74 deletions(-) diff --git a/docs/designs/dynamic_tool_discovery.md b/docs/designs/dynamic_tool_discovery.md index a330cc14..30f2f082 100644 --- a/docs/designs/dynamic_tool_discovery.md +++ b/docs/designs/dynamic_tool_discovery.md @@ -27,7 +27,6 @@ at DI-wiring time. There is no mechanism to defer, filter, or paginate that list - Reduce upfront token cost for applications with many tools by deferring full definitions until the model indicates it needs them. - Non-breaking: existing applications that do not opt in behave identically. -- Model-agnostic: must work for any LLM accessible via DIAL without provider-specific extensions. - Shaped for future configurability (per-tool granularity, semantic search, result caching) without implementing those knobs in MVP. @@ -43,28 +42,27 @@ at DI-wiring time. There is no mechanism to defer, filter, or paginate that list | Existing lazy-injection precedent | `src/quickapp/orchestrator_attachment_strategies/lazy_on_demand/` | | Toolset config root | `src/quickapp/config/toolsets/toolset.py`, `BaseToolSet` | | Internal tool base | `src/quickapp/common/staged_base_tool.py` (`StagedBaseTool`) | - -The only existing "on-demand" pattern in the codebase is `LazyOnDemandStrategyModule`, which -injects the `internal_attachments_get_content` tool per-request via the same DI `@multiprovider` -extension point. Dynamic tool discovery should follow the same pattern. +| Stream parsing | `src/quickapp/common/chat_completion_stream/parse.py` | +| Orchestrator loop | `src/quickapp/core/agent/orchestrator.py` | --- ## Decision Points -Three orthogonal decisions drive the design space: +Four orthogonal decisions drive the design space: | # | Decision | Choices | |---|---|---| | A | **Deferral granularity** | Per-toolset vs per-tool | -| B | **Discovery surface** | Keyword search · Compact manifest · Semantic search | -| C | **How full definitions reach the LLM** | As tool-call result text · Injected into `payload["tools"]` | +| B | **Discovery surface** | Keyword search · Compact manifest · Semantic search · Subagent-based | +| C | **How full definitions reach the LLM** | As tool-call result text · Injected into `payload["tools"]` · Server-side expansion (`tool_reference`) | +| D | **Provider scope** | Model-agnostic (any DIAL deployment) vs Anthropic-native API feature | --- ## Options -### Option 1 — Per-toolset deferred flag + keyword-search discovery tool +### Option 1 — Per-toolset deferred flag + custom keyword-search discovery tool **Mechanism:** Add `deferred: bool = False` to `BaseToolSet`. When `deferred=true`, the toolset does not @@ -96,15 +94,15 @@ the actual tool call. **Pros:** - Follows `LazyOnDemandStrategyModule` pattern exactly — no orchestrator changes. +- Model-agnostic: works with any DIAL deployment. - Non-breaking opt-in. - Token savings are immediate (entire toolset suppressed). -- Simple to implement and test. **Cons:** - Model must understand and follow the discovery protocol → system-prompt engineering required. -- Two round-trips before a tool can be used for the first time. - Full definition returned as text; the model cannot use the schema for structured argument - generation on the same call. + generation on the same call (requires an extra round-trip to actually call the tool natively). +- Keyword search quality may be insufficient for large or ambiguously-named tool catalogs. --- @@ -123,110 +121,427 @@ JSON for any tool on demand. **Pros:** - Model always has full name-space visibility (all names + descriptions visible). - Single shared discovery tool regardless of toolset count. -- One fewer round-trip than Option 1 when the model knows which tool it wants. +- Model-agnostic. **Cons:** - Manifest can still be substantial for 100+ tools. -- Requires careful framing: empty-parameter tool entries may confuse some models. -- Full definition is still returned as text (same as Option 1); native schema generation - not available on the definition-fetch call. +- Full definition returned as text; same two-round-trip issue as Option 1. --- ### Option 3 — Orchestrator-level dynamic tool injection **Mechanism:** -A discovery tool (keyword or semantic) returns tool names. The orchestrator intercepts the -discovery tool result and **injects the corresponding full schemas into `payload["tools"]` -on the next LLM call** — not into the message history. The model then uses the tool natively -with proper schema-based argument generation. +A discovery tool returns tool names. The orchestrator intercepts the result and **injects the +corresponding full schemas into `payload["tools"]` on the next LLM call** — not into the +message history. The model then uses the tool natively with proper schema-based argument +generation. **Changes required:** - Orchestrator maintains `_discovered_tool_names: set[str]` state across iterations. - `_ChatCompletionConfigBuilder` accepts a per-request "additional tools" override. -- Discovery tool result is processed as a side-effect by the orchestrator before the next - iteration, not treated as a regular TOOL message. +- Discovery tool result processed as a side-effect before the next iteration. - Discovered names must optionally be persisted in conversation state for multi-turn continuity. -**Round-trip cost:** +1 (discover call → definitions in tools array → native call). +**Round-trip cost:** +1 (discover call → definitions in `tools` array → native call). **Pros:** -- LLM interacts with discovered tools natively (proper argument schema, no prompt workarounds). -- Cleanest UX: from the model's perspective, discovered tools behave identically to pre-loaded ones. +- LLM interacts with discovered tools natively (proper schema, no prompt workarounds). +- Cleanest UX: discovered tools behave identically to pre-loaded ones from the model's perspective. +- Model-agnostic. **Cons:** - Significant orchestrator changes. -- Discovered-tool state must survive across iterations and potentially across conversation turns - (state serialisation). -- More complex error paths (discovery result processed as side-effect, not normal tool result). +- Discovered-tool state must survive across iterations and possibly across conversation turns. +- More complex error paths. --- -### Option 4 — Semantic search (enhancement layer) +### Option 4 — Anthropic native `defer_loading` + server-side Tool Search -Replaces keyword matching in Options 1–3 with vector-similarity search on tool descriptions, -using a DIAL embedding deployment. +**Mechanism:** +Anthropic's Messages API supports `defer_loading: true` on individual tool definitions and a +built-in server-side search tool. The API runs the search on Anthropic's infrastructure and +returns `tool_reference` blocks that it auto-expands into full definitions before the model sees +them. -**Pros:** better recall for natural-language queries; fewer false positives in large catalogs. -**Cons:** extra DIAL deployment dependency; per-discovery-call latency; adds significant -complexity for marginal MVP gain. +**API contract:** +```json +{ + "tools": [ + { "type": "tool_search_tool_bm25_20251119", "name": "tool_search_tool_bm25" }, + { + "name": "my_tool", + "description": "...", + "input_schema": { ... }, + "defer_loading": true + } + ] +} +``` -Treat as a **post-MVP strategy swap** on top of Options 1 or 3. +- `defer_loading: true` controls what enters the **model's context window**, not what is sent + over the wire. All definitions are still transmitted to the API on every request. +- The API excludes deferred tools from the system-prompt prefix → **prompt cache is preserved**. +- Two search variants: `tool_search_tool_regex_20251119` (Python regex patterns) and + `tool_search_tool_bm25_20251119` (BM25 natural-language queries). +- Supports up to **10,000 deferred tools**. +- The response contains `server_tool_use` and `tool_search_tool_result` blocks (server-executed; + no client `tool_result` reply needed for these) plus a `tool_reference` block that the API + expands automatically. + +**Model support:** Claude Haiku 4.5, Sonnet 4.5, Opus 4.5 and all newer models. +Claude Opus 4.1 and earlier do not support this feature. + +**Custom client-side variant:** The `tool_reference` response format is also usable by a +custom search tool (embedding-based, semantic, etc.): return `tool_reference` blocks in a +standard `tool_result`, and the API expands them the same way. + +**Changes required in QuickApp:** +1. `_ChatCompletionConfigBuilder` — optionally set `defer_loading: true` on tool dicts and + include the tool search tool entry. +2. `parse.py` / stream handler — parse `server_tool_use` and `tool_search_tool_result` block + types (currently unknown to the parser). +3. `orchestrator.py` message builder — preserve `server_tool_use` and `tool_search_tool_result` + blocks verbatim in the ASSISTANT message history (do not treat them as executable tool calls). +4. Config — a new `OrchestratorConfig.tool_search` sub-config controlling search variant and + which toolsets/tools to defer. +5. Capability detection — fall back gracefully when the orchestrator deployment is not an + Anthropic model that supports this feature. + +**Round-trip cost:** +1 search turn. But definitions are expanded server-side within that +same turn, so the model can call the discovered tool in the very next turn with no extra +client round-trip. + +**Pros:** +- No custom search implementation to maintain — Anthropic handles indexing, matching, and + schema expansion. +- Prompt cache preserved (deferred tools excluded from the stable prefix). +- Native `tool_reference` expansion means the model always works with real tool schemas. +- Supports per-tool granularity on the API side. + +**Cons:** +- **Not model-agnostic**: only works when the orchestrator deployment exposes Anthropic's + Messages API (Claude Sonnet/Opus 4.5+). Other DIAL deployments (GPT-4, Gemini, etc.) + do not have this feature. +- Full definitions still transmitted to the API on every request (wire cost unchanged, only + context-window cost reduced). +- New block types (`server_tool_use`, `tool_search_tool_result`, `tool_reference`) require + changes to the stream parser and message history handling. +- `defer_loading: true` and `cache_control` cannot be set on the same tool (API returns 400). --- -## Comparison +### Option 5 — MCP-protocol progressive discovery (three-layer) + +**Mechanism:** +Follows the [MCP client best practices](https://modelcontextprotocol.io/docs/2026-07-28/develop/clients/client-best-practices) +progressive discovery pattern. The host fetches all tool definitions via `tools/list` at startup +but exposes them to the model through a three-layer surface: + +| Layer | Tool | What it returns | +|---|---|---| +| 1 — Catalog | `search_tools(query)` | `[{name, description}]` — names + one-liners only | +| 2 — Inspect | `get_tool_details(name)` | Full `inputSchema` for one tool | +| 3 — Execute | `{tool_name}(...)` | Normal tool execution | + +Full definitions enter the context only at Layer 2, after the model identifies the specific +tool it wants. When implemented with custom client-side `tool_reference` blocks (see Option 4 +custom variant), the Layer 2 inspection call can be eliminated and the Layer 1 result can +directly expand into full definitions. + +**Threshold recommendation (from MCP spec):** switch to progressive discovery once tool +definitions exceed 1–5% of the model's context window. + +**Dynamic server management extension:** connect MCP servers lazily — maintain a server +registry, connect only when the model requests a server's capabilities. Requires changes to +`_MCPToolInitializer` to support deferred connection and `tools/list` fetching. + +**Round-trip cost:** +1 (search) or +2 (search + inspect) before first tool use. + +**Pros:** +- Fully model-agnostic (any LLM; no Anthropic-specific API features). +- Covers both tool-level and server-level deferral. +- Aligns with the emerging MCP ecosystem standard, future-proofing integration as MCP + clients converge on this pattern. +- Layer 2 (inspect) is optional: with Option 4's `tool_reference` extension it collapses to + a single extra round-trip. +- `list_changed` notification support (already a concept in MCP) enables cache invalidation. + +**Cons:** +- Two round-trips (catalog + inspect) in the full three-layer form. +- Dynamic server management is a significant additional change (`_MCPToolInitializer`, + connection lifecycle, reconnect on demand). +- Without the `tool_reference` trick (Option 4 custom variant), full definition is returned as + text — same schema-generation problem as Options 1–2. + +--- + +### Option 6 — Subagent-routed catalog search + orchestrator injection (Recommended) + +**Mechanism:** +Combines a lightweight separate-LLM search step (for token-efficient routing) with orchestrator-level +tool injection (for native schema-based calling). It is model-agnostic and requires no +Anthropic-specific API features. + +#### Startup: catalog build + +Each toolset that opts in builds a compact **catalog entry** — `{name, description}` only — and +stores it in a per-toolset `ToolCatalog`. Full `OpenAiToolConfig` definitions are fetched and +held in memory as today but are **not forwarded to `payload["tools"]`**. + +```python +# Stored at init time, never sent to the main LLM upfront +catalog: list[ToolCatalogEntry] # [{name, description}, ...] +definitions: dict[str, OpenAiToolConfigDict] # name → full schema +``` + +#### Main orchestrator call + +`payload["tools"]` contains only two meta-tools plus any always-on (non-deferred) tools: + +``` +tool_search(query: str) → list[{name, description}] +tool_discovery(tool_name: str) → full OpenAiToolConfig JSON +``` + +Deferred tool definitions are **absent from the tools array**. The main LLM sees a minimal +surface. + +#### tool_search execution: separate chat completion + +When the main LLM calls `tool_search`, the tool handler fires a **separate, isolated chat +completion** — a fresh context with no conversation history and no system prompt: + +``` +model: configurable (defaults to the orchestrator deployment; a cheap/fast + model such as a Haiku-class DIAL deployment is recommended) +messages: [{"role": "user", "content": }] +system: minimal routing instruction + + compact catalog injected as context (all names + descriptions) +tools: none +``` + +The routing LLM returns the best-matching tool names and descriptions. Because this call carries +no conversation history or system prompt, its token cost is proportional only to the catalog +size — not to conversation length. + +**Result returned to main LLM:** `[{name, description}, ...]` + +#### tool_discovery execution: local lookup, no LLM + +When the main LLM calls `tool_discovery(tool_name)`, the handler does a plain dictionary lookup +against the in-memory `definitions` map and returns the full `OpenAiToolConfig` JSON. No LLM +call is made. + +**Result returned to main LLM:** full tool schema as JSON text in the tool result. + +#### Orchestrator injection (Path A) + +The orchestrator intercepts any `tool_discovery` result and: +1. Parses the returned tool name(s) from the result. +2. Adds the corresponding `OpenAiToolConfigDict` to a per-iteration `_pending_tools: dict[str, OpenAiToolConfigDict]`. +3. On the **next** call to `_ChatCompletionConfigBuilder.build()`, the pending definitions are + merged into `payload["tools"]`. +4. The main LLM now sees the discovered tool natively and calls it with proper schema-based + argument generation. + +Discovered tools accumulate across iterations within a turn. Optionally they are serialised +into `custom_content.state` so they persist across conversation turns (avoiding rediscovery on +the next user message). + +#### Flow diagram + +``` +Turn start + │ + ▼ +Main LLM call + tools = [tool_search, tool_discovery, ...always-on] + messages = full conversation history + │ + ├─ LLM calls tool_search("find Salesforce tools") + │ │ + │ └─ Separate chat completion (fresh context): + │ model = fast routing model + │ context = compact catalog (names + descriptions only) + │ input = "find Salesforce tools" + │ → [{name: "sf_query", description: "..."}, ...] + │ Result returned to main LLM + │ + ├─ LLM calls tool_discovery("sf_query") + │ │ + │ └─ Local dict lookup → full OpenAiToolConfig JSON + │ Orchestrator registers "sf_query" in _pending_tools + │ Result returned to main LLM + │ + ▼ +Next main LLM call + tools = [tool_search, tool_discovery, ...always-on, sf_query ← injected] + │ + └─ LLM calls sf_query(object="Account", ...) natively ✓ +``` + +#### Deferral threshold -| | Option 1 | Option 2 | Option 3 | +Even when `deferred: true` is set, a toolset is loaded eagerly if it is small enough that +deferring it would cost more (extra round-trips) than it saves (token reduction). Two guards +are evaluated at startup; a toolset is deferred only when it clears **both**: + +| Guard | Config key | Default | Check | |---|---|---|---| -| Orchestrator changes | None | None | Significant | -| Round-trips before first use | +1 | +1 | +1 | -| Native schema on discovery turn | No | No | Yes | -| Full name-space always visible | No | Yes | No (configurable) | -| Implementation complexity | Low | Low–Medium | High | -| Follows existing lazy pattern | Yes | Partial | No | +| Tool count | `min_tools_for_deferral` | `5` | `len(catalog) >= threshold` | +| Token estimate | `min_tokens_for_deferral` | `1000` | `estimated_tokens >= threshold` | + +Token estimate is computed as `sum(len(json.dumps(schema)) for schema in definitions.values())` +— a cheap character-count proxy evaluated once at startup. It is intentionally approximate; +exact tokenisation is not worth the overhead here. + +``` +deferred_effective = ( + config.deferred + and len(catalog) >= discovery.min_tools_for_deferral + and estimated_tokens >= discovery.min_tokens_for_deferral +) +``` + +A toolset with `deferred: false` is always loaded eagerly regardless of size. A toolset with +`deferred: true` that falls below either threshold is silently promoted to eager and its tools +are included in `payload["tools"]` as normal — no discovery overhead, no behavioural change +visible to the model. + +#### Configuration + +```json +{ + "orchestrator": { + "tool_discovery": { + "enabled": true, + "routing_deployment": "claude-haiku-dial-deployment", + "min_tools_for_deferral": 5, + "min_tokens_for_deferral": 1000 + } + }, + "tool_sets": [ + { + "name": "salesforce", + "type": "mcp", + "deferred": true, + "server": { "url": "..." } + }, + { + "name": "internal-utils", + "type": "internal", + "deferred": false + } + ] +} +``` + +- `deferred: true` on a toolset opts it into the catalog. Default: `false` (existing behaviour preserved). +- `routing_deployment` names the DIAL deployment used for the `tool_search` separate completion. + When omitted, it falls back to the orchestrator's own deployment. +- `min_tools_for_deferral` and `min_tokens_for_deferral` are global guards; toolsets that do + not clear both are silently promoted to eager loading. Both default to values that make + deferral a no-op for small toolsets. +- Non-deferred toolsets continue to populate `payload["tools"]` immediately, as today. + +#### Changes required + +| Area | Change | +|---|---| +| `BaseToolSet` | Add `deferred: bool = False` field | +| `OrchestratorConfig` | Add `tool_discovery: ToolDiscoveryConfig` sub-config (`enabled`, `routing_deployment`, `min_tools_for_deferral`, `min_tokens_for_deferral`) | +| Toolset initialisation modules | After building the catalog, apply deferral thresholds; promote under-threshold toolsets to eager; for remaining deferred toolsets build `ToolCatalog` + `definitions` map and skip `provide_openai_tools` contribution | +| `AgentModule` | Inject `ToolCatalogRegistry` (merged catalog across all effectively-deferred toolsets); expose `tool_search` and `tool_discovery` as `StagedBaseTool` implementations via `@multiprovider` | +| `tool_search` tool | Fires isolated `AssistantInvoker`-like completion; no messages/system prompt, only catalog context | +| `tool_discovery` tool | Dict lookup on `ToolCatalogRegistry.definitions`; no LLM call | +| `orchestrator.py` | After each iteration, check tool results for `tool_discovery` outputs; merge returned schemas into `_pending_tools`; pass to `_ChatCompletionConfigBuilder` on next call | +| `_ChatCompletionConfigBuilder` | Accept `extra_tool_dicts` parameter; merge into `payload["tools"]` | +| State serialisation (optional) | Persist `_pending_tools` names in `custom_content.state["discovered_tools"]` for cross-turn reuse | + +**Round-trip cost:** +2 turns before first native tool use (search → discovery → tool call). +Subsequent calls to the same tool within a turn are free (already in `_pending_tools`). With +cross-turn state, rediscovery is skipped on later turns. + +**Token cost of `tool_search` call:** +`catalog_tokens(N tools) + query_tokens` — independent of conversation length. For 200 tools +with 20-token descriptions each, this is ~4 K tokens regardless of how long the conversation is. + +**Pros:** +- Fully model-agnostic: works with any DIAL deployment as the orchestrator. +- Separate LLM routing call avoids spending main-context tokens on search; scales with catalog + size, not conversation size. +- Native schema injection (Path A) means the main LLM always calls discovered tools with proper + structured arguments — no prompt workarounds. +- Non-breaking opt-in: `deferred: false` by default preserves all existing behaviour. +- Routing model is configurable — can use a cheap/fast deployment to minimise cost. +- Extensible: the `tool_search` implementation can be swapped to embedding-based or keyword-only + without changing the orchestrator or injection logic. + +**Cons:** +- +2 round-trips before first native use of a deferred tool. +- The separate chat completion introduces a new code path for firing isolated completions. +- Orchestrator needs `_pending_tools` state; cross-turn persistence requires state serialisation. +- `tool_search` quality depends on the routing model and catalog description quality. --- -## Recommendation +## Comparison -**MVP: Option 1** (per-toolset `deferred` flag + keyword-search discovery tool). +| | Option 1 | Option 2 | Option 3 | Option 4 | Option 5 | **Option 6** | +|---|---|---|---|---|---|---| +| Model-agnostic | Yes | Yes | Yes | No (Anthropic 4.5+) | Yes | **Yes** | +| Orchestrator changes | None | None | Significant | Medium | Medium | **Medium** | +| Native schema on discovered tool call | No | No | Yes | Yes | Depends | **Yes** | +| Full name-space visible upfront | No | Yes | No | No | No | **No** | +| Prompt cache preserved | — | — | — | Yes (by design) | — | **Partial** ¹ | +| Per-tool granularity | Toolset | Toolset | Toolset | Per-tool | Per-tool | **Toolset** | +| Search uses separate LLM call | No | No | No | No (server-side) | No | **Yes** | +| Implementation complexity | Low | Low | High | Medium | Medium–High | **Medium** | +| Follows existing lazy pattern | Yes | Partial | No | No | No | **Partial** | +| Custom search logic possible | Yes | Yes | Yes | Yes | Yes | **Yes** | + +¹ Deferred tools are absent from `payload["tools"]` in main calls, so the stable tools prefix +(always-on tools + meta-tools) is cacheable. Discovered tools appended per-iteration break the +cache for that iteration only. + +--- -Rationale: -- Directly mirrors `LazyOnDemandStrategyModule` — the implementation pattern is already proven. -- No orchestrator changes keeps the blast radius small. -- Delivers meaningful token savings (entire toolset suppressed) with an opt-in default. -- Leaves room to upgrade to Option 3 once discovery behaviour is validated in production. +## Recommendation -**Follow-on: Option 3** once the MVP is validated. It removes the prompt-engineering dependency -and makes discovered tools first-class. +**Option 6** is the recommended approach. It is the only option that is simultaneously +model-agnostic, delivers native schema-based tool calling after discovery, and isolates the +search token cost from the conversation context. -**Option 4** (semantic) can be offered as an alternative search strategy within either Option 1 -or 3 as a configuration toggle. +Options 1–3 are useful reference points for the individual sub-problems Option 6 combines. +Option 4 is the right choice if the team decides to target Anthropic deployments exclusively +and wants to offload search infrastructure entirely. Option 5 (full MCP three-layer + dynamic +server management) remains the long-term architecture for MCP toolsets and can be layered on +top of Option 6 incrementally. --- ## Open Questions -1. **Granularity:** Should `deferred` live on `BaseToolSet` (per-toolset) or also be settable - per-tool inside a toolset? Per-toolset is simpler and covers the main use case (defer an - entire MCP server); per-tool is more surgical for mixed sets. +1. **Granularity:** `deferred` on `BaseToolSet` (per-toolset) or also settable per-tool within + a toolset? Per-toolset covers MCP servers and REST API groups cleanly; per-tool is needed + only for mixed toolsets where some tools are always-on. -2. **Discovery tool naming:** One tool per deferred toolset (`{toolset}_discover`) or a single - global `discover_tools(toolset?: str, query: str)`? A single tool reduces clutter in the - `tools` array but couples discovery to a specific naming convention. +2. **Routing model:** Should `routing_deployment` default to the orchestrator deployment, or + should there be a system-wide fallback configured at the application level? -3. **Search quality:** Is substring keyword match on name + description sufficient for MVP, or - should we index descriptions (e.g. TF-IDF) for better recall? Keyword match is simple and - deterministic; TF-IDF adds a build-time index step. +3. **Search implementation in MVP:** Plain keyword/substring match on catalog names and + descriptions, or a real LLM routing call from the start? Keyword match is deterministic and + has no latency; LLM routing handles synonyms and fuzzy intent but adds a network call. -4. **System-prompt alignment:** Should deferred toolset descriptions be injected into the - system prompt (to help the model decide when to search) or omitted entirely? Including a - brief catalog summary reduces false-negative discovery calls. +4. **Multi-turn persistence:** Serialise `_pending_tools` into `custom_content.state` so + rediscovery is skipped on subsequent turns, or always rediscover? Persistence saves + round-trips but grows state size. -5. **Conversation-turn persistence (Option 3):** Should discovered tool names persist across - turns (stored in conversation state) so the model does not need to rediscover on every turn? - This trades discovery latency for state-size growth. +5. **Always-on tools threshold:** Should any heuristic automatically promote a recently + discovered tool to always-on (e.g. if it has been discovered in the last N turns), or is + that always explicit config? --- @@ -234,8 +549,16 @@ or 3 as a configuration toggle. | Item | Reason | |---|---| -| Semantic / embedding-based search | Post-MVP strategy swap | -| Per-tool granularity within a toolset | Per-toolset is sufficient for initial use case | -| Dynamic schema injection into `payload["tools"]` | Option 3 — follow-on | -| Result caching across turns | State management complexity; defer | -| Provider-specific tool-pagination APIs | Not available via DIAL today | +| Embedding-based or subagent-based search | Swap-in strategy on top of Option 6 search step | +| Dynamic MCP server connection/disconnection | Significant lifecycle change; Option 5 follow-on | +| Per-tool granularity within a toolset | Per-toolset is sufficient for the initial use case | +| Automatic context-window threshold triggering | Always-opt-in is simpler and more predictable | + +--- + +## References + +- [MCP Client Best Practices — Progressive Tool Discovery](https://modelcontextprotocol.io/docs/2026-07-28/develop/clients/client-best-practices) +- [Anthropic Tool Search Tool](https://platform.claude.com/docs/en/agents-and-tools/tool-use/tool-search-tool) +- [Anthropic — Advanced Tool Use](https://www.anthropic.com/engineering/advanced-tool-use) +- [Anthropic — Effective Context Engineering](https://www.anthropic.com/engineering/effective-context-engineering-for-ai-agents) From 476e3b56c578770bb9eb3df702ba3fc688854b2f Mon Sep 17 00:00:00 2001 From: Oleksandr Gubarets Date: Wed, 9 Sep 2026 15:08:08 +0300 Subject: [PATCH 03/10] implement deferred tool context for MCP and REST toolsets. --- README.md | 176 ++++++----- docker-compose.yml | 3 +- docs/designs/dynamic_tool_discovery.md | 295 +++++++++--------- docs/generated-app-schema.json | 72 +++++ src/quickapp/app_factory.py | 2 + src/quickapp/common/tool_names.py | 2 + src/quickapp/config/application.py | 5 + src/quickapp/config/toolsets/base.py | 8 + .../agent/_chat_completion_config_builder.py | 16 +- src/quickapp/core/agent/agent_module.py | 12 +- .../mcp_tooling/_mcp_tool_initializer.py | 42 ++- .../rest_api_tooling_module.py | 35 ++- src/quickapp/tool_discovery/__init__.py | 0 .../tool_discovery/_anonymous_agent.py | 72 +++++ .../tool_discovery/_deferred_tools_context.py | 36 +++ .../_lazy_loaded_tools_holder.py | 25 ++ src/quickapp/tool_discovery/_tool_configs.py | 35 +++ .../tool_discovery/_tool_discovery_config.py | 46 +++ .../_tool_search_stage_wrapper.py | 19 ++ .../tool_discovery/_tool_search_tool.py | 93 ++++++ .../tool_discovery/tool_discovery_module.py | 38 +++ .../integration_tests/test_runner/config.py | 3 + .../agent_tests/test_assistant_invoker.py | 2 + .../test_tool_choice_config_builder.py | 3 + .../test_mcp_initializer_interactive_login.py | 2 + .../test_mcp_tool_initializer.py | 15 + 26 files changed, 820 insertions(+), 237 deletions(-) create mode 100644 src/quickapp/tool_discovery/__init__.py create mode 100644 src/quickapp/tool_discovery/_anonymous_agent.py create mode 100644 src/quickapp/tool_discovery/_deferred_tools_context.py create mode 100644 src/quickapp/tool_discovery/_lazy_loaded_tools_holder.py create mode 100644 src/quickapp/tool_discovery/_tool_configs.py create mode 100644 src/quickapp/tool_discovery/_tool_discovery_config.py create mode 100644 src/quickapp/tool_discovery/_tool_search_stage_wrapper.py create mode 100644 src/quickapp/tool_discovery/_tool_search_tool.py create mode 100644 src/quickapp/tool_discovery/tool_discovery_module.py diff --git a/README.md b/README.md index d72c23a8..98bb4d86 100644 --- a/README.md +++ b/README.md @@ -40,7 +40,8 @@ Features in Preview are marked with a `[Preview]` tag in documentation. - [Configuration Reference](./CONFIGURATION.md) - Full configuration model, environment variables, and examples - [Agent Skills](docs/skills.md) - How to create and manage reusable agent skills -- [Config-Driven Hooks](docs/designs/config_driven_hooks.md) `[Preview]` - Declarative synthetic tool call injection at orchestrator seams +- [Config-Driven Hooks](docs/designs/config_driven_hooks.md) `[Preview]` - Declarative synthetic tool call injection at + orchestrator seams - [Technical Documentation](./docs/README.md) - Internal architecture and design documents ## Quick start (general) @@ -59,7 +60,8 @@ file: ### Hooks `[Preview]` -Hooks let you pre-populate the agent's message history with synthetic tool call results — without writing Python code. Each hook fires at a named orchestrator seam and injects a `(ASSISTANT/tool_calls, TOOL)` message pair. +Hooks let you pre-populate the agent's message history with synthetic tool call results — without writing Python code. +Each hook fires at a named orchestrator seam and injects a `(ASSISTANT/tool_calls, TOOL)` message pair. Enable with `ENABLE_PREVIEW_FEATURES=true`, then add a `hooks` array to the app manifest: @@ -71,7 +73,9 @@ Enable with `ENABLE_PREVIEW_FEATURES=true`, then add a `hooks` array to the app "event": "on_request_start", "toolset_name": "memory_server", "tool_name": "get_memories", - "arguments": { "user_id": "123" }, + "arguments": { + "user_id": "123" + }, "frequency": "always" } ] @@ -80,14 +84,14 @@ Enable with `ENABLE_PREVIEW_FEATURES=true`, then add a `hooks` array to the app Key fields: -| Field | Description | -|---|---| -| `kind` | Hook type. Only `"tool_call"` is supported today. | -| `event` | Orchestrator seam. Only `"on_request_start"` is wired today. | -| `toolset_name` | Toolset prefix for REST API / MCP tools. Omit for DIAL Deployment and Internal tools. | -| `tool_name` | Tool name within the toolset, or the exact function name when `toolset_name` is omitted. | -| `arguments` | Arguments forwarded to the tool call. | -| `frequency` | `"always"` — inject on every request. `"append_if_changed"` (default) — inject only when the result differs from the last injection. | +| Field | Description | +|----------------|--------------------------------------------------------------------------------------------------------------------------------------| +| `kind` | Hook type. Only `"tool_call"` is supported today. | +| `event` | Orchestrator seam. Only `"on_request_start"` is wired today. | +| `toolset_name` | Toolset prefix for REST API / MCP tools. Omit for DIAL Deployment and Internal tools. | +| `tool_name` | Tool name within the toolset, or the exact function name when `toolset_name` is omitted. | +| `arguments` | Arguments forwarded to the tool call. | +| `frequency` | `"always"` — inject on every request. `"append_if_changed"` (default) — inject only when the result differs from the last injection. | See [Config-Driven Hooks design doc](docs/designs/config_driven_hooks.md) for the full reference. @@ -106,13 +110,14 @@ your gateways or downstream services expect. ### Stage display level -Controls which tool-execution stages are surfaced in the DIAL UI for each app. Set `features.stage_display.level` in the app manifest: +Controls which tool-execution stages are surfaced in the DIAL UI for each app. Set `features.stage_display.level` in the +app manifest: -| Value | Behavior | -|---|---| -| `none` | No stages shown at all, not even for errors | -| `error` | Show stages only for failed tool calls | -| `info` | Show stages for regular tool calls and errors (default) | +| Value | Behavior | +|---------|----------------------------------------------------------------| +| `none` | No stages shown at all, not even for errors | +| `error` | Show stages only for failed tool calls | +| `info` | Show stages for regular tool calls and errors (default) | | `debug` | Show stages for all tool calls, including internal/system ones | ```json @@ -127,75 +132,77 @@ Controls which tool-execution stages are surfaced in the DIAL UI for each app. S ### Environment Variables -| Variable | Default | Required | Description | -|--------------------------------------------|----------------------------|----------|--------------------------------------------------------------------------------------------------------------| -| **DIAL Core** | | | | -| `DIAL_URL` | — | Yes | URL of the DIAL Core API | -| `DIAL_API_VERSION` | `2025-01-01-preview` | No | API version for DIAL Core API | -| `APP_SCHEMA_ID` | `https://mydial.epam.com/custom_application_schemas/quickapps2` | No | Full application type schema `$id` emitted in the generated app schema. When unset, the built-in default is used. | -| **Proxy** | | | | -| `PROXY_LANGUAGE_HEADER` | `accept-language` | No | Name of the incoming HTTP request header that carries the locale for UI display (stage name localization). Override when a reverse proxy rewrites the standard `Accept-Language` header before forwarding the request. | -| **Logging** | | | | -| `DIAL_SDK_LOG_FORMAT` | `text` | No | Console log output format: `text` (human-readable) or `json` (escape-safe, one record per line). See [docs/logging.md](docs/logging.md). | -| `DIAL_SDK_TEXT_LOG_FORMAT` | [see docs/logging.md](docs/logging.md) | No | Custom `%`-style format string for `text` output. Unset (default) keeps the built-in format with the conditional OTEL trace block. | -| `DIAL_SDK_JSON_LOG_FORMAT` | [see docs/logging.md](docs/logging.md) | No | Custom template for `json` output — a JSON document whose string leaves are `%`-style format strings, values escaped via `json.dumps`. | -| `LOG_LEVEL` | `INFO` | No | Root logger level (all loggers except quickapp) | -| `QUICKAPP_LOG_LEVEL` | `INFO` | No | Log level for quickapp loggers | -| `LOG_PAYLOADS` | `false` | No | Emit payload content (message bodies, tool-call arguments, tool/LLM response bodies) at DEBUG. When `false`, no payload content is logged at **any** level and the payload-capable third-party loggers (`openai`/`httpx`/`httpcore`) are capped at INFO. **Local development only** — see [Payload Logging](#payload-logging). | -| `LOG_PAYLOADS_MAX_LENGTH` | `2000` | No | Per-field character cap applied to each payload value when `LOG_PAYLOADS=true`; longer values are truncated. Inert when `LOG_PAYLOADS=false`. | -| **Agent** | | | | -| `DEFAULT_AGENT_MAX_ITERATIONS` | `15` | No | Maximum number of orchestrator iterations (`-1` for infinite) | -| `DEFAULT_ORCHESTRATOR_DEPLOYMENT_ID` | — | No | Default DIAL deployment id used as the orchestrator model when a QuickApp manifest omits `orchestrator.deployment`. Also surfaces as the JSON-schema `default` for that field so DIAL Core can pre-fill new manifests. Apps can override per-app. | -| `SHOW_USAGE_STATISTICS` | `false` | No | Include usage statistics in chat completion stream | -| `SHOW_EXECUTION_TIME_STAGE` | `false` | No | Show execution time stage in the UI | -| **Python Interpreter** | | | | -| `PY_INTERPRETER_LOCAL_RUN` | `false` | No | Run PyInterpreter locally instead of via DIAL Core API | -| `PY_INTERPRETER_URL` | *(falls back to DIAL_URL)* | No | URL of the PyInterpreter service | -| `PY_INTERPRETER_API_KEY` | — | No | API key for local-run PyInterpreter | -| `PY_INTERPRETER_DEFAULT_SESSION_ID` | — | No | Default session ID for the PyInterpreter | -| `PY_INTERPRETER_CLIENT_MAX_RETRIES` | `3` | No | Max retries for PyInterpreter client requests | -| **Tool Timeouts** | | | | -| `DEFAULT_TOOL_TIMEOUT_SECONDS` | `300.0` | No | Deployment-wide default timeout (seconds, `0 < x ≤ 3600`) applied to every tool call (deployment, REST API, MCP, Python interpreter). Apps can override per-app via `tool_defaults.timeout_seconds`. | -| `DEFAULT_FILE_LOADING_SIZE_LIMIT` | `10485760` | No | Deployment-wide default maximum size (in bytes) for files the agent downloads. Apps can override per-app via `features.file_loading.size_limit`. | -| **Stage Display** | | | | -| `DEFAULT_STAGE_DISPLAY_LEVEL` | — | No | Deployment-wide override for stage visibility threshold (`none`, `error`, `info`, `debug`; case-insensitive). When set, wins over every app's `features.stage_display.level`. Unset (default) defers to the per-app config, which defaults to `info`. | -| **DIAL Files — Tool-Response Offload** | | | | -| `TOOL_CALL_RESULT_OFFLOAD__ENABLED_BY_DEFAULT` | `true` | No | Default value of the per-app `enabled` flag (`features.dial_files.tool_call_result_offload.enabled`). Apps override per-app; `enabled: false` disables offload for that app. | -| `TOOL_CALL_RESULT_OFFLOAD__SIZE_THRESHOLD` | `40000` | No | Default byte threshold above which a tool-call response is offloaded to a DIAL file. Apps override per-app via `features.dial_files.tool_call_result_offload.size_threshold`. | -| `TOOL_CALL_RESULT_OFFLOAD__EXCLUDED_TOOLS` | `[]` | No | Default JSON list of **additional** tool names exempt from offloading. The read-back tools (`internal_file_read_lines`, `internal_file_search`) are always excluded regardless of this value, so a large read-back slice is never re-offloaded. Apps add more per-app via `features.dial_files.tool_call_result_offload.excluded_tools`. | -| **External URL Egress** | | | | -| `EXTERNAL_URL_FETCH_ENABLED` | `false` | No | Admin cap on fetching external (non-DIAL) URLs. When `false` (default), no app may fetch external URLs regardless of its manifest; the deployment-handoff branch (deployments with `features.url_attachments`) is unaffected. Apps can opt out per-app via `features.external_url_fetch.enabled=false` even when the admin allows. | -| `EXTERNAL_URL_FETCH_HOST_ALLOWLIST` | — | No | Comma-separated allowlist of host patterns for external URL fetches. Unset (default) means no admin-level host restriction. Patterns: exact host (`example.com`) or `*.example.com` for any subdomain. Re-checked on every redirect hop. Per-app `features.external_url_fetch.host_allowlist` narrows further (intersection) but never expands. | -| `EXTERNAL_URL_FETCH_MAX_REDIRECTS` | `5` | No | Maximum HTTP redirects on external URL fetches. Each hop is SSRF-checked. Hard ceiling 10. | -| `EXTERNAL_URL_FETCH_CONNECT_TIMEOUT_SECONDS` | `5.0` | No | TCP connect timeout (seconds) for external URL fetches. Read/write/pool timeouts use the resolved tool timeout. | -| **Feature Gating** | | | | -| `ENABLE_PREVIEW_FEATURES` | `false` | No | Enable preview features across the deployment (schema visibility + runtime activation) | -| **Templates** | | | | -| `PREDEFINED_EXTRA_PATHS` | — | No | JSON list of directories layered on top of built-in predefined content (later entries override earlier ones) | -| `CONFIG_PROMPT_MAPPING` | *(built-in mapping)* | No | JSON mapping of predefined system prompts to DIAL Core deployments | -| **Observability** | | | | -| `OTEL_SERVICE_NAME` | `quickapps` | No | Service name stamped on all exported telemetry (traces, metrics, logs) | -| `OTEL_TRACES_EXPORTER` | — | No | Set to `otlp` to enable tracing and export spans over OTLP/gRPC. Instruments the FastAPI server and outgoing HTTP clients (`httpx`, `requests`, `aiohttp`, `urllib`) and stamps trace context onto log records — see [docs/logging.md](docs/logging.md). | -| `OTEL_METRICS_EXPORTER` | — | No | Comma-separated metric exporters: `otlp` (push over OTLP/gRPC) and/or `prometheus` (serve a scrape endpoint). Enables FastAPI and system/process metrics. | -| `OTEL_LOGS_EXPORTER` | — | No | Set to `otlp` to export log records (INFO and above) over OTLP/gRPC alongside console output — see [docs/logging.md](docs/logging.md). | -| `OTEL_EXPORTER_OTLP_ENDPOINT` | `http://localhost:4317` | No | OTLP/gRPC collector endpoint shared by trace, metric, and log export. One of the [standard OpenTelemetry SDK variables](https://opentelemetry.io/docs/specs/otel/configuration/sdk-environment-variables/), which the underlying exporters honor as usual (per-signal endpoints, headers, timeouts, resource attributes, …). | -| `OTEL_EXPORTER_PROMETHEUS_PORT` | `9464` | No | Port of the Prometheus scrape endpoint (effective only with `prometheus` in `OTEL_METRICS_EXPORTER`) | -| **Scripts & Tests** | | | | -| `REMOTE_DIAL_URL` | — | No | URL of the remote DIAL Core, used only by `generate_dial_config` script and e2e/integration tests | -| `REMOTE_DIAL_API_KEY` | — | No | API key of the remote DIAL Core, used only by `generate_dial_config` script and e2e/integration tests | +| Variable | Default | Required | Description | +|------------------------------------------------|-----------------------------------------------------------------|----------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| **DIAL Core** | | | | +| `DIAL_URL` | — | Yes | URL of the DIAL Core API | +| `DIAL_API_VERSION` | `2025-01-01-preview` | No | API version for DIAL Core API | +| `APP_SCHEMA_ID` | `https://mydial.epam.com/custom_application_schemas/quickapps2` | No | Full application type schema `$id` emitted in the generated app schema. When unset, the built-in default is used. | +| **Proxy** | | | | +| `PROXY_LANGUAGE_HEADER` | `accept-language` | No | Name of the incoming HTTP request header that carries the locale for UI display (stage name localization). Override when a reverse proxy rewrites the standard `Accept-Language` header before forwarding the request. | +| **Logging** | | | | +| `DIAL_SDK_LOG_FORMAT` | `text` | No | Console log output format: `text` (human-readable) or `json` (escape-safe, one record per line). See [docs/logging.md](docs/logging.md). | +| `DIAL_SDK_TEXT_LOG_FORMAT` | [see docs/logging.md](docs/logging.md) | No | Custom `%`-style format string for `text` output. Unset (default) keeps the built-in format with the conditional OTEL trace block. | +| `DIAL_SDK_JSON_LOG_FORMAT` | [see docs/logging.md](docs/logging.md) | No | Custom template for `json` output — a JSON document whose string leaves are `%`-style format strings, values escaped via `json.dumps`. | +| `LOG_LEVEL` | `INFO` | No | Root logger level (all loggers except quickapp) | +| `QUICKAPP_LOG_LEVEL` | `INFO` | No | Log level for quickapp loggers | +| `LOG_PAYLOADS` | `false` | No | Emit payload content (message bodies, tool-call arguments, tool/LLM response bodies) at DEBUG. When `false`, no payload content is logged at **any** level and the payload-capable third-party loggers (`openai`/`httpx`/`httpcore`) are capped at INFO. **Local development only** — see [Payload Logging](#payload-logging). | +| `LOG_PAYLOADS_MAX_LENGTH` | `2000` | No | Per-field character cap applied to each payload value when `LOG_PAYLOADS=true`; longer values are truncated. Inert when `LOG_PAYLOADS=false`. | +| **Agent** | | | | +| `DEFAULT_AGENT_MAX_ITERATIONS` | `15` | No | Maximum number of orchestrator iterations (`-1` for infinite) | +| `DEFAULT_ORCHESTRATOR_DEPLOYMENT_ID` | — | No | Default DIAL deployment id used as the orchestrator model when a QuickApp manifest omits `orchestrator.deployment`. Also surfaces as the JSON-schema `default` for that field so DIAL Core can pre-fill new manifests. Apps can override per-app. | +| `SHOW_USAGE_STATISTICS` | `false` | No | Include usage statistics in chat completion stream | +| `SHOW_EXECUTION_TIME_STAGE` | `false` | No | Show execution time stage in the UI | +| **Python Interpreter** | | | | +| `PY_INTERPRETER_LOCAL_RUN` | `false` | No | Run PyInterpreter locally instead of via DIAL Core API | +| `PY_INTERPRETER_URL` | *(falls back to DIAL_URL)* | No | URL of the PyInterpreter service | +| `PY_INTERPRETER_API_KEY` | — | No | API key for local-run PyInterpreter | +| `PY_INTERPRETER_DEFAULT_SESSION_ID` | — | No | Default session ID for the PyInterpreter | +| `PY_INTERPRETER_CLIENT_MAX_RETRIES` | `3` | No | Max retries for PyInterpreter client requests | +| **Tool Timeouts** | | | | +| `DEFAULT_TOOL_TIMEOUT_SECONDS` | `300.0` | No | Deployment-wide default timeout (seconds, `0 < x ≤ 3600`) applied to every tool call (deployment, REST API, MCP, Python interpreter). Apps can override per-app via `tool_defaults.timeout_seconds`. | +| `DEFAULT_FILE_LOADING_SIZE_LIMIT` | `10485760` | No | Deployment-wide default maximum size (in bytes) for files the agent downloads. Apps can override per-app via `features.file_loading.size_limit`. | +| **Stage Display** | | | | +| `DEFAULT_STAGE_DISPLAY_LEVEL` | — | No | Deployment-wide override for stage visibility threshold (`none`, `error`, `info`, `debug`; case-insensitive). When set, wins over every app's `features.stage_display.level`. Unset (default) defers to the per-app config, which defaults to `info`. | +| **DIAL Files — Tool-Response Offload** | | | | +| `TOOL_CALL_RESULT_OFFLOAD__ENABLED_BY_DEFAULT` | `true` | No | Default value of the per-app `enabled` flag (`features.dial_files.tool_call_result_offload.enabled`). Apps override per-app; `enabled: false` disables offload for that app. | +| `TOOL_CALL_RESULT_OFFLOAD__SIZE_THRESHOLD` | `40000` | No | Default byte threshold above which a tool-call response is offloaded to a DIAL file. Apps override per-app via `features.dial_files.tool_call_result_offload.size_threshold`. | +| `TOOL_CALL_RESULT_OFFLOAD__EXCLUDED_TOOLS` | `[]` | No | Default JSON list of **additional** tool names exempt from offloading. The read-back tools (`internal_file_read_lines`, `internal_file_search`) are always excluded regardless of this value, so a large read-back slice is never re-offloaded. Apps add more per-app via `features.dial_files.tool_call_result_offload.excluded_tools`. | +| **External URL Egress** | | | | +| `EXTERNAL_URL_FETCH_ENABLED` | `false` | No | Admin cap on fetching external (non-DIAL) URLs. When `false` (default), no app may fetch external URLs regardless of its manifest; the deployment-handoff branch (deployments with `features.url_attachments`) is unaffected. Apps can opt out per-app via `features.external_url_fetch.enabled=false` even when the admin allows. | +| `EXTERNAL_URL_FETCH_HOST_ALLOWLIST` | — | No | Comma-separated allowlist of host patterns for external URL fetches. Unset (default) means no admin-level host restriction. Patterns: exact host (`example.com`) or `*.example.com` for any subdomain. Re-checked on every redirect hop. Per-app `features.external_url_fetch.host_allowlist` narrows further (intersection) but never expands. | +| `EXTERNAL_URL_FETCH_MAX_REDIRECTS` | `5` | No | Maximum HTTP redirects on external URL fetches. Each hop is SSRF-checked. Hard ceiling 10. | +| `EXTERNAL_URL_FETCH_CONNECT_TIMEOUT_SECONDS` | `5.0` | No | TCP connect timeout (seconds) for external URL fetches. Read/write/pool timeouts use the resolved tool timeout. | +| **Dynamic Tool Discovery** `[Preview]` | | | | +| `MIN_TOOLS_FOR_DEFERRAL` | `5` | No | Deployment-wide minimum toolset size for deferral to apply. Toolsets with fewer tools than this threshold are promoted to eager loading even when `deferred=true`. Apps override per-app via `orchestrator.tool_discovery.min_tools_for_deferral`. Requires `ENABLE_PREVIEW_FEATURES=true`. | +| **Feature Gating** | | | | +| `ENABLE_PREVIEW_FEATURES` | `false` | No | Enable preview features across the deployment (schema visibility + runtime activation) | +| **Templates** | | | | +| `PREDEFINED_EXTRA_PATHS` | — | No | JSON list of directories layered on top of built-in predefined content (later entries override earlier ones) | +| `CONFIG_PROMPT_MAPPING` | *(built-in mapping)* | No | JSON mapping of predefined system prompts to DIAL Core deployments | +| **Observability** | | | | +| `OTEL_SERVICE_NAME` | `quickapps` | No | Service name stamped on all exported telemetry (traces, metrics, logs) | +| `OTEL_TRACES_EXPORTER` | — | No | Set to `otlp` to enable tracing and export spans over OTLP/gRPC. Instruments the FastAPI server and outgoing HTTP clients (`httpx`, `requests`, `aiohttp`, `urllib`) and stamps trace context onto log records — see [docs/logging.md](docs/logging.md). | +| `OTEL_METRICS_EXPORTER` | — | No | Comma-separated metric exporters: `otlp` (push over OTLP/gRPC) and/or `prometheus` (serve a scrape endpoint). Enables FastAPI and system/process metrics. | +| `OTEL_LOGS_EXPORTER` | — | No | Set to `otlp` to export log records (INFO and above) over OTLP/gRPC alongside console output — see [docs/logging.md](docs/logging.md). | +| `OTEL_EXPORTER_OTLP_ENDPOINT` | `http://localhost:4317` | No | OTLP/gRPC collector endpoint shared by trace, metric, and log export. One of the [standard OpenTelemetry SDK variables](https://opentelemetry.io/docs/specs/otel/configuration/sdk-environment-variables/), which the underlying exporters honor as usual (per-signal endpoints, headers, timeouts, resource attributes, …). | +| `OTEL_EXPORTER_PROMETHEUS_PORT` | `9464` | No | Port of the Prometheus scrape endpoint (effective only with `prometheus` in `OTEL_METRICS_EXPORTER`) | +| **Scripts & Tests** | | | | +| `REMOTE_DIAL_URL` | — | No | URL of the remote DIAL Core, used only by `generate_dial_config` script and e2e/integration tests | +| `REMOTE_DIAL_API_KEY` | — | No | API key of the remote DIAL Core, used only by `generate_dial_config` script and e2e/integration tests | #### Deprecated Environment Variables > [!CAUTION] > These variables still work but will be removed in a future major version. -| Variable | Replacement | Description | -|---------------------------------|--------------------------------------------------------------------|------------------------------------------------------------------------------------------------------------------------------| -| `PREDEFINED_BASE_PATH` | `PREDEFINED_EXTRA_PATHS` | If set alone, treated as a single extra layer on top of the built-in content | -| `PY_INTERPRETER_CLIENT_TIMEOUT` | `DEFAULT_TOOL_TIMEOUT_SECONDS` or `tool_defaults.timeout_seconds` | When set, still controls the PyInterpreter client timeout (seconds, default `60.0`), but the unified tool-timeout settings are preferred. | -| `LOG_FORMAT` | `DIAL_SDK_TEXT_LOG_FORMAT` or `DIAL_SDK_LOG_FORMAT=json` | When set, still controls the `text` output format (and wins over the replacements); a warning is emitted at startup. See [docs/logging.md](docs/logging.md). | -| `LOG_DATE_FORMAT` | — | Still honored alongside `LOG_FORMAT`; going forward the timestamp format is fixed to `%Y-%m-%d %H:%M:%S` (the previous default). | -| `OTEL_PYTHON_LOG_CORRELATION` | — *(automatic)* | Deprecated by aidial-sdk; a warning is emitted at startup. Trace fields are stamped onto log records whenever tracing is enabled, so the switch is redundant — and setting it installs OTel's legacy root-logger format, which double-logs SDK records and bypasses this service's console formatting. See [docs/logging.md](docs/logging.md). | +| Variable | Replacement | Description | +|---------------------------------|-------------------------------------------------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `PREDEFINED_BASE_PATH` | `PREDEFINED_EXTRA_PATHS` | If set alone, treated as a single extra layer on top of the built-in content | +| `PY_INTERPRETER_CLIENT_TIMEOUT` | `DEFAULT_TOOL_TIMEOUT_SECONDS` or `tool_defaults.timeout_seconds` | When set, still controls the PyInterpreter client timeout (seconds, default `60.0`), but the unified tool-timeout settings are preferred. | +| `LOG_FORMAT` | `DIAL_SDK_TEXT_LOG_FORMAT` or `DIAL_SDK_LOG_FORMAT=json` | When set, still controls the `text` output format (and wins over the replacements); a warning is emitted at startup. See [docs/logging.md](docs/logging.md). | +| `LOG_DATE_FORMAT` | — | Still honored alongside `LOG_FORMAT`; going forward the timestamp format is fixed to `%Y-%m-%d %H:%M:%S` (the previous default). | +| `OTEL_PYTHON_LOG_CORRELATION` | — *(automatic)* | Deprecated by aidial-sdk; a warning is emitted at startup. Trace fields are stamped onto log records whenever tracing is enabled, so the switch is redundant — and setting it installs OTel's legacy root-logger format, which double-logs SDK records and bypasses this service's console formatting. See [docs/logging.md](docs/logging.md). | **Notes:** @@ -223,7 +230,8 @@ content into the logs. `LOG_PAYLOADS=true` is the single, explicit exception: it re-enables the payload-bearing DEBUG records (message context, tool-call arguments, raw responses), each field truncated to `LOG_PAYLOADS_MAX_LENGTH`, and lifts the INFO cap on the wire-level third-party loggers (`openai`, `httpx`, `httpcore`). Every payload record is prefixed -with a `[payload]` marker so these lines can be found — or excluded — with a single filter. Forwarded header **values** are +with a `[payload]` marker so these lines can be found — or excluded — with a single filter. Forwarded header **values** +are never logged, even with the switch on. The switch is additive to the level — content appears only when `QUICKAPP_LOG_LEVEL=DEBUG` **and** `LOG_PAYLOADS=true`. @@ -365,7 +373,8 @@ never logged, even with the switch on. The switch is additive to the level — c - Notes: - If you want to run Quick Apps in Docker instead of on the host, update - [application-schemas.json](docker_compose_files/core/configuration/application-schemas.json) and change the Quick + [application-schemas.json](docker_compose_files/core/configuration/application-schemas.json) and change the + Quick Apps host from `host.docker.internal:5000` to `quick-apps:5000`. - When running via docker-compose the compose files set service hostnames (for example DIAL URL inside containers is http://core:8080). Those container-internal hostnames are not valid from your host machine — use @@ -446,7 +455,8 @@ never logged, even with the switch on. The switch is additive to the level — c ## E2E & Integration tests -Refer to [Testing Guide](./src/tests/integration_tests/README.md) for detailed instructions on setting up and running tests. +Refer to [Testing Guide](./src/tests/integration_tests/README.md) for detailed instructions on setting up and running +tests. ## More diff --git a/docker-compose.yml b/docker-compose.yml index fbef0ee0..e2d4a7a9 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -71,6 +71,7 @@ services: - core - keycloak environment: + NEXTAUTH_SECRET: "secret" AUTH_SESSION_SECRET: "48c69bd559cffffb2cbee3951430f324e0cd0c79fd8a83de0a991648cae706c8" AUTH_CALLBACK_BASE_URL: "http://localhost:3012" AUTH_POST_LOGOUT_REDIRECT_URI: "http://localhost:3012" @@ -81,7 +82,7 @@ services: THEMES_CONFIG_URL: "http://themes:8080" AUTH_KEYCLOAK_CLIENT_ID: "dial-local" AUTH_KEYCLOAK_SECRET: "dial-local-dev-secret" - AUTH_KEYCLOAK_HOST: "keycloak.localtest.me:8443/realms/dial-dev" + AUTH_KEYCLOAK_HOST: "https://keycloak.localtest.me:8443/realms/dial-dev" AUTH_KEYCLOAK_ADMIN_ROLE_NAMES: "admin" AUTH_KEYCLOAK_DIAL_ROLES_FIELD: "realm_access.roles" NODE_EXTRA_CA_CERTS: "/certs/ca.crt" diff --git a/docs/designs/dynamic_tool_discovery.md b/docs/designs/dynamic_tool_discovery.md index 30f2f082..28e4d631 100644 --- a/docs/designs/dynamic_tool_discovery.md +++ b/docs/designs/dynamic_tool_discovery.md @@ -1,7 +1,8 @@ # Design: Dynamic Tool Discovery -- **Status:** Draft +- **Status:** Implemented - **Issue:** [#430](https://github.com/epam/ai-dial-quickapps-backend/issues/430) +- **Chosen approach:** Option #6 ## Problem Statement @@ -277,138 +278,133 @@ registry, connect only when the model requests a server's capabilities. Requires --- -### Option 6 — Subagent-routed catalog search + orchestrator injection (Recommended) +### Option 6 — `DeferredRequestContext` + anonymous agent search + orchestrator injection (Recommended) **Mechanism:** -Combines a lightweight separate-LLM search step (for token-efficient routing) with orchestrator-level -tool injection (for native schema-based calling). It is model-agnostic and requires no -Anthropic-specific API features. +At request time, MCP and REST initializers split their output between the normal `RequestContext` +(eager tools) and a new `DeferredRequestContext` (deferred tools). A single `tool_search` meta-tool +is injected; when triggered it fires an isolated "anonymous agent" chat completion that consults +only the deferred catalog. The orchestrator intercepts the result and injects full schemas natively +into the next round. No Anthropic-specific API features are required. -#### Startup: catalog build +#### Step 1 — Request initialisation -Each toolset that opts in builds a compact **catalog entry** — `{name, description}` only — and -stores it in a per-toolset `ToolCatalog`. Full `OpenAiToolConfig` definitions are fetched and -held in memory as today but are **not forwarded to `payload["tools"]`**. +When a new chat completion request arrives, MCP and REST toolset initializers run as today and +build full `OpenAiToolConfig` definitions. They then apply the deferral decision: -```python -# Stored at init time, never sent to the main LLM upfront -catalog: list[ToolCatalogEntry] # [{name, description}, ...] -definitions: dict[str, OpenAiToolConfigDict] # name → full schema ``` +if toolset.deferred and len(tools) >= discovery.min_tools_for_deferral: + → push {name, description} entries + full definitions to DeferredRequestContext +else: + → push full definitions to RequestContext (eager, as today) +``` + +`DeferredRequestContext` holds two structures per toolset: +- **catalog**: `list[{name, description}]` — compact, never forwarded to the main LLM +- **definitions**: `dict[str, OpenAiToolConfigDict]` — full schemas, used by the lazy-initializer -#### Main orchestrator call +Toolsets with `deferred: false`, or that fall below the tool-count threshold, go straight into +`RequestContext` unchanged. -`payload["tools"]` contains only two meta-tools plus any always-on (non-deferred) tools: +#### Step 2 — Main orchestrator call + +`payload["tools"]` is built from `RequestContext` (eager tools) plus the single `tool_search` +meta-tool injected by `AgentModule`. Deferred tools are **absent**. ``` -tool_search(query: str) → list[{name, description}] -tool_discovery(tool_name: str) → full OpenAiToolConfig JSON +payload["tools"] = [tool_search, ...eager tools from RequestContext] +payload["messages"] = full conversation history ``` -Deferred tool definitions are **absent from the tools array**. The main LLM sees a minimal -surface. +The description of `tool_search` explicitly states that additional tools are available and can +be discovered on demand, so the model knows to search before assuming a capability is missing. -#### tool_search execution: separate chat completion +#### Step 3 — `tool_search` execution: anonymous agent -When the main LLM calls `tool_search`, the tool handler fires a **separate, isolated chat -completion** — a fresh context with no conversation history and no system prompt: +When the orchestrator calls `tool_search(query)`, its handler delegates to a new +**`AnonymousAgent`** module — a self-contained, isolated chat completion with no conversation +history and no system prompt from the main request: ``` -model: configurable (defaults to the orchestrator deployment; a cheap/fast - model such as a Haiku-class DIAL deployment is recommended) -messages: [{"role": "user", "content": }] -system: minimal routing instruction + - compact catalog injected as context (all names + descriptions) -tools: none +model: service_model (config: defaults to orchestrator deployment) +system: "You are a tool routing assistant. Given a user query, return the names of + the tools from the following catalog that are most relevant. + Catalog: [{name, description}, ...]" ← injected from DeferredRequestContext +messages:[{"role": "user", "content": }] +tools: none ``` -The routing LLM returns the best-matching tool names and descriptions. Because this call carries -no conversation history or system prompt, its token cost is proportional only to the catalog -size — not to conversation length. +The anonymous agent returns a list of tool names. Because this call carries no conversation +history and no application system prompt, its token cost is bounded by the catalog size alone. -**Result returned to main LLM:** `[{name, description}, ...]` +**Result returned to the main LLM:** `[{name, description}]` of matched tools, confirming what +is now available to load. -#### tool_discovery execution: local lookup, no LLM +#### Step 4 — Lazy-initializer builds OpenAI definitions -When the main LLM calls `tool_discovery(tool_name)`, the handler does a plain dictionary lookup -against the in-memory `definitions` map and returns the full `OpenAiToolConfig` JSON. No LLM -call is made. +The `tool_search` handler passes the matched tool names to an internal **lazy-initializer**, +which looks up each name in `DeferredRequestContext.definitions` and returns the corresponding +`OpenAiToolConfigDict` objects. No LLM call is made at this step. -**Result returned to main LLM:** full tool schema as JSON text in the tool result. +#### Step 5 — Orchestrator injection (Path A) -#### Orchestrator injection (Path A) +The orchestrator intercepts the `tool_search` result and: +1. Reads the list of matched tool names from the result. +2. Calls the lazy-initializer to retrieve their full `OpenAiToolConfigDict`s. +3. Accumulates them in `_lazy_loaded_tools: dict[str, OpenAiToolConfigDict]` (persists across + iterations within the turn). +4. On the **next** `_ChatCompletionConfigBuilder.build()` call, `_lazy_loaded_tools` is merged + into `payload["tools"]`. -The orchestrator intercepts any `tool_discovery` result and: -1. Parses the returned tool name(s) from the result. -2. Adds the corresponding `OpenAiToolConfigDict` to a per-iteration `_pending_tools: dict[str, OpenAiToolConfigDict]`. -3. On the **next** call to `_ChatCompletionConfigBuilder.build()`, the pending definitions are - merged into `payload["tools"]`. -4. The main LLM now sees the discovered tool natively and calls it with proper schema-based - argument generation. +The main LLM now sees the discovered tools natively alongside `tool_search` and the eager tools, +and calls them with proper schema-based argument generation. -Discovered tools accumulate across iterations within a turn. Optionally they are serialised -into `custom_content.state` so they persist across conversation turns (avoiding rediscovery on -the next user message). +Optionally, discovered tool names are serialised into `custom_content.state["lazy_loaded_tools"]` +so they survive across conversation turns, avoiding rediscovery on the next user message. #### Flow diagram ``` -Turn start - │ - ▼ -Main LLM call - tools = [tool_search, tool_discovery, ...always-on] - messages = full conversation history - │ - ├─ LLM calls tool_search("find Salesforce tools") - │ │ - │ └─ Separate chat completion (fresh context): - │ model = fast routing model - │ context = compact catalog (names + descriptions only) - │ input = "find Salesforce tools" - │ → [{name: "sf_query", description: "..."}, ...] - │ Result returned to main LLM +New request arrives │ - ├─ LLM calls tool_discovery("sf_query") - │ │ - │ └─ Local dict lookup → full OpenAiToolConfig JSON - │ Orchestrator registers "sf_query" in _pending_tools - │ Result returned to main LLM + ├─ MCP initializer: len(tools) >= threshold → DeferredRequestContext + │ (catalog + definitions) + ├─ REST initializer: len(tools) < threshold → RequestContext (eager) │ ▼ -Next main LLM call - tools = [tool_search, tool_discovery, ...always-on, sf_query ← injected] +Orchestrator — iteration 1 + payload["tools"] = [tool_search, ...eager tools] + payload["messages"] = full conversation history + │ + └─ Main LLM calls tool_search("I need to query Salesforce contacts") + │ + └─ AnonymousAgent (isolated chat completion): + model = service_model + system = routing prompt + catalog from DeferredRequestContext + message = "I need to query Salesforce contacts" + → ["sf_query_contacts", "sf_list_contacts"] + Lazy-initializer: names → full OpenAiToolConfigDicts + Orchestrator stores in _lazy_loaded_tools + Result returned to main LLM: [{name, description}, ...] + +Orchestrator — iteration 2 + payload["tools"] = [tool_search, ...eager tools, + sf_query_contacts ←injected, + sf_list_contacts ←injected] │ - └─ LLM calls sf_query(object="Account", ...) natively ✓ + └─ Main LLM calls sf_query_contacts(filter="LastName='Smith'") natively ✓ ``` #### Deferral threshold -Even when `deferred: true` is set, a toolset is loaded eagerly if it is small enough that -deferring it would cost more (extra round-trips) than it saves (token reduction). Two guards -are evaluated at startup; a toolset is deferred only when it clears **both**: - -| Guard | Config key | Default | Check | -|---|---|---|---| -| Tool count | `min_tools_for_deferral` | `5` | `len(catalog) >= threshold` | -| Token estimate | `min_tokens_for_deferral` | `1000` | `estimated_tokens >= threshold` | - -Token estimate is computed as `sum(len(json.dumps(schema)) for schema in definitions.values())` -— a cheap character-count proxy evaluated once at startup. It is intentionally approximate; -exact tokenisation is not worth the overhead here. +A toolset with `deferred: true` is only placed into `DeferredRequestContext` if its tool count +meets the minimum. Below the threshold it is loaded eagerly — no discovery overhead: ``` -deferred_effective = ( - config.deferred - and len(catalog) >= discovery.min_tools_for_deferral - and estimated_tokens >= discovery.min_tokens_for_deferral -) +deferred_effective = toolset.deferred and len(tools) >= discovery.min_tools_for_deferral ``` -A toolset with `deferred: false` is always loaded eagerly regardless of size. A toolset with -`deferred: true` that falls below either threshold is silently promoted to eager and its tools -are included in `payload["tools"]` as normal — no discovery overhead, no behavioural change -visible to the model. +`deferred: false` always means eager, regardless of count. #### Configuration @@ -417,9 +413,8 @@ visible to the model. "orchestrator": { "tool_discovery": { "enabled": true, - "routing_deployment": "claude-haiku-dial-deployment", - "min_tools_for_deferral": 5, - "min_tokens_for_deferral": 1000 + "service_model": "claude-haiku-dial-deployment", + "min_tools_for_deferral": 5 } }, "tool_sets": [ @@ -438,52 +433,57 @@ visible to the model. } ``` -- `deferred: true` on a toolset opts it into the catalog. Default: `false` (existing behaviour preserved). -- `routing_deployment` names the DIAL deployment used for the `tool_search` separate completion. - When omitted, it falls back to the orchestrator's own deployment. -- `min_tools_for_deferral` and `min_tokens_for_deferral` are global guards; toolsets that do - not clear both are silently promoted to eager loading. Both default to values that make - deferral a no-op for small toolsets. -- Non-deferred toolsets continue to populate `payload["tools"]` immediately, as today. +- `deferred: true` opts the toolset into `DeferredRequestContext`. Default: `false`. +- `service_model` names the DIAL deployment used for the `AnonymousAgent` chat completion. + When omitted, falls back to the orchestrator's own deployment. +- `min_tools_for_deferral` is the tool-count guard below which a deferred toolset is silently + promoted to eager. Default: `5`. +- Non-deferred and below-threshold toolsets populate `RequestContext` immediately, as today. #### Changes required | Area | Change | |---|---| | `BaseToolSet` | Add `deferred: bool = False` field | -| `OrchestratorConfig` | Add `tool_discovery: ToolDiscoveryConfig` sub-config (`enabled`, `routing_deployment`, `min_tools_for_deferral`, `min_tokens_for_deferral`) | -| Toolset initialisation modules | After building the catalog, apply deferral thresholds; promote under-threshold toolsets to eager; for remaining deferred toolsets build `ToolCatalog` + `definitions` map and skip `provide_openai_tools` contribution | -| `AgentModule` | Inject `ToolCatalogRegistry` (merged catalog across all effectively-deferred toolsets); expose `tool_search` and `tool_discovery` as `StagedBaseTool` implementations via `@multiprovider` | -| `tool_search` tool | Fires isolated `AssistantInvoker`-like completion; no messages/system prompt, only catalog context | -| `tool_discovery` tool | Dict lookup on `ToolCatalogRegistry.definitions`; no LLM call | -| `orchestrator.py` | After each iteration, check tool results for `tool_discovery` outputs; merge returned schemas into `_pending_tools`; pass to `_ChatCompletionConfigBuilder` on next call | -| `_ChatCompletionConfigBuilder` | Accept `extra_tool_dicts` parameter; merge into `payload["tools"]` | -| State serialisation (optional) | Persist `_pending_tools` names in `custom_content.state["discovered_tools"]` for cross-turn reuse | - -**Round-trip cost:** +2 turns before first native tool use (search → discovery → tool call). -Subsequent calls to the same tool within a turn are free (already in `_pending_tools`). With -cross-turn state, rediscovery is skipped on later turns. - -**Token cost of `tool_search` call:** -`catalog_tokens(N tools) + query_tokens` — independent of conversation length. For 200 tools -with 20-token descriptions each, this is ~4 K tokens regardless of how long the conversation is. +| `OrchestratorConfig` | Add `tool_discovery: ToolDiscoveryConfig` sub-config (`enabled`, `service_model`, `min_tools_for_deferral`) | +| `DeferredRequestContext` | New DI-scoped object: holds per-toolset `catalog` list and `definitions` dict; populated by initializers during request setup | +| MCP & REST initializer modules | After building tool definitions, evaluate `deferred_effective`; route to `DeferredRequestContext` or `RequestContext` accordingly | +| `AnonymousAgent` | New module: fires a single isolated `chat.completions.create` call (no history, no app system prompt); takes `service_model`, a system prompt with the catalog, and a user query; returns matched tool names | +| `tool_search` (`StagedBaseTool`) | New internal tool injected via `AgentModule @multiprovider`; calls `AnonymousAgent`, passes results to the lazy-initializer, returns `[{name, description}]` to the main LLM | +| Lazy-initializer | Thin helper: given a list of tool names, looks up `DeferredRequestContext.definitions` and returns `list[OpenAiToolConfigDict]` | +| `orchestrator.py` | After each iteration, detect `tool_search` results; call lazy-initializer; accumulate in `_lazy_loaded_tools`; pass to `_ChatCompletionConfigBuilder` on next call | +| `_ChatCompletionConfigBuilder` | Accept `lazy_tool_dicts: list[OpenAiToolConfigDict]`; merge into `payload["tools"]` | +| State serialisation (optional) | Persist `_lazy_loaded_tools` names in `custom_content.state["lazy_loaded_tools"]` for cross-turn reuse | + +**Round-trip cost:** +1 turn before first native tool use (search + inject → tool call). +The anonymous agent call happens inside the `tool_search` tool execution, not as a separate +orchestrator iteration. Subsequent calls to the same tool within a turn are free (already in +`_lazy_loaded_tools`). With cross-turn state, rediscovery is skipped on later turns. + +**Token cost of `tool_search` (anonymous agent call):** +`catalog_tokens + query_tokens` — independent of conversation length. For 200 deferred tools +with ~20-token descriptions each, this is ~4 K tokens regardless of how long the conversation is. **Pros:** -- Fully model-agnostic: works with any DIAL deployment as the orchestrator. -- Separate LLM routing call avoids spending main-context tokens on search; scales with catalog - size, not conversation size. -- Native schema injection (Path A) means the main LLM always calls discovered tools with proper - structured arguments — no prompt workarounds. -- Non-breaking opt-in: `deferred: false` by default preserves all existing behaviour. -- Routing model is configurable — can use a cheap/fast deployment to minimise cost. -- Extensible: the `tool_search` implementation can be swapped to embedding-based or keyword-only - without changing the orchestrator or injection logic. +- Fully model-agnostic: works with any DIAL deployment as orchestrator. +- `DeferredRequestContext` cleanly separates eager and deferred tool state at the DI layer — + no orchestrator logic needed to decide what to defer. +- Anonymous agent isolates search cost from main conversation tokens; scales with catalog size, + not conversation length. +- Native schema injection means the main LLM calls discovered tools with proper structured + arguments from the iteration after discovery. +- Non-breaking opt-in: `deferred: false` by default; threshold guard prevents regression for + small toolsets. +- `AnonymousAgent` is a reusable module independent of tool discovery. +- Search strategy is swappable (keyword, embedding, different model) without touching the + orchestrator or injection logic. **Cons:** -- +2 round-trips before first native use of a deferred tool. -- The separate chat completion introduces a new code path for firing isolated completions. -- Orchestrator needs `_pending_tools` state; cross-turn persistence requires state serialisation. -- `tool_search` quality depends on the routing model and catalog description quality. +- +1 orchestrator iteration before first native use of a deferred tool. +- `AnonymousAgent` introduces a new code path for isolated completions. +- `_lazy_loaded_tools` state in the orchestrator; optional cross-turn persistence requires + state serialisation. +- Search quality depends on service model and catalog description quality. --- @@ -502,9 +502,9 @@ with 20-token descriptions each, this is ~4 K tokens regardless of how long the | Follows existing lazy pattern | Yes | Partial | No | No | No | **Partial** | | Custom search logic possible | Yes | Yes | Yes | Yes | Yes | **Yes** | -¹ Deferred tools are absent from `payload["tools"]` in main calls, so the stable tools prefix -(always-on tools + meta-tools) is cacheable. Discovered tools appended per-iteration break the -cache for that iteration only. +¹ Deferred tools are absent from `payload["tools"]` in main calls, so the stable prefix +(eager tools + `tool_search`) is cacheable. Lazy-loaded tools appended after discovery break +the cache for that iteration only. --- @@ -528,20 +528,21 @@ top of Option 6 incrementally. a toolset? Per-toolset covers MCP servers and REST API groups cleanly; per-tool is needed only for mixed toolsets where some tools are always-on. -2. **Routing model:** Should `routing_deployment` default to the orchestrator deployment, or - should there be a system-wide fallback configured at the application level? +2. **`service_model` default:** Should it fall back to the orchestrator deployment, or require + explicit configuration? Defaulting to the orchestrator deployment is the simplest path but + misses the cost-saving opportunity of routing to a cheaper model. -3. **Search implementation in MVP:** Plain keyword/substring match on catalog names and - descriptions, or a real LLM routing call from the start? Keyword match is deterministic and - has no latency; LLM routing handles synonyms and fuzzy intent but adds a network call. +3. **Search implementation in MVP:** Pure LLM routing via `AnonymousAgent` from the start, or + offer a keyword-only fallback that skips the anonymous agent call entirely? Keyword match + has zero latency and no model dependency; LLM routing handles synonyms and fuzzy intent. -4. **Multi-turn persistence:** Serialise `_pending_tools` into `custom_content.state` so - rediscovery is skipped on subsequent turns, or always rediscover? Persistence saves - round-trips but grows state size. +4. **Multi-turn persistence:** Serialise `_lazy_loaded_tools` names into + `custom_content.state["lazy_loaded_tools"]` so rediscovery is skipped on subsequent turns, + or always rediscover? Persistence saves round-trips but grows state size. -5. **Always-on tools threshold:** Should any heuristic automatically promote a recently - discovered tool to always-on (e.g. if it has been discovered in the last N turns), or is - that always explicit config? +5. **Always-on threshold:** Should there be a heuristic that automatically promotes a + frequently-discovered tool to eager loading (e.g. seen in last N turns), or is that always + explicit config? --- @@ -549,10 +550,10 @@ top of Option 6 incrementally. | Item | Reason | |---|---| -| Embedding-based or subagent-based search | Swap-in strategy on top of Option 6 search step | +| Embedding-based search in `AnonymousAgent` | Swap-in strategy; `AnonymousAgent` interface is the extension point | | Dynamic MCP server connection/disconnection | Significant lifecycle change; Option 5 follow-on | | Per-tool granularity within a toolset | Per-toolset is sufficient for the initial use case | -| Automatic context-window threshold triggering | Always-opt-in is simpler and more predictable | +| Automatic threshold based on token count | Tool-count threshold is simpler and good enough for MVP | --- diff --git a/docs/generated-app-schema.json b/docs/generated-app-schema.json index 9e2b66c7..4d0d0428 100644 --- a/docs/generated-app-schema.json +++ b/docs/generated-app-schema.json @@ -684,6 +684,12 @@ "title": "Enabled", "type": "boolean" }, + "deferred": { + "default": true, + "description": "When true, this toolset's tool schemas are withheld from the initial LLM payload. Requires orchestrator.tool_discovery.enabled=true. Tools are discovered on demand via the tool_search meta-tool.", + "title": "Deferred", + "type": "boolean" + }, "type": { "const": "dial-deployment", "default": "dial-deployment", @@ -759,6 +765,12 @@ "title": "Enabled", "type": "boolean" }, + "deferred": { + "default": true, + "description": "When true, this toolset's tool schemas are withheld from the initial LLM payload. Requires orchestrator.tool_discovery.enabled=true. Tools are discovered on demand via the tool_search meta-tool.", + "title": "Deferred", + "type": "boolean" + }, "type": { "const": "dial-app", "default": "dial-app", @@ -1239,6 +1251,12 @@ "title": "Enabled", "type": "boolean" }, + "deferred": { + "default": true, + "description": "When true, this toolset's tool schemas are withheld from the initial LLM payload. Requires orchestrator.tool_discovery.enabled=true. Tools are discovered on demand via the tool_search meta-tool.", + "title": "Deferred", + "type": "boolean" + }, "type": { "const": "dial-mcp", "default": "dial-mcp", @@ -1821,6 +1839,12 @@ "title": "Enabled", "type": "boolean" }, + "deferred": { + "default": true, + "description": "When true, this toolset's tool schemas are withheld from the initial LLM payload. Requires orchestrator.tool_discovery.enabled=true. Tools are discovered on demand via the tool_search meta-tool.", + "title": "Deferred", + "type": "boolean" + }, "type": { "const": "internal", "default": "internal", @@ -2215,6 +2239,12 @@ "title": "Enabled", "type": "boolean" }, + "deferred": { + "default": true, + "description": "When true, this toolset's tool schemas are withheld from the initial LLM payload. Requires orchestrator.tool_discovery.enabled=true. Tools are discovered on demand via the tool_search meta-tool.", + "title": "Deferred", + "type": "boolean" + }, "type": { "const": "mcp", "default": "mcp", @@ -3063,6 +3093,12 @@ "title": "Enabled", "type": "boolean" }, + "deferred": { + "default": true, + "description": "When true, this toolset's tool schemas are withheld from the initial LLM payload. Requires orchestrator.tool_discovery.enabled=true. Tools are discovered on demand via the tool_search meta-tool.", + "title": "Deferred", + "type": "boolean" + }, "type": { "const": "rest-api", "default": "rest-api", @@ -3355,6 +3391,38 @@ "title": "ToolCallTimestampConfig", "type": "object" }, + "ToolDiscoveryConfig": { + "properties": { + "enabled": { + "default": false, + "description": "Enable dynamic tool discovery. When true, toolsets with deferred=true are withheld from the initial LLM payload and surfaced via the tool_search meta-tool.", + "title": "Enabled", + "type": "boolean" + }, + "service_model": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "DIAL deployment used for the anonymous routing call inside tool_search. When omitted, falls back to the orchestrator's own deployment.", + "title": "Service Model" + }, + "min_tools_for_deferral": { + "default": 5, + "description": "Minimum number of tools in a toolset for deferral to apply. Toolsets with fewer tools than this threshold are promoted to eager loading even when deferred=true, avoiding discovery overhead for small toolsets. Default: 5 (or the value of MIN_TOOLS_FOR_DEFERRAL env var)", + "minimum": 1, + "title": "Min Tools For Deferral", + "type": "integer" + } + }, + "title": "ToolDiscoveryConfig", + "type": "object" + }, "ToolDisplayConfig": { "properties": { "stage": { @@ -3611,6 +3679,10 @@ ], "default": null, "description": "How the orchestrator receives request-scoped attachments. When unset, the orchestrator gets no admin/user attachments on the native path (legacy behaviour: USER `image/*` passes through, other MIMEs are surfaced as XML metadata only)." + }, + "tool_discovery": { + "$ref": "#/$defs/ToolDiscoveryConfig", + "description": "Dynamic tool discovery configuration. When enabled, toolsets with deferred=true are withheld from the initial LLM payload and discovered on demand via the tool_search meta-tool." } }, "required": [ diff --git a/src/quickapp/app_factory.py b/src/quickapp/app_factory.py index 5b2651a4..20f5f5f6 100644 --- a/src/quickapp/app_factory.py +++ b/src/quickapp/app_factory.py @@ -31,6 +31,7 @@ from quickapp.skills.skills_module import SkillsModule from quickapp.starters.starters_module import StartersModule from quickapp.timestamp_tooling.timestamp_module import TimestampModule +from quickapp.tool_discovery.tool_discovery_module import ToolDiscoveryModule from quickapp.web_tooling.web_tooling_module import WebToolingModule @@ -59,6 +60,7 @@ def build_di_modules() -> list[Module]: SkillsModule(), DialPromptSkillsModule(), TimestampModule(), + ToolDiscoveryModule(), AgentHooksModule(), DialFilesToolingModule(), WebToolingModule(), diff --git a/src/quickapp/common/tool_names.py b/src/quickapp/common/tool_names.py index fcc3347d..8ffc88ea 100644 --- a/src/quickapp/common/tool_names.py +++ b/src/quickapp/common/tool_names.py @@ -15,6 +15,8 @@ # DIAL files tools — all share the ``internal_file_`` prefix. INTERNAL_FILE_TOOL_NAME_PREFIX = "internal_file_" +INTERNAL_TOOL_SEARCH_TOOL_NAME = "internal_tool_search" + INTERNAL_FILE_LIST_TOOL_NAME = f"{INTERNAL_FILE_TOOL_NAME_PREFIX}list" INTERNAL_FILE_READ_LINES_TOOL_NAME = f"{INTERNAL_FILE_TOOL_NAME_PREFIX}read_lines" INTERNAL_FILE_SEARCH_TOOL_NAME = f"{INTERNAL_FILE_TOOL_NAME_PREFIX}search" diff --git a/src/quickapp/config/application.py b/src/quickapp/config/application.py index e1b7b280..a24521f1 100644 --- a/src/quickapp/config/application.py +++ b/src/quickapp/config/application.py @@ -24,6 +24,7 @@ from quickapp.config.timestamp import TimestampConfig, ToolCallTimestampConfig from quickapp.config.toolsets.toolset import ToolSet from quickapp.config.web_fetch import WebFetchConfig +from quickapp.tool_discovery._tool_discovery_config import ToolDiscoveryConfig logger = logging.getLogger(__name__) @@ -95,6 +96,10 @@ class OrchestratorConfig(BaseModel): "MIMEs are surfaced as XML metadata only)." ), ) + tool_discovery: ToolDiscoveryConfig = Field( + default_factory=ToolDiscoveryConfig, + description="Dynamic tool discovery configuration. When enabled, toolsets with deferred=true are withheld from the initial LLM payload and discovered on demand via the tool_search meta-tool.", + ) def nullify_preview_fields(model: BaseModel) -> None: diff --git a/src/quickapp/config/toolsets/base.py b/src/quickapp/config/toolsets/base.py index 22c5a42c..832af3cb 100644 --- a/src/quickapp/config/toolsets/base.py +++ b/src/quickapp/config/toolsets/base.py @@ -17,3 +17,11 @@ class BaseToolSet(BaseModel): default=None, description="The description of the tool set." ) enabled: bool = Field(default=True, description="Whether the toolset is enabled.") + deferred: bool = Field( + default=True, + description=( + "When true, this toolset's tool schemas are withheld from the initial LLM payload. " + "Requires orchestrator.tool_discovery.enabled=true. " + "Tools are discovered on demand via the tool_search meta-tool." + ), + ) diff --git a/src/quickapp/core/agent/_chat_completion_config_builder.py b/src/quickapp/core/agent/_chat_completion_config_builder.py index 5fa32045..7a0ac2b1 100644 --- a/src/quickapp/core/agent/_chat_completion_config_builder.py +++ b/src/quickapp/core/agent/_chat_completion_config_builder.py @@ -13,6 +13,7 @@ from quickapp.config.application import ApplicationConfig from quickapp.core.agent._tool_choice_holder import _ToolChoiceHolder from quickapp.core.agent.models import STATE_KEY_ORCHESTRATOR, OpenAiToolConfigDict +from quickapp.tool_discovery._lazy_loaded_tools_holder import _LazyLoadedToolsHolder logger = logging.getLogger(__name__) @@ -28,6 +29,7 @@ def __init__( pre_invocation_transformers: list[PreInvocationTransformer], presentation_settings: PresentationSettings, forwarded_headers: ForwardedHeaders, + lazy_loaded_tools_holder: _LazyLoadedToolsHolder, ) -> None: self.__config: ApplicationConfig = config self.__tools: list[OpenAiToolConfigDict] = tools @@ -36,17 +38,25 @@ def __init__( self.__pre_invocation_transformers = pre_invocation_transformers self.__presentation_settings = presentation_settings self.__forwarded_headers = forwarded_headers + self.__lazy_loaded_tools_holder = lazy_loaded_tools_holder def build(self, messages: list[Message]) -> dict[str, Any]: chat_completion_config = self.__config.orchestrator.deployment.parameters.model_dump( exclude_none=True ) prepared_messages = self._prepare_messages(messages) + eager_names: set[str] = {t.get("function", {}).get("name", "") for t in self.__tools} + lazy_tools = [ + t + for t in self.__lazy_loaded_tools_holder.get_all() + if t.get("function", {}).get("name", "") not in eager_names + ] + all_tools = self.__tools + lazy_tools payload: dict[str, Any] = { "messages": prepared_messages, "stream": True, "model": self.__config.orchestrator.deployment.deployment_id, - "tools": self.__tools, + "tools": all_tools, } if self.__response_format: @@ -75,11 +85,13 @@ def build(self, messages: list[Message]) -> dict[str, Any]: chat_completion_config.update(payload) if logger.isEnabledFor(logging.DEBUG): logger.debug( - "Chat completion config: messages=%d, roles=%s, tools=%d, response_format=%s, " + "Chat completion config: messages=%d, roles=%s, tools=%d (eager=%d, lazy=%d), response_format=%s, " "model=%s, forwarded_headers=%s", len(prepared_messages), summarize_roles(prepared_messages), + len(all_tools), len(self.__tools), + len(lazy_tools), "response_format" in chat_completion_config, chat_completion_config.get("model"), # Header NAMES only — forwarded X-* header values are never logged, even diff --git a/src/quickapp/core/agent/agent_module.py b/src/quickapp/core/agent/agent_module.py index a73c209f..9e9d25cc 100644 --- a/src/quickapp/core/agent/agent_module.py +++ b/src/quickapp/core/agent/agent_module.py @@ -62,6 +62,8 @@ OrchestratorDeploymentCacheService, ) from quickapp.core.application._request_context import _RequestContext +from quickapp.tool_discovery._deferred_tools_context import _DeferredToolsContext +from quickapp.tool_discovery._lazy_loaded_tools_holder import _LazyLoadedToolsHolder DEFAULT_QUERY_PARAM = ConfigurableSchemaSimpleType( type=JsonTypeEnum.string, @@ -103,6 +105,8 @@ def configure(self, binder: Binder) -> None: ) binder.bind(AssistantInvoker, to=AssistantInvoker, scope=NoScope) binder.bind(_ChatCompletionConfigBuilder, to=_ChatCompletionConfigBuilder, scope=NoScope) + binder.bind(_DeferredToolsContext, to=_DeferredToolsContext, scope=request_scope) + binder.bind(_LazyLoadedToolsHolder, to=_LazyLoadedToolsHolder, scope=request_scope) binder.bind(ChatStreamSinkFactory, to=ChatStreamSinkFactory, scope=NoScope) binder.bind(ChatCompletionStreamHandler, to=ChatCompletionStreamHandler, scope=NoScope) binder.bind(_AttachmentFilter, to=_AttachmentFilter, scope=request_scope) @@ -164,12 +168,18 @@ def provide_openai_client( @multiprovider def provide_openai_tools( - self, tools: list[StagedBaseTool], static_tools: list[StaticTool] + self, + tools: list[StagedBaseTool], + static_tools: list[StaticTool], + deferred_context: _DeferredToolsContext, ) -> list[OpenAiToolConfigDict]: + deferred_names = deferred_context.deferred_names openai_functions = [] for tool in tools: if isinstance(tool.tool_config, BaseOpenAITool): open_ai_tool: OpenAiToolConfig = tool.tool_config.open_ai_tool + if open_ai_tool.function.name in deferred_names: + continue open_ai_tool = self._remove_const_params(open_ai_tool) if isinstance(tool.tool_config, DialDeploymentTool): open_ai_tool = self._append_default_props(open_ai_tool) diff --git a/src/quickapp/mcp_tooling/_mcp_tool_initializer.py b/src/quickapp/mcp_tooling/_mcp_tool_initializer.py index b03c5596..6bdf9499 100644 --- a/src/quickapp/mcp_tooling/_mcp_tool_initializer.py +++ b/src/quickapp/mcp_tooling/_mcp_tool_initializer.py @@ -1,6 +1,6 @@ import asyncio import logging -from typing import Any +from typing import Any, cast from urllib.parse import unquote import httpx @@ -16,6 +16,7 @@ from quickapp.common.json_schema_converter import JsonSchemaConverter from quickapp.common.localized_string import resolve_localized from quickapp.common.utils import posix_path_last_segment, sanitize_toolname +from quickapp.config.application import ApplicationConfig from quickapp.config.tools.base import ( JsonTypeEnum, OpenAiToolConfig, @@ -32,6 +33,7 @@ from quickapp.mcp_tooling._mcp_eager_resource import MCPEagerTextResource from quickapp.mcp_tooling._mcp_resource_meta import MCPResourceMeta from quickapp.mcp_tooling._mcp_server_capabilities import MCPServerCapabilities +from quickapp.tool_discovery._deferred_tools_context import _DeferredToolsContext from ._di_types import DialToolsetCacheService from ._mcp_tool import _MCPTool @@ -131,6 +133,8 @@ def __init__( tool_config_service: ToolConfigCoreService, login_service: InteractiveLoginService, accept_language: ACCEPT_LANGUAGE, + app_config: ApplicationConfig, + deferred_context: _DeferredToolsContext, ): # Resolved lazily in initialize() because dial_app_tooling contributes # to this multibinder only after _DialAppResolver runs. @@ -146,6 +150,8 @@ def __init__( self.__tool_config_service: ToolConfigCoreService = tool_config_service self.__login_service: InteractiveLoginService = login_service self.__accept_language: ACCEPT_LANGUAGE = accept_language + self.__app_config: ApplicationConfig = app_config + self.__deferred_context: _DeferredToolsContext = deferred_context @staticmethod # todo add Title to config so that we could use it in stage name @@ -284,6 +290,40 @@ async def _load_tools( ) created_tools.append(mcp_tool) if created_tools: + discovery_cfg = self.__app_config.orchestrator.tool_discovery + deferred_effective = ( + toolset_info.deferred + and discovery_cfg.enabled + and len(created_tools) >= discovery_cfg.min_tools_for_deferral + ) + if deferred_effective: + named_tools: list[tuple[StagedBaseTool, str]] = [ + (t, cast(str, t.openai_function_name())) + for t in created_tools + if t.openai_function_name() is not None + ] + catalog: list[dict[str, str]] = [ + { + "name": name, + "description": cast( + str, + t.tool_config.open_ai_tool.function.description or "", # type: ignore[union-attr] + ), + } + for t, name in named_tools + ] + definitions: dict[str, dict[str, Any]] = { + name: t.tool_config.open_ai_tool.model_dump( # type: ignore[union-attr] + mode="json", exclude_none=True + ) + for t, name in named_tools + } + self.__deferred_context.register_deferred_tools(catalog, definitions) + logger.debug( + "Deferred %d tools from MCP toolset '%s' into DeferredToolsContext", + len(created_tools), + resolve_localized(resolved_toolset.name), + ) self.__mcp_context.extend_tools(created_tools) async def _load_resources( diff --git a/src/quickapp/rest_api_tooling/rest_api_tooling_module.py b/src/quickapp/rest_api_tooling/rest_api_tooling_module.py index 71d553b2..cccfacc3 100644 --- a/src/quickapp/rest_api_tooling/rest_api_tooling_module.py +++ b/src/quickapp/rest_api_tooling/rest_api_tooling_module.py @@ -10,6 +10,7 @@ from quickapp.config.application import ApplicationConfig from quickapp.config.tools.rest_api import RestApiTool from quickapp.config.toolsets.rest_api import RestApiToolSet +from quickapp.tool_discovery._deferred_tools_context import _DeferredToolsContext from ._request_detail_builder import _RequestDetailsBuilder from ._rest_api_stage_wrapper import _RestApiStageWrapper @@ -33,14 +34,44 @@ def __provide_rest_api_tools( app_config: ApplicationConfig, tool_builder: ClassAssistedBuilder[_RestApiTool], accept_language: ACCEPT_LANGUAGE, + deferred_context: _DeferredToolsContext, ) -> list[StagedBaseTool]: result: list[StagedBaseTool] = [] + discovery_cfg = app_config.orchestrator.tool_discovery for toolset_info in app_config.tool_sets: if isinstance(toolset_info, RestApiToolSet) and toolset_info.enabled: toolset_stage_name = resolve_localized(toolset_info.name, accept_language) - result.extend( - self.__create_rest_api_tools(toolset_info, tool_builder, toolset_stage_name) + tools = self.__create_rest_api_tools(toolset_info, tool_builder, toolset_stage_name) + deferred_effective = ( + toolset_info.deferred + and discovery_cfg.enabled + and len(tools) >= discovery_cfg.min_tools_for_deferral ) + if deferred_effective: + from quickapp.config.tools.base import BaseOpenAITool + + catalog = [ + { + "name": t.tool_config.open_ai_tool.function.name, + "description": t.tool_config.open_ai_tool.function.description or "", + } + for t in tools + if isinstance(t.tool_config, BaseOpenAITool) + ] + definitions = { + t.tool_config.open_ai_tool.function.name: t.tool_config.open_ai_tool.model_dump( + mode="json", exclude_none=True + ) + for t in tools + if isinstance(t.tool_config, BaseOpenAITool) + } + deferred_context.register_deferred_tools(catalog, definitions) + logger.debug( + "Deferred %d tools from REST toolset '%s' into DeferredToolsContext", + len(tools), + toolset_stage_name, + ) + result.extend(tools) return result @staticmethod diff --git a/src/quickapp/tool_discovery/__init__.py b/src/quickapp/tool_discovery/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/quickapp/tool_discovery/_anonymous_agent.py b/src/quickapp/tool_discovery/_anonymous_agent.py new file mode 100644 index 00000000..ac5d6859 --- /dev/null +++ b/src/quickapp/tool_discovery/_anonymous_agent.py @@ -0,0 +1,72 @@ +import json +import logging + +from injector import inject + +from quickapp.common import ORCHESTRATOR_AZURE_CLIENT +from quickapp.config.application import ApplicationConfig + +logger = logging.getLogger(__name__) + +_ROUTING_SYSTEM_PROMPT = ( + "You are a tool routing assistant. " + "Given a user query, return the names of the tools from the provided catalog " + "that are most relevant to fulfilling that query. " + "Respond with a JSON array of tool name strings only — no explanation, no markdown. " + "Example: [\"tool_a\", \"tool_b\"]\n\n" + "Catalog:\n{catalog}" +) + + +@inject +class _AnonymousAgent: + """Fires an isolated, non-streaming LLM call to route a query against the deferred tool catalog. + + No conversation history or application system prompt is included — token cost is bounded + by the catalog size alone. + """ + + def __init__( + self, + client: ORCHESTRATOR_AZURE_CLIENT, + config: ApplicationConfig, + ) -> None: + self.__client = client + self.__config = config + + async def route(self, query: str, catalog: list[dict[str, str]]) -> list[str]: + """Return tool names from catalog that best match query.""" + if not catalog: + return [] + + service_model = ( + self.__config.orchestrator.tool_discovery.service_model + or self.__config.orchestrator.deployment.deployment_id + ) + + catalog_text = "\n".join( + f"- {entry['name']}: {entry.get('description', '')}" for entry in catalog + ) + system_content = _ROUTING_SYSTEM_PROMPT.format(catalog=catalog_text) + + try: + response = await self.__client.chat.completions.create( + model=service_model, + messages=[ + {"role": "system", "content": system_content}, + {"role": "user", "content": query}, + ], + stream=False, + ) + except Exception: + logger.exception("Anonymous agent routing call failed for query=%r", query) + return [] + + raw = (response.choices[0].message.content or "").strip() + try: + names = json.loads(raw) + if isinstance(names, list): + return [n for n in names if isinstance(n, str)] + except (json.JSONDecodeError, ValueError): + logger.warning("Anonymous agent returned non-JSON response: %r", raw) + return [] diff --git a/src/quickapp/tool_discovery/_deferred_tools_context.py b/src/quickapp/tool_discovery/_deferred_tools_context.py new file mode 100644 index 00000000..fa5904a9 --- /dev/null +++ b/src/quickapp/tool_discovery/_deferred_tools_context.py @@ -0,0 +1,36 @@ +from injector import inject + +from quickapp.core.agent.models import OpenAiToolConfigDict + + +@inject +class _DeferredToolsContext: + """Request-scoped holder for tool catalog and full definitions of deferred toolsets. + + Populated by toolset initializers (MCP, REST) for toolsets marked deferred=True. + Read by AgentModule to filter schemas from the main LLM payload, and by + _ToolSearchTool to serve the compact catalog and look up full definitions. + """ + + def __init__(self) -> None: + self._catalog: list[dict[str, str]] = [] + self._definitions: dict[str, OpenAiToolConfigDict] = {} + + def register_deferred_tools( + self, + catalog_entries: list[dict[str, str]], + definitions: dict[str, OpenAiToolConfigDict], + ) -> None: + self._catalog.extend(catalog_entries) + self._definitions.update(definitions) + + @property + def deferred_names(self) -> frozenset[str]: + return frozenset(self._definitions.keys()) + + @property + def catalog(self) -> list[dict[str, str]]: + return list(self._catalog) + + def get_definition(self, name: str) -> OpenAiToolConfigDict | None: + return self._definitions.get(name) diff --git a/src/quickapp/tool_discovery/_lazy_loaded_tools_holder.py b/src/quickapp/tool_discovery/_lazy_loaded_tools_holder.py new file mode 100644 index 00000000..048a26bb --- /dev/null +++ b/src/quickapp/tool_discovery/_lazy_loaded_tools_holder.py @@ -0,0 +1,25 @@ +from injector import inject + +from quickapp.core.agent.models import OpenAiToolConfigDict + + +@inject +class _LazyLoadedToolsHolder: + """Request-scoped accumulator for tool schemas discovered via tool_search. + + _ToolSearchTool writes to this holder during execution. + _ChatCompletionConfigBuilder reads from it on every build() call and merges + the accumulated schemas into payload["tools"]. + """ + + def __init__(self) -> None: + self._tools: dict[str, OpenAiToolConfigDict] = {} + + def add(self, tools: list[OpenAiToolConfigDict]) -> None: + for tool in tools: + name: str = tool.get("function", {}).get("name", "") + if name: + self._tools[name] = tool + + def get_all(self) -> list[OpenAiToolConfigDict]: + return list(self._tools.values()) diff --git a/src/quickapp/tool_discovery/_tool_configs.py b/src/quickapp/tool_discovery/_tool_configs.py new file mode 100644 index 00000000..5223778d --- /dev/null +++ b/src/quickapp/tool_discovery/_tool_configs.py @@ -0,0 +1,35 @@ +from quickapp.common.tool_names import INTERNAL_TOOL_SEARCH_TOOL_NAME +from quickapp.config.tools.base import ( + ConfigurableSchemaSimpleType, + JsonTypeEnum, + OpenAiToolConfig, + OpenAiToolFunction, + OpenAiToolFunctionParameters, +) +from quickapp.config.tools.display.tool import ToolDisplayConfig, ToolStageConfig +from quickapp.config.tools.internal import InternalTool + +TOOL_SEARCH_TOOL_CONFIG = InternalTool( + open_ai_tool=OpenAiToolConfig( + function=OpenAiToolFunction( + name=INTERNAL_TOOL_SEARCH_TOOL_NAME, + description=( + "Search for additional tools available to this assistant. " + "Use this when you need a capability that is not listed in the current tool list. " + "Returns the names and descriptions of matching tools; " + "those tools will be available to call immediately after." + ), + parameters=OpenAiToolFunctionParameters( + type=JsonTypeEnum.object, + properties={ + "query": ConfigurableSchemaSimpleType( + type=JsonTypeEnum.string, + description="A natural-language description of the capability you need.", + ) + }, + required=["query"], + ), + ) + ), + display=ToolDisplayConfig(stage=ToolStageConfig(name="Searching tools")), +) diff --git a/src/quickapp/tool_discovery/_tool_discovery_config.py b/src/quickapp/tool_discovery/_tool_discovery_config.py new file mode 100644 index 00000000..512987a3 --- /dev/null +++ b/src/quickapp/tool_discovery/_tool_discovery_config.py @@ -0,0 +1,46 @@ +from pydantic import BaseModel, ConfigDict, Field +from pydantic.fields import FieldInfo +from pydantic_settings import BaseSettings, SettingsConfigDict + + +class ToolDiscoverySettings(BaseSettings): + model_config = SettingsConfigDict() + + min_tools_for_deferral: int = Field( + default=5, + ge=1, + description="Minimum toolset size for deferral to apply deployment-wide.", + alias="MIN_TOOLS_FOR_DEFERRAL", + ) + + +def _min_tools_for_deferral_field() -> FieldInfo: + description = ( + "Minimum number of tools in a toolset for deferral to apply. " + "Toolsets with fewer tools than this threshold are promoted to eager loading " + "even when deferred=true, avoiding discovery overhead for small toolsets. " + "Default: 5 (or the value of MIN_TOOLS_FOR_DEFERRAL env var)" + ) + return Field( # type: ignore[return-value] + default_factory=lambda: ToolDiscoverySettings().min_tools_for_deferral, + json_schema_extra={"default": 5}, + ge=1, + description=description, + ) + + +class ToolDiscoveryConfig(BaseModel): + model_config = ConfigDict(frozen=True) + + enabled: bool = Field( + default=False, + description="Enable dynamic tool discovery. When true, toolsets with deferred=true are withheld from the initial LLM payload and surfaced via the tool_search meta-tool.", + ) + service_model: str | None = Field( + default=None, + description=( + "DIAL deployment used for the anonymous routing call inside tool_search. " + "When omitted, falls back to the orchestrator's own deployment." + ), + ) + min_tools_for_deferral: int = _min_tools_for_deferral_field() # type: ignore[assignment] diff --git a/src/quickapp/tool_discovery/_tool_search_stage_wrapper.py b/src/quickapp/tool_discovery/_tool_search_stage_wrapper.py new file mode 100644 index 00000000..049dde39 --- /dev/null +++ b/src/quickapp/tool_discovery/_tool_search_stage_wrapper.py @@ -0,0 +1,19 @@ +from typing import Any + +from injector import inject + +from quickapp.common import TimedStageWrapper, ToolCallResult + + +@inject +class _ToolSearchStageWrapper(TimedStageWrapper): + + def _get_formatted_parameters(self, parameters: dict[str, Any]) -> str: + query = parameters.get("query", "") + return f"{query}\n\r" if query else "" + + def _build_debug_info_from_exception(self, exception: Exception) -> str: + return f"### Exception:\n\r{exception}\n\r" + + def _build_debug_info_from_result(self, result: ToolCallResult) -> str: + return f"### Discovered tools:\n\r{result.content}\n\r" diff --git a/src/quickapp/tool_discovery/_tool_search_tool.py b/src/quickapp/tool_discovery/_tool_search_tool.py new file mode 100644 index 00000000..1e31a796 --- /dev/null +++ b/src/quickapp/tool_discovery/_tool_search_tool.py @@ -0,0 +1,93 @@ +import json +import logging +from typing import Any + +from injector import AssistedBuilder, inject + +from quickapp.common import StagedBaseTool, ToolCallResult +from quickapp.common.abstract.base_tool_argument_transformer import ToolArgumentTransformer +from quickapp.common.perf_timer.perf_timer import PerformanceTimer +from quickapp.config.application import StageDisplayLevel +from quickapp.config.tools.internal import InternalTool +from quickapp.tool_discovery._anonymous_agent import _AnonymousAgent +from quickapp.tool_discovery._deferred_tools_context import _DeferredToolsContext +from quickapp.tool_discovery._lazy_loaded_tools_holder import _LazyLoadedToolsHolder +from quickapp.tool_discovery._tool_search_stage_wrapper import _ToolSearchStageWrapper + +logger = logging.getLogger(__name__) + + +@inject +class _ToolSearchTool(StagedBaseTool): + """Meta-tool that discovers deferred tools on demand via an anonymous LLM routing call.""" + + def __init__( + self, + stage_wrapper_builder: AssistedBuilder[_ToolSearchStageWrapper], + tool_config: InternalTool, + perf_timer: PerformanceTimer, + deferred_context: _DeferredToolsContext, + lazy_holder: _LazyLoadedToolsHolder, + anonymous_agent: _AnonymousAgent, + stage_display_level: StageDisplayLevel = StageDisplayLevel.INFO, + argument_transformers: list[ToolArgumentTransformer] | None = None, + **kwargs: Any, + ): + super().__init__( + stage_wrapper_builder=stage_wrapper_builder, # type: ignore[arg-type] + tool_config=tool_config, + perf_timer=perf_timer, + stage_display_level=stage_display_level, + argument_transformers=argument_transformers, + **kwargs, + ) + self.__deferred_context = deferred_context + self.__lazy_holder = lazy_holder + self.__anonymous_agent = anonymous_agent + + async def _run_in_stage_async( + self, + stage_wrapper: Any = None, + tool_call_id: str | None = None, + *args: Any, + **kwargs: Any, + ) -> ToolCallResult: + query: str = kwargs.get("query", "") + catalog = self.__deferred_context.catalog + + if not catalog: + result = ToolCallResult( + content="No additional tools are available for discovery.", + content_type="text/plain", + ) + if stage_wrapper: + stage_wrapper.add_result(result) + return result + + matched_names = await self.__anonymous_agent.route(query, catalog) + + discovered: list[dict[str, str]] = [] + new_definitions = [] + for name in matched_names: + definition = self.__deferred_context.get_definition(name) + if definition is None: + logger.warning("tool_search matched unknown tool name %r — skipping", name) + continue + description: str = definition.get("function", {}).get("description", "") + discovered.append({"name": name, "description": description}) + new_definitions.append(definition) + + if new_definitions: + self.__lazy_holder.add(new_definitions) + + if discovered: + content = json.dumps(discovered, ensure_ascii=False) + content_type = "application/json" + else: + content = "No matching tools found for the given query." + content_type = "text/plain" + + result = ToolCallResult(content=content, content_type=content_type) + if stage_wrapper: + stage_wrapper.add_result(result) + return result diff --git a/src/quickapp/tool_discovery/tool_discovery_module.py b/src/quickapp/tool_discovery/tool_discovery_module.py new file mode 100644 index 00000000..0d0c8e5f --- /dev/null +++ b/src/quickapp/tool_discovery/tool_discovery_module.py @@ -0,0 +1,38 @@ +import logging + +from fastapi_injector import request_scope +from injector import AssistedBuilder, Binder, Module, multiprovider + +from quickapp.common import StagedBaseTool +from quickapp.common.preview import preview_module +from quickapp.config.application import ApplicationConfig +from quickapp.tool_discovery._anonymous_agent import _AnonymousAgent +from quickapp.tool_discovery._tool_configs import TOOL_SEARCH_TOOL_CONFIG +from quickapp.tool_discovery._tool_search_stage_wrapper import _ToolSearchStageWrapper +from quickapp.tool_discovery._tool_search_tool import _ToolSearchTool + +logger = logging.getLogger(__name__) + + +@preview_module +class ToolDiscoveryModule(Module): + + def configure(self, binder: Binder) -> None: + binder.bind(_AnonymousAgent, to=_AnonymousAgent, scope=request_scope) + binder.bind(_ToolSearchTool, to=_ToolSearchTool, scope=request_scope) + binder.bind(_ToolSearchStageWrapper, to=_ToolSearchStageWrapper) + + @multiprovider + def _provide_tool_search_tool( + self, + config: ApplicationConfig, + tool_builder: AssistedBuilder[_ToolSearchTool], + ) -> list[StagedBaseTool]: + if not config.orchestrator.tool_discovery.enabled: + return [] + + tool = tool_builder.build( + tool_config=TOOL_SEARCH_TOOL_CONFIG, + ) + logger.debug("ToolDiscoveryModule: tool_search meta-tool registered") + return [tool] diff --git a/src/tests/integration_tests/test_runner/config.py b/src/tests/integration_tests/test_runner/config.py index 31d6fd3f..ce9b8fb7 100644 --- a/src/tests/integration_tests/test_runner/config.py +++ b/src/tests/integration_tests/test_runner/config.py @@ -23,6 +23,9 @@ "integration_simple": ["test_tool_set_chat_hub"], "e2e": ["test_tool_set_chat_hub", "test_tool_set_py_interpreter"], "lazy_admin_context": [], + # Preview: MCP toolset with deferred=true for dynamic tool discovery tests. + # Requires ENABLE_PREVIEW_FEATURES=true and orchestrator.tool_discovery.enabled=true. + "tool_discovery": ["test_mcp_tool_deferred"], } diff --git a/src/tests/unit_tests/agent_tests/test_assistant_invoker.py b/src/tests/unit_tests/agent_tests/test_assistant_invoker.py index afe368af..27e76b3f 100644 --- a/src/tests/unit_tests/agent_tests/test_assistant_invoker.py +++ b/src/tests/unit_tests/agent_tests/test_assistant_invoker.py @@ -10,6 +10,7 @@ from quickapp.core.agent import AssistantInvoker from quickapp.core.agent._chat_completion_config_builder import _ChatCompletionConfigBuilder from quickapp.core.agent._tool_choice_holder import _ToolChoiceHolder +from quickapp.tool_discovery._lazy_loaded_tools_holder import _LazyLoadedToolsHolder def _presentation_settings(show_usage: bool): @@ -69,6 +70,7 @@ def _make_config_builder( pre_invocation_transformers=[mock_filter], presentation_settings=_presentation_settings(show_usage), forwarded_headers=forwarded_headers, + lazy_loaded_tools_holder=_LazyLoadedToolsHolder(), ) diff --git a/src/tests/unit_tests/agent_tests/test_tool_choice_config_builder.py b/src/tests/unit_tests/agent_tests/test_tool_choice_config_builder.py index b15ca310..30a30500 100644 --- a/src/tests/unit_tests/agent_tests/test_tool_choice_config_builder.py +++ b/src/tests/unit_tests/agent_tests/test_tool_choice_config_builder.py @@ -6,6 +6,7 @@ from quickapp.core.agent._chat_completion_config_builder import _ChatCompletionConfigBuilder from quickapp.core.agent._tool_choice_holder import _ToolChoiceHolder +from quickapp.tool_discovery._lazy_loaded_tools_holder import _LazyLoadedToolsHolder def _make_builder( @@ -24,6 +25,7 @@ def _make_builder( pre_invocation_transformers=[], presentation_settings=MagicMock(show_usage_statistics=False), forwarded_headers=None, + lazy_loaded_tools_holder=_LazyLoadedToolsHolder(), ) @@ -70,6 +72,7 @@ def test_tool_choice_consumed_only_on_first_build(self): pre_invocation_transformers=[], presentation_settings=MagicMock(show_usage_statistics=False), forwarded_headers=None, + lazy_loaded_tools_holder=_LazyLoadedToolsHolder(), ) first = builder.build([]) assert first["tool_choice"] == "required" diff --git a/src/tests/unit_tests/mcp_tool_tests/test_mcp_initializer_interactive_login.py b/src/tests/unit_tests/mcp_tool_tests/test_mcp_initializer_interactive_login.py index 95d13b4e..eb18827a 100644 --- a/src/tests/unit_tests/mcp_tool_tests/test_mcp_initializer_interactive_login.py +++ b/src/tests/unit_tests/mcp_tool_tests/test_mcp_initializer_interactive_login.py @@ -128,6 +128,8 @@ def _make_initializer( tool_config_service=MagicMock(), login_service=login_service, accept_language=None, + app_config=MagicMock(), + deferred_context=MagicMock(), ) return initializer, mcp_context, login_service diff --git a/src/tests/unit_tests/mcp_tool_tests/test_mcp_tool_initializer.py b/src/tests/unit_tests/mcp_tool_tests/test_mcp_tool_initializer.py index ce6302f9..daab01b2 100644 --- a/src/tests/unit_tests/mcp_tool_tests/test_mcp_tool_initializer.py +++ b/src/tests/unit_tests/mcp_tool_tests/test_mcp_tool_initializer.py @@ -29,6 +29,13 @@ from tests.unit_tests.common.common import make_provider, noop_timeout_resolver +def _make_app_config_mock() -> MagicMock: + """Return an app_config mock with tool discovery disabled.""" + m = MagicMock() + m.orchestrator.tool_discovery.enabled = False + return m + + def _setup_open_init_session(conn: MagicMock, supports_tools: bool = True) -> MagicMock: """Configure conn.open_init_session to yield (mock_session, mock_init_result).""" session = MagicMock() @@ -227,6 +234,8 @@ def _create(protocol: MCPProtocol, allowed_tools=None, name="test_toolset"): MagicMock(), # tool_config_service MagicMock(), # login_service None, # accept_language + _make_app_config_mock(), # app_config + MagicMock(), # deferred_context ) return initializer, mcp_context @@ -314,6 +323,8 @@ async def test_initialize_multiple_toolsets(tool1, tool2, builder_mock): MagicMock(), # tool_config_service MagicMock(), # login_service None, # accept_language + _make_app_config_mock(), # app_config + MagicMock(), # deferred_context ) await initializer.initialize() @@ -403,6 +414,8 @@ async def test_no_exception_if_toolset_list_is_empty(): MagicMock(), # tool_config_service MagicMock(), # login_service None, # accept_language + _make_app_config_mock(), # app_config + MagicMock(), # deferred_context ) await initializer.initialize() mcp_context.append_tool.assert_not_called() @@ -594,6 +607,8 @@ async def test_initialize_surfaces_session_terminated_through_nested_exception_g MagicMock(), # tool_config_service MagicMock(), # login_service None, # accept_language + _make_app_config_mock(), # app_config + MagicMock(), # deferred_context ) await initializer.initialize() From 448952bdb0a576cdaa0ffb858ff4994657a3fe05 Mon Sep 17 00:00:00 2001 From: Oleksandr Gubarets Date: Thu, 10 Sep 2026 11:15:04 +0300 Subject: [PATCH 04/10] fix code review --- README.md | 58 +++++++++++++ docs/designs/dynamic_tool_discovery.md | 8 +- docs/generated-app-schema.json | 85 ++++++++++++++++--- src/quickapp/config/application.py | 6 +- src/quickapp/config/toolsets/base.py | 11 ++- .../agent/_chat_completion_config_builder.py | 2 +- .../core/agent/_deferred_tools_context.py | 58 +++++++++++++ .../agent}/_lazy_loaded_tools_holder.py | 0 src/quickapp/core/agent/agent_module.py | 4 +- .../mcp_tooling/_mcp_tool_initializer.py | 33 ++----- .../rest_api_tooling_module.py | 29 ++----- src/quickapp/tool_discovery/__init__.py | 3 + .../tool_discovery/_anonymous_agent.py | 9 +- .../tool_discovery/_deferred_tools_context.py | 36 -------- .../tool_discovery/_tool_discovery_config.py | 4 +- .../_tool_search_stage_wrapper.py | 6 +- .../tool_discovery/_tool_search_tool.py | 4 +- .../tool_discovery/tool_discovery_module.py | 2 +- .../test_runner/test_mcp_tool_deferred.json | 11 +++ .../agent_tests/test_assistant_invoker.py | 2 +- .../test_tool_choice_config_builder.py | 2 +- 21 files changed, 248 insertions(+), 125 deletions(-) create mode 100644 src/quickapp/core/agent/_deferred_tools_context.py rename src/quickapp/{tool_discovery => core/agent}/_lazy_loaded_tools_holder.py (100%) delete mode 100644 src/quickapp/tool_discovery/_deferred_tools_context.py create mode 100644 src/tests/integration_tests/test_runner/test_mcp_tool_deferred.json diff --git a/README.md b/README.md index 98bb4d86..2460b80c 100644 --- a/README.md +++ b/README.md @@ -95,6 +95,64 @@ Key fields: See [Config-Driven Hooks design doc](docs/designs/config_driven_hooks.md) for the full reference. +### Dynamic Tool Discovery `[Preview]` + +Dynamic tool discovery defers large toolsets from the initial LLM payload and surfaces them +on demand via a `tool_search` meta-tool. The orchestrator calls `tool_search` with a natural-language +query when it needs a tool it hasn't seen yet; a lightweight anonymous LLM routing call selects +the relevant tool schemas and injects them into the next iteration. + +Enable with `ENABLE_PREVIEW_FEATURES=true`, then add `orchestrator.tool_discovery` to the app manifest: + +```json +{ + "orchestrator": { + "deployment": { "deployment_id": "gpt-4o" }, + "tool_discovery": { + "enabled": true, + "service_model": "gpt-4o-mini", + "min_tools_for_deferral": 5 + } + }, + "tool_sets": [ + { + "name": "my-mcp-server", + "type": "mcp", + "mcp_server_info": { + "url": "http://localhost:8003/mcp", + "protocol": "streamable_http" + } + } + ] +} +``` + +Toolsets are deferred by default — omitting `deferred` or setting it to `true` both defer the +toolset. To keep a specific toolset always eager, set `"deferred": false` on that toolset: + +```json +{ + "name": "always-eager-toolset", + "type": "rest_api", + "deferred": false, + "open_api": { "url": "https://api.example.com/openapi.json" } +} +``` + +Key fields: + +| Field | Default | Description | +|------------------------------------------------|---------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `orchestrator.tool_discovery.enabled` | `false` | Activates dynamic discovery for this app. Must be `true` for deferral to take effect. | +| `orchestrator.tool_discovery.service_model` | — | DIAL deployment used for the anonymous routing call inside `tool_search`. Falls back to the orchestrator's own deployment when omitted. | +| `orchestrator.tool_discovery.min_tools_for_deferral` | `5` | Minimum number of tools in a toolset for deferral to apply. Toolsets smaller than this threshold are promoted to eager loading even when `deferred: true`. | +| `.deferred` | `true` | Per-toolset opt-out. Set to `false` to force a specific toolset into the initial payload regardless of `tool_discovery.enabled`. | + +The `MIN_TOOLS_FOR_DEFERRAL` environment variable sets the deployment-wide default for +`min_tools_for_deferral`; individual apps can override it in their manifest. + +See [Dynamic Tool Discovery design doc](docs/designs/dynamic_tool_discovery.md) for the full reference. + ### Forwarding headers Incoming request headers whose names start with `X-` (case-insensitive) are automatically forwarded to all outbound diff --git a/docs/designs/dynamic_tool_discovery.md b/docs/designs/dynamic_tool_discovery.md index 28e4d631..3edd81f3 100644 --- a/docs/designs/dynamic_tool_discovery.md +++ b/docs/designs/dynamic_tool_discovery.md @@ -117,7 +117,7 @@ JSON for any tool on demand. **Round-trip cost:** +1 before first use of any previously-unseen tool. -**Config:** opt-in flag at `ApplicationConfig` (global) or per-toolset. +**Config:** opt-out flag per-toolset (`deferred: false` to disable for a specific toolset). **Pros:** - Model always has full name-space visibility (all names + descriptions visible). @@ -433,7 +433,7 @@ deferred_effective = toolset.deferred and len(tools) >= discovery.min_tools_for_ } ``` -- `deferred: true` opts the toolset into `DeferredRequestContext`. Default: `false`. +- `deferred: true` (or unset/`null`) opts the toolset into `DeferredRequestContext`. Default: `true` (omitting the field defers by default). - `service_model` names the DIAL deployment used for the `AnonymousAgent` chat completion. When omitted, falls back to the orchestrator's own deployment. - `min_tools_for_deferral` is the tool-count guard below which a deferred toolset is silently @@ -472,8 +472,8 @@ with ~20-token descriptions each, this is ~4 K tokens regardless of how long the not conversation length. - Native schema injection means the main LLM calls discovered tools with proper structured arguments from the iteration after discovery. -- Non-breaking opt-in: `deferred: false` by default; threshold guard prevents regression for - small toolsets. +- Opt-out: `deferred: true` by default; threshold guard prevents regression for small toolsets. + Set `deferred: false` on a toolset to keep it always-eager. - `AnonymousAgent` is a reusable module independent of tool discovery. - Search strategy is swappable (keyword, embedding, different model) without touching the orchestrator or injection logic. diff --git a/docs/generated-app-schema.json b/docs/generated-app-schema.json index 4d0d0428..c6fabc7a 100644 --- a/docs/generated-app-schema.json +++ b/docs/generated-app-schema.json @@ -685,10 +685,18 @@ "type": "boolean" }, "deferred": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], "default": true, - "description": "When true, this toolset's tool schemas are withheld from the initial LLM payload. Requires orchestrator.tool_discovery.enabled=true. Tools are discovered on demand via the tool_search meta-tool.", + "description": "When true or unset, this toolset's tool schemas are withheld from the initial LLM payload. Requires orchestrator.tool_discovery.enabled=true. Tools are discovered on demand via the tool_search meta-tool. Set to false to keep this toolset always eager.", "title": "Deferred", - "type": "boolean" + "x-preview": true }, "type": { "const": "dial-deployment", @@ -766,10 +774,18 @@ "type": "boolean" }, "deferred": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], "default": true, - "description": "When true, this toolset's tool schemas are withheld from the initial LLM payload. Requires orchestrator.tool_discovery.enabled=true. Tools are discovered on demand via the tool_search meta-tool.", + "description": "When true or unset, this toolset's tool schemas are withheld from the initial LLM payload. Requires orchestrator.tool_discovery.enabled=true. Tools are discovered on demand via the tool_search meta-tool. Set to false to keep this toolset always eager.", "title": "Deferred", - "type": "boolean" + "x-preview": true }, "type": { "const": "dial-app", @@ -1252,10 +1268,18 @@ "type": "boolean" }, "deferred": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], "default": true, - "description": "When true, this toolset's tool schemas are withheld from the initial LLM payload. Requires orchestrator.tool_discovery.enabled=true. Tools are discovered on demand via the tool_search meta-tool.", + "description": "When true or unset, this toolset's tool schemas are withheld from the initial LLM payload. Requires orchestrator.tool_discovery.enabled=true. Tools are discovered on demand via the tool_search meta-tool. Set to false to keep this toolset always eager.", "title": "Deferred", - "type": "boolean" + "x-preview": true }, "type": { "const": "dial-mcp", @@ -1840,10 +1864,18 @@ "type": "boolean" }, "deferred": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], "default": true, - "description": "When true, this toolset's tool schemas are withheld from the initial LLM payload. Requires orchestrator.tool_discovery.enabled=true. Tools are discovered on demand via the tool_search meta-tool.", + "description": "When true or unset, this toolset's tool schemas are withheld from the initial LLM payload. Requires orchestrator.tool_discovery.enabled=true. Tools are discovered on demand via the tool_search meta-tool. Set to false to keep this toolset always eager.", "title": "Deferred", - "type": "boolean" + "x-preview": true }, "type": { "const": "internal", @@ -2240,10 +2272,18 @@ "type": "boolean" }, "deferred": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], "default": true, - "description": "When true, this toolset's tool schemas are withheld from the initial LLM payload. Requires orchestrator.tool_discovery.enabled=true. Tools are discovered on demand via the tool_search meta-tool.", + "description": "When true or unset, this toolset's tool schemas are withheld from the initial LLM payload. Requires orchestrator.tool_discovery.enabled=true. Tools are discovered on demand via the tool_search meta-tool. Set to false to keep this toolset always eager.", "title": "Deferred", - "type": "boolean" + "x-preview": true }, "type": { "const": "mcp", @@ -3094,10 +3134,18 @@ "type": "boolean" }, "deferred": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], "default": true, - "description": "When true, this toolset's tool schemas are withheld from the initial LLM payload. Requires orchestrator.tool_discovery.enabled=true. Tools are discovered on demand via the tool_search meta-tool.", + "description": "When true or unset, this toolset's tool schemas are withheld from the initial LLM payload. Requires orchestrator.tool_discovery.enabled=true. Tools are discovered on demand via the tool_search meta-tool. Set to false to keep this toolset always eager.", "title": "Deferred", - "type": "boolean" + "x-preview": true }, "type": { "const": "rest-api", @@ -3681,8 +3729,17 @@ "description": "How the orchestrator receives request-scoped attachments. When unset, the orchestrator gets no admin/user attachments on the native path (legacy behaviour: USER `image/*` passes through, other MIMEs are surfaced as XML metadata only)." }, "tool_discovery": { - "$ref": "#/$defs/ToolDiscoveryConfig", - "description": "Dynamic tool discovery configuration. When enabled, toolsets with deferred=true are withheld from the initial LLM payload and discovered on demand via the tool_search meta-tool." + "anyOf": [ + { + "$ref": "#/$defs/ToolDiscoveryConfig" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Dynamic tool discovery configuration. When enabled, toolsets with deferred=true are withheld from the initial LLM payload and discovered on demand via the tool_search meta-tool.", + "x-preview": true } }, "required": [ diff --git a/src/quickapp/config/application.py b/src/quickapp/config/application.py index a24521f1..519d80d5 100644 --- a/src/quickapp/config/application.py +++ b/src/quickapp/config/application.py @@ -24,7 +24,7 @@ from quickapp.config.timestamp import TimestampConfig, ToolCallTimestampConfig from quickapp.config.toolsets.toolset import ToolSet from quickapp.config.web_fetch import WebFetchConfig -from quickapp.tool_discovery._tool_discovery_config import ToolDiscoveryConfig +from quickapp.tool_discovery import ToolDiscoveryConfig logger = logging.getLogger(__name__) @@ -96,8 +96,8 @@ class OrchestratorConfig(BaseModel): "MIMEs are surfaced as XML metadata only)." ), ) - tool_discovery: ToolDiscoveryConfig = Field( - default_factory=ToolDiscoveryConfig, + tool_discovery: ToolDiscoveryConfig | None = PreviewField( # type: ignore[assignment] + default=None, description="Dynamic tool discovery configuration. When enabled, toolsets with deferred=true are withheld from the initial LLM payload and discovered on demand via the tool_search meta-tool.", ) diff --git a/src/quickapp/config/toolsets/base.py b/src/quickapp/config/toolsets/base.py index 832af3cb..9cbf74f5 100644 --- a/src/quickapp/config/toolsets/base.py +++ b/src/quickapp/config/toolsets/base.py @@ -1,5 +1,6 @@ from pydantic import BaseModel, Field +from quickapp.common.base_config import PreviewField from quickapp.common.localized_string import LocalizedString @@ -17,11 +18,13 @@ class BaseToolSet(BaseModel): default=None, description="The description of the tool set." ) enabled: bool = Field(default=True, description="Whether the toolset is enabled.") - deferred: bool = Field( - default=True, + deferred: bool | None = PreviewField( # type: ignore[assignment] + default=None, + json_schema_extra={"default": True}, description=( - "When true, this toolset's tool schemas are withheld from the initial LLM payload. " + "When true or unset, this toolset's tool schemas are withheld from the initial LLM payload. " "Requires orchestrator.tool_discovery.enabled=true. " - "Tools are discovered on demand via the tool_search meta-tool." + "Tools are discovered on demand via the tool_search meta-tool. " + "Set to false to keep this toolset always eager." ), ) diff --git a/src/quickapp/core/agent/_chat_completion_config_builder.py b/src/quickapp/core/agent/_chat_completion_config_builder.py index 7a0ac2b1..a5ee78a9 100644 --- a/src/quickapp/core/agent/_chat_completion_config_builder.py +++ b/src/quickapp/core/agent/_chat_completion_config_builder.py @@ -11,9 +11,9 @@ from quickapp.common.payload_logging import log_payload, payloads_enabled, summarize_roles from quickapp.common.presentation_settings import PresentationSettings from quickapp.config.application import ApplicationConfig +from quickapp.core.agent._lazy_loaded_tools_holder import _LazyLoadedToolsHolder from quickapp.core.agent._tool_choice_holder import _ToolChoiceHolder from quickapp.core.agent.models import STATE_KEY_ORCHESTRATOR, OpenAiToolConfigDict -from quickapp.tool_discovery._lazy_loaded_tools_holder import _LazyLoadedToolsHolder logger = logging.getLogger(__name__) diff --git a/src/quickapp/core/agent/_deferred_tools_context.py b/src/quickapp/core/agent/_deferred_tools_context.py new file mode 100644 index 00000000..b8b489ca --- /dev/null +++ b/src/quickapp/core/agent/_deferred_tools_context.py @@ -0,0 +1,58 @@ +from injector import inject + +from quickapp.common import StagedBaseTool +from quickapp.config.tools.base import BaseOpenAITool +from quickapp.core.agent.models import OpenAiToolConfigDict + + +@inject +class _DeferredToolsContext: + """Request-scoped holder for tool catalog and full definitions of deferred toolsets.""" + + def __init__(self) -> None: + self._catalog: list[dict[str, str]] = [] + self._definitions: dict[str, OpenAiToolConfigDict] = {} + + def register_deferred_tools( + self, + catalog_entries: list[dict[str, str]], + definitions: dict[str, OpenAiToolConfigDict], + ) -> None: + self._catalog.extend(catalog_entries) + self._definitions.update(definitions) + + @property + def deferred_names(self) -> frozenset[str]: + return frozenset(self._definitions.keys()) + + @property + def catalog(self) -> list[dict[str, str]]: + return list(self._catalog) + + def get_definition(self, name: str) -> OpenAiToolConfigDict | None: + return self._definitions.get(name) + + +def register_tools_as_deferred( + tools: list[StagedBaseTool], + context: _DeferredToolsContext, +) -> None: + """Build catalog + definitions from tools and register them as deferred.""" + entries: list[tuple[StagedBaseTool, str]] = [ + (t, name) + for t in tools + if isinstance(t.tool_config, BaseOpenAITool) + and (name := t.tool_config.open_ai_tool.function.name) + ] + catalog: list[dict[str, str]] = [ + { + "name": name, + "description": t.tool_config.open_ai_tool.function.description or "", # type: ignore[union-attr] + } + for t, name in entries + ] + definitions: dict[str, OpenAiToolConfigDict] = { + name: t.tool_config.open_ai_tool.model_dump(mode="json", exclude_none=True) # type: ignore[union-attr] + for t, name in entries + } + context.register_deferred_tools(catalog, definitions) diff --git a/src/quickapp/tool_discovery/_lazy_loaded_tools_holder.py b/src/quickapp/core/agent/_lazy_loaded_tools_holder.py similarity index 100% rename from src/quickapp/tool_discovery/_lazy_loaded_tools_holder.py rename to src/quickapp/core/agent/_lazy_loaded_tools_holder.py diff --git a/src/quickapp/core/agent/agent_module.py b/src/quickapp/core/agent/agent_module.py index 9e9d25cc..17d32758 100644 --- a/src/quickapp/core/agent/agent_module.py +++ b/src/quickapp/core/agent/agent_module.py @@ -47,6 +47,8 @@ ) from quickapp.core.agent._attachment_filter import _AttachmentFilter from quickapp.core.agent._chat_completion_config_builder import _ChatCompletionConfigBuilder +from quickapp.core.agent._deferred_tools_context import _DeferredToolsContext +from quickapp.core.agent._lazy_loaded_tools_holder import _LazyLoadedToolsHolder from quickapp.core.agent._messages_transformers import _AddSystemPromptTransformer from quickapp.core.agent._orchestrator_deployment_initializer import ( _OrchestratorDeploymentInitializer, @@ -62,8 +64,6 @@ OrchestratorDeploymentCacheService, ) from quickapp.core.application._request_context import _RequestContext -from quickapp.tool_discovery._deferred_tools_context import _DeferredToolsContext -from quickapp.tool_discovery._lazy_loaded_tools_holder import _LazyLoadedToolsHolder DEFAULT_QUERY_PARAM = ConfigurableSchemaSimpleType( type=JsonTypeEnum.string, diff --git a/src/quickapp/mcp_tooling/_mcp_tool_initializer.py b/src/quickapp/mcp_tooling/_mcp_tool_initializer.py index 6bdf9499..a7c8d2ff 100644 --- a/src/quickapp/mcp_tooling/_mcp_tool_initializer.py +++ b/src/quickapp/mcp_tooling/_mcp_tool_initializer.py @@ -1,6 +1,6 @@ import asyncio import logging -from typing import Any, cast +from typing import Any from urllib.parse import unquote import httpx @@ -27,13 +27,16 @@ from quickapp.config.toolsets.authorization import MCPApiKeyAuthorization from quickapp.config.toolsets.dial_mcp import DialMCPToolSet from quickapp.config.toolsets.mcp import MCPProtocol, MCPServerInfo, MCPToolSet +from quickapp.core.agent._deferred_tools_context import ( + _DeferredToolsContext, + register_tools_as_deferred, +) from quickapp.dial_core_services._interactive_login_service import InteractiveLoginService from quickapp.dial_core_services._login_result import LoginResult from quickapp.dial_core_services.tool_config_service import ToolConfigCoreService from quickapp.mcp_tooling._mcp_eager_resource import MCPEagerTextResource from quickapp.mcp_tooling._mcp_resource_meta import MCPResourceMeta from quickapp.mcp_tooling._mcp_server_capabilities import MCPServerCapabilities -from quickapp.tool_discovery._deferred_tools_context import _DeferredToolsContext from ._di_types import DialToolsetCacheService from ._mcp_tool import _MCPTool @@ -292,33 +295,13 @@ async def _load_tools( if created_tools: discovery_cfg = self.__app_config.orchestrator.tool_discovery deferred_effective = ( - toolset_info.deferred + toolset_info.deferred is not False + and discovery_cfg is not None and discovery_cfg.enabled and len(created_tools) >= discovery_cfg.min_tools_for_deferral ) if deferred_effective: - named_tools: list[tuple[StagedBaseTool, str]] = [ - (t, cast(str, t.openai_function_name())) - for t in created_tools - if t.openai_function_name() is not None - ] - catalog: list[dict[str, str]] = [ - { - "name": name, - "description": cast( - str, - t.tool_config.open_ai_tool.function.description or "", # type: ignore[union-attr] - ), - } - for t, name in named_tools - ] - definitions: dict[str, dict[str, Any]] = { - name: t.tool_config.open_ai_tool.model_dump( # type: ignore[union-attr] - mode="json", exclude_none=True - ) - for t, name in named_tools - } - self.__deferred_context.register_deferred_tools(catalog, definitions) + register_tools_as_deferred(created_tools, self.__deferred_context) logger.debug( "Deferred %d tools from MCP toolset '%s' into DeferredToolsContext", len(created_tools), diff --git a/src/quickapp/rest_api_tooling/rest_api_tooling_module.py b/src/quickapp/rest_api_tooling/rest_api_tooling_module.py index cccfacc3..10e13c74 100644 --- a/src/quickapp/rest_api_tooling/rest_api_tooling_module.py +++ b/src/quickapp/rest_api_tooling/rest_api_tooling_module.py @@ -10,7 +10,10 @@ from quickapp.config.application import ApplicationConfig from quickapp.config.tools.rest_api import RestApiTool from quickapp.config.toolsets.rest_api import RestApiToolSet -from quickapp.tool_discovery._deferred_tools_context import _DeferredToolsContext +from quickapp.core.agent._deferred_tools_context import ( + _DeferredToolsContext, + register_tools_as_deferred, +) from ._request_detail_builder import _RequestDetailsBuilder from ._rest_api_stage_wrapper import _RestApiStageWrapper @@ -37,35 +40,19 @@ def __provide_rest_api_tools( deferred_context: _DeferredToolsContext, ) -> list[StagedBaseTool]: result: list[StagedBaseTool] = [] - discovery_cfg = app_config.orchestrator.tool_discovery for toolset_info in app_config.tool_sets: if isinstance(toolset_info, RestApiToolSet) and toolset_info.enabled: toolset_stage_name = resolve_localized(toolset_info.name, accept_language) tools = self.__create_rest_api_tools(toolset_info, tool_builder, toolset_stage_name) + discovery_cfg = app_config.orchestrator.tool_discovery deferred_effective = ( - toolset_info.deferred + toolset_info.deferred is not False + and discovery_cfg is not None and discovery_cfg.enabled and len(tools) >= discovery_cfg.min_tools_for_deferral ) if deferred_effective: - from quickapp.config.tools.base import BaseOpenAITool - - catalog = [ - { - "name": t.tool_config.open_ai_tool.function.name, - "description": t.tool_config.open_ai_tool.function.description or "", - } - for t in tools - if isinstance(t.tool_config, BaseOpenAITool) - ] - definitions = { - t.tool_config.open_ai_tool.function.name: t.tool_config.open_ai_tool.model_dump( - mode="json", exclude_none=True - ) - for t in tools - if isinstance(t.tool_config, BaseOpenAITool) - } - deferred_context.register_deferred_tools(catalog, definitions) + register_tools_as_deferred(tools, deferred_context) logger.debug( "Deferred %d tools from REST toolset '%s' into DeferredToolsContext", len(tools), diff --git a/src/quickapp/tool_discovery/__init__.py b/src/quickapp/tool_discovery/__init__.py index e69de29b..02859f96 100644 --- a/src/quickapp/tool_discovery/__init__.py +++ b/src/quickapp/tool_discovery/__init__.py @@ -0,0 +1,3 @@ +from quickapp.tool_discovery._tool_discovery_config import ToolDiscoveryConfig + +__all__ = ["ToolDiscoveryConfig"] diff --git a/src/quickapp/tool_discovery/_anonymous_agent.py b/src/quickapp/tool_discovery/_anonymous_agent.py index ac5d6859..a27fe8a1 100644 --- a/src/quickapp/tool_discovery/_anonymous_agent.py +++ b/src/quickapp/tool_discovery/_anonymous_agent.py @@ -1,6 +1,7 @@ import json import logging +import openai from injector import inject from quickapp.common import ORCHESTRATOR_AZURE_CLIENT @@ -39,10 +40,10 @@ async def route(self, query: str, catalog: list[dict[str, str]]) -> list[str]: if not catalog: return [] + discovery = self.__config.orchestrator.tool_discovery service_model = ( - self.__config.orchestrator.tool_discovery.service_model - or self.__config.orchestrator.deployment.deployment_id - ) + discovery.service_model if discovery is not None else None + ) or self.__config.orchestrator.deployment.deployment_id catalog_text = "\n".join( f"- {entry['name']}: {entry.get('description', '')}" for entry in catalog @@ -58,7 +59,7 @@ async def route(self, query: str, catalog: list[dict[str, str]]) -> list[str]: ], stream=False, ) - except Exception: + except openai.OpenAIError: logger.exception("Anonymous agent routing call failed for query=%r", query) return [] diff --git a/src/quickapp/tool_discovery/_deferred_tools_context.py b/src/quickapp/tool_discovery/_deferred_tools_context.py deleted file mode 100644 index fa5904a9..00000000 --- a/src/quickapp/tool_discovery/_deferred_tools_context.py +++ /dev/null @@ -1,36 +0,0 @@ -from injector import inject - -from quickapp.core.agent.models import OpenAiToolConfigDict - - -@inject -class _DeferredToolsContext: - """Request-scoped holder for tool catalog and full definitions of deferred toolsets. - - Populated by toolset initializers (MCP, REST) for toolsets marked deferred=True. - Read by AgentModule to filter schemas from the main LLM payload, and by - _ToolSearchTool to serve the compact catalog and look up full definitions. - """ - - def __init__(self) -> None: - self._catalog: list[dict[str, str]] = [] - self._definitions: dict[str, OpenAiToolConfigDict] = {} - - def register_deferred_tools( - self, - catalog_entries: list[dict[str, str]], - definitions: dict[str, OpenAiToolConfigDict], - ) -> None: - self._catalog.extend(catalog_entries) - self._definitions.update(definitions) - - @property - def deferred_names(self) -> frozenset[str]: - return frozenset(self._definitions.keys()) - - @property - def catalog(self) -> list[dict[str, str]]: - return list(self._catalog) - - def get_definition(self, name: str) -> OpenAiToolConfigDict | None: - return self._definitions.get(name) diff --git a/src/quickapp/tool_discovery/_tool_discovery_config.py b/src/quickapp/tool_discovery/_tool_discovery_config.py index 512987a3..df43f208 100644 --- a/src/quickapp/tool_discovery/_tool_discovery_config.py +++ b/src/quickapp/tool_discovery/_tool_discovery_config.py @@ -1,11 +1,9 @@ from pydantic import BaseModel, ConfigDict, Field from pydantic.fields import FieldInfo -from pydantic_settings import BaseSettings, SettingsConfigDict +from pydantic_settings import BaseSettings class ToolDiscoverySettings(BaseSettings): - model_config = SettingsConfigDict() - min_tools_for_deferral: int = Field( default=5, ge=1, diff --git a/src/quickapp/tool_discovery/_tool_search_stage_wrapper.py b/src/quickapp/tool_discovery/_tool_search_stage_wrapper.py index 049dde39..803079dc 100644 --- a/src/quickapp/tool_discovery/_tool_search_stage_wrapper.py +++ b/src/quickapp/tool_discovery/_tool_search_stage_wrapper.py @@ -10,10 +10,10 @@ class _ToolSearchStageWrapper(TimedStageWrapper): def _get_formatted_parameters(self, parameters: dict[str, Any]) -> str: query = parameters.get("query", "") - return f"{query}\n\r" if query else "" + return f"> ##### Query:\n{query}\n" if query else "" def _build_debug_info_from_exception(self, exception: Exception) -> str: - return f"### Exception:\n\r{exception}\n\r" + return f"> ##### Exception:\n{exception}\n" def _build_debug_info_from_result(self, result: ToolCallResult) -> str: - return f"### Discovered tools:\n\r{result.content}\n\r" + return f"> ##### Discovered tools:\n{result.content}\n" diff --git a/src/quickapp/tool_discovery/_tool_search_tool.py b/src/quickapp/tool_discovery/_tool_search_tool.py index 1e31a796..3f830fe5 100644 --- a/src/quickapp/tool_discovery/_tool_search_tool.py +++ b/src/quickapp/tool_discovery/_tool_search_tool.py @@ -9,9 +9,9 @@ from quickapp.common.perf_timer.perf_timer import PerformanceTimer from quickapp.config.application import StageDisplayLevel from quickapp.config.tools.internal import InternalTool +from quickapp.core.agent._deferred_tools_context import _DeferredToolsContext +from quickapp.core.agent._lazy_loaded_tools_holder import _LazyLoadedToolsHolder from quickapp.tool_discovery._anonymous_agent import _AnonymousAgent -from quickapp.tool_discovery._deferred_tools_context import _DeferredToolsContext -from quickapp.tool_discovery._lazy_loaded_tools_holder import _LazyLoadedToolsHolder from quickapp.tool_discovery._tool_search_stage_wrapper import _ToolSearchStageWrapper logger = logging.getLogger(__name__) diff --git a/src/quickapp/tool_discovery/tool_discovery_module.py b/src/quickapp/tool_discovery/tool_discovery_module.py index 0d0c8e5f..b2df8206 100644 --- a/src/quickapp/tool_discovery/tool_discovery_module.py +++ b/src/quickapp/tool_discovery/tool_discovery_module.py @@ -28,7 +28,7 @@ def _provide_tool_search_tool( config: ApplicationConfig, tool_builder: AssistedBuilder[_ToolSearchTool], ) -> list[StagedBaseTool]: - if not config.orchestrator.tool_discovery.enabled: + if not config.orchestrator.tool_discovery or not config.orchestrator.tool_discovery.enabled: return [] tool = tool_builder.build( diff --git a/src/tests/integration_tests/test_runner/test_mcp_tool_deferred.json b/src/tests/integration_tests/test_runner/test_mcp_tool_deferred.json new file mode 100644 index 00000000..0fc125eb --- /dev/null +++ b/src/tests/integration_tests/test_runner/test_mcp_tool_deferred.json @@ -0,0 +1,11 @@ +{ + "name": "mcp-local-toolset", + "description": "Set with MCP tools (deferred)", + "type": "mcp", + "deferred": true, + "mcp_server_info": { + "url": "http://localhost:8003/mcp", + "protocol": "streamable_http", + "authorization": null + } +} diff --git a/src/tests/unit_tests/agent_tests/test_assistant_invoker.py b/src/tests/unit_tests/agent_tests/test_assistant_invoker.py index 27e76b3f..0a75f784 100644 --- a/src/tests/unit_tests/agent_tests/test_assistant_invoker.py +++ b/src/tests/unit_tests/agent_tests/test_assistant_invoker.py @@ -9,8 +9,8 @@ from quickapp.common.stage_close_registry import DeferredStageCloseRegistry from quickapp.core.agent import AssistantInvoker from quickapp.core.agent._chat_completion_config_builder import _ChatCompletionConfigBuilder +from quickapp.core.agent._lazy_loaded_tools_holder import _LazyLoadedToolsHolder from quickapp.core.agent._tool_choice_holder import _ToolChoiceHolder -from quickapp.tool_discovery._lazy_loaded_tools_holder import _LazyLoadedToolsHolder def _presentation_settings(show_usage: bool): diff --git a/src/tests/unit_tests/agent_tests/test_tool_choice_config_builder.py b/src/tests/unit_tests/agent_tests/test_tool_choice_config_builder.py index 30a30500..2b903559 100644 --- a/src/tests/unit_tests/agent_tests/test_tool_choice_config_builder.py +++ b/src/tests/unit_tests/agent_tests/test_tool_choice_config_builder.py @@ -5,8 +5,8 @@ from aidial_sdk.exceptions import InvalidRequestError from quickapp.core.agent._chat_completion_config_builder import _ChatCompletionConfigBuilder +from quickapp.core.agent._lazy_loaded_tools_holder import _LazyLoadedToolsHolder from quickapp.core.agent._tool_choice_holder import _ToolChoiceHolder -from quickapp.tool_discovery._lazy_loaded_tools_holder import _LazyLoadedToolsHolder def _make_builder( From 46cd5179a19136b5c29d172478477ee8029746f5 Mon Sep 17 00:00:00 2001 From: Oleksandr Gubarets Date: Thu, 10 Sep 2026 11:34:25 +0300 Subject: [PATCH 05/10] fix code review --- docker-compose.yml | 3 +- .../agent/_chat_completion_config_builder.py | 4 +- .../core/agent/_deferred_tools_context.py | 58 ------------------ src/quickapp/core/agent/agent_module.py | 10 +-- ..._holder.py => lazy_loaded_tools_holder.py} | 2 +- .../mcp_tooling/_mcp_tool_initializer.py | 33 +++++----- .../rest_api_tooling_module.py | 29 ++++----- .../tool_discovery/_anonymous_agent.py | 9 +-- .../tool_discovery/_deferred_tools_context.py | 61 +++++++++++++++++++ .../tool_discovery/_tool_search_tool.py | 11 ++-- .../agent_tests/test_assistant_invoker.py | 4 +- .../test_tool_choice_config_builder.py | 6 +- 12 files changed, 112 insertions(+), 118 deletions(-) delete mode 100644 src/quickapp/core/agent/_deferred_tools_context.py rename src/quickapp/core/agent/{_lazy_loaded_tools_holder.py => lazy_loaded_tools_holder.py} (96%) create mode 100644 src/quickapp/tool_discovery/_deferred_tools_context.py diff --git a/docker-compose.yml b/docker-compose.yml index e2d4a7a9..fbef0ee0 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -71,7 +71,6 @@ services: - core - keycloak environment: - NEXTAUTH_SECRET: "secret" AUTH_SESSION_SECRET: "48c69bd559cffffb2cbee3951430f324e0cd0c79fd8a83de0a991648cae706c8" AUTH_CALLBACK_BASE_URL: "http://localhost:3012" AUTH_POST_LOGOUT_REDIRECT_URI: "http://localhost:3012" @@ -82,7 +81,7 @@ services: THEMES_CONFIG_URL: "http://themes:8080" AUTH_KEYCLOAK_CLIENT_ID: "dial-local" AUTH_KEYCLOAK_SECRET: "dial-local-dev-secret" - AUTH_KEYCLOAK_HOST: "https://keycloak.localtest.me:8443/realms/dial-dev" + AUTH_KEYCLOAK_HOST: "keycloak.localtest.me:8443/realms/dial-dev" AUTH_KEYCLOAK_ADMIN_ROLE_NAMES: "admin" AUTH_KEYCLOAK_DIAL_ROLES_FIELD: "realm_access.roles" NODE_EXTRA_CA_CERTS: "/certs/ca.crt" diff --git a/src/quickapp/core/agent/_chat_completion_config_builder.py b/src/quickapp/core/agent/_chat_completion_config_builder.py index a5ee78a9..19d9e35b 100644 --- a/src/quickapp/core/agent/_chat_completion_config_builder.py +++ b/src/quickapp/core/agent/_chat_completion_config_builder.py @@ -11,8 +11,8 @@ from quickapp.common.payload_logging import log_payload, payloads_enabled, summarize_roles from quickapp.common.presentation_settings import PresentationSettings from quickapp.config.application import ApplicationConfig -from quickapp.core.agent._lazy_loaded_tools_holder import _LazyLoadedToolsHolder from quickapp.core.agent._tool_choice_holder import _ToolChoiceHolder +from quickapp.core.agent.lazy_loaded_tools_holder import LazyLoadedToolsHolder from quickapp.core.agent.models import STATE_KEY_ORCHESTRATOR, OpenAiToolConfigDict logger = logging.getLogger(__name__) @@ -29,7 +29,7 @@ def __init__( pre_invocation_transformers: list[PreInvocationTransformer], presentation_settings: PresentationSettings, forwarded_headers: ForwardedHeaders, - lazy_loaded_tools_holder: _LazyLoadedToolsHolder, + lazy_loaded_tools_holder: LazyLoadedToolsHolder, ) -> None: self.__config: ApplicationConfig = config self.__tools: list[OpenAiToolConfigDict] = tools diff --git a/src/quickapp/core/agent/_deferred_tools_context.py b/src/quickapp/core/agent/_deferred_tools_context.py deleted file mode 100644 index b8b489ca..00000000 --- a/src/quickapp/core/agent/_deferred_tools_context.py +++ /dev/null @@ -1,58 +0,0 @@ -from injector import inject - -from quickapp.common import StagedBaseTool -from quickapp.config.tools.base import BaseOpenAITool -from quickapp.core.agent.models import OpenAiToolConfigDict - - -@inject -class _DeferredToolsContext: - """Request-scoped holder for tool catalog and full definitions of deferred toolsets.""" - - def __init__(self) -> None: - self._catalog: list[dict[str, str]] = [] - self._definitions: dict[str, OpenAiToolConfigDict] = {} - - def register_deferred_tools( - self, - catalog_entries: list[dict[str, str]], - definitions: dict[str, OpenAiToolConfigDict], - ) -> None: - self._catalog.extend(catalog_entries) - self._definitions.update(definitions) - - @property - def deferred_names(self) -> frozenset[str]: - return frozenset(self._definitions.keys()) - - @property - def catalog(self) -> list[dict[str, str]]: - return list(self._catalog) - - def get_definition(self, name: str) -> OpenAiToolConfigDict | None: - return self._definitions.get(name) - - -def register_tools_as_deferred( - tools: list[StagedBaseTool], - context: _DeferredToolsContext, -) -> None: - """Build catalog + definitions from tools and register them as deferred.""" - entries: list[tuple[StagedBaseTool, str]] = [ - (t, name) - for t in tools - if isinstance(t.tool_config, BaseOpenAITool) - and (name := t.tool_config.open_ai_tool.function.name) - ] - catalog: list[dict[str, str]] = [ - { - "name": name, - "description": t.tool_config.open_ai_tool.function.description or "", # type: ignore[union-attr] - } - for t, name in entries - ] - definitions: dict[str, OpenAiToolConfigDict] = { - name: t.tool_config.open_ai_tool.model_dump(mode="json", exclude_none=True) # type: ignore[union-attr] - for t, name in entries - } - context.register_deferred_tools(catalog, definitions) diff --git a/src/quickapp/core/agent/agent_module.py b/src/quickapp/core/agent/agent_module.py index 17d32758..ad3a022e 100644 --- a/src/quickapp/core/agent/agent_module.py +++ b/src/quickapp/core/agent/agent_module.py @@ -47,8 +47,6 @@ ) from quickapp.core.agent._attachment_filter import _AttachmentFilter from quickapp.core.agent._chat_completion_config_builder import _ChatCompletionConfigBuilder -from quickapp.core.agent._deferred_tools_context import _DeferredToolsContext -from quickapp.core.agent._lazy_loaded_tools_holder import _LazyLoadedToolsHolder from quickapp.core.agent._messages_transformers import _AddSystemPromptTransformer from quickapp.core.agent._orchestrator_deployment_initializer import ( _OrchestratorDeploymentInitializer, @@ -57,6 +55,7 @@ from quickapp.core.agent._prompt_providers import ConfigBasedPromptProvider from quickapp.core.agent._tool_choice_holder import _ToolChoiceHolder from quickapp.core.agent.assistant_invoker import AssistantInvoker +from quickapp.core.agent.lazy_loaded_tools_holder import LazyLoadedToolsHolder from quickapp.core.agent.models import OpenAiToolConfigDict from quickapp.core.agent.orchestrator import Orchestrator from quickapp.core.agent.orchestrator_capabilities import OrchestratorCapabilities @@ -64,6 +63,7 @@ OrchestratorDeploymentCacheService, ) from quickapp.core.application._request_context import _RequestContext +from quickapp.tool_discovery._deferred_tools_context import DeferredToolsContext DEFAULT_QUERY_PARAM = ConfigurableSchemaSimpleType( type=JsonTypeEnum.string, @@ -105,8 +105,8 @@ def configure(self, binder: Binder) -> None: ) binder.bind(AssistantInvoker, to=AssistantInvoker, scope=NoScope) binder.bind(_ChatCompletionConfigBuilder, to=_ChatCompletionConfigBuilder, scope=NoScope) - binder.bind(_DeferredToolsContext, to=_DeferredToolsContext, scope=request_scope) - binder.bind(_LazyLoadedToolsHolder, to=_LazyLoadedToolsHolder, scope=request_scope) + binder.bind(DeferredToolsContext, to=DeferredToolsContext, scope=request_scope) + binder.bind(LazyLoadedToolsHolder, to=LazyLoadedToolsHolder, scope=request_scope) binder.bind(ChatStreamSinkFactory, to=ChatStreamSinkFactory, scope=NoScope) binder.bind(ChatCompletionStreamHandler, to=ChatCompletionStreamHandler, scope=NoScope) binder.bind(_AttachmentFilter, to=_AttachmentFilter, scope=request_scope) @@ -171,7 +171,7 @@ def provide_openai_tools( self, tools: list[StagedBaseTool], static_tools: list[StaticTool], - deferred_context: _DeferredToolsContext, + deferred_context: DeferredToolsContext, ) -> list[OpenAiToolConfigDict]: deferred_names = deferred_context.deferred_names openai_functions = [] diff --git a/src/quickapp/core/agent/_lazy_loaded_tools_holder.py b/src/quickapp/core/agent/lazy_loaded_tools_holder.py similarity index 96% rename from src/quickapp/core/agent/_lazy_loaded_tools_holder.py rename to src/quickapp/core/agent/lazy_loaded_tools_holder.py index 048a26bb..ab007bd0 100644 --- a/src/quickapp/core/agent/_lazy_loaded_tools_holder.py +++ b/src/quickapp/core/agent/lazy_loaded_tools_holder.py @@ -4,7 +4,7 @@ @inject -class _LazyLoadedToolsHolder: +class LazyLoadedToolsHolder: """Request-scoped accumulator for tool schemas discovered via tool_search. _ToolSearchTool writes to this holder during execution. diff --git a/src/quickapp/mcp_tooling/_mcp_tool_initializer.py b/src/quickapp/mcp_tooling/_mcp_tool_initializer.py index a7c8d2ff..97c9391c 100644 --- a/src/quickapp/mcp_tooling/_mcp_tool_initializer.py +++ b/src/quickapp/mcp_tooling/_mcp_tool_initializer.py @@ -27,16 +27,16 @@ from quickapp.config.toolsets.authorization import MCPApiKeyAuthorization from quickapp.config.toolsets.dial_mcp import DialMCPToolSet from quickapp.config.toolsets.mcp import MCPProtocol, MCPServerInfo, MCPToolSet -from quickapp.core.agent._deferred_tools_context import ( - _DeferredToolsContext, - register_tools_as_deferred, -) from quickapp.dial_core_services._interactive_login_service import InteractiveLoginService from quickapp.dial_core_services._login_result import LoginResult from quickapp.dial_core_services.tool_config_service import ToolConfigCoreService from quickapp.mcp_tooling._mcp_eager_resource import MCPEagerTextResource from quickapp.mcp_tooling._mcp_resource_meta import MCPResourceMeta from quickapp.mcp_tooling._mcp_server_capabilities import MCPServerCapabilities +from quickapp.tool_discovery._deferred_tools_context import ( + DeferredToolsContext, + is_toolset_deferred, +) from ._di_types import DialToolsetCacheService from ._mcp_tool import _MCPTool @@ -137,7 +137,7 @@ def __init__( login_service: InteractiveLoginService, accept_language: ACCEPT_LANGUAGE, app_config: ApplicationConfig, - deferred_context: _DeferredToolsContext, + deferred_context: DeferredToolsContext, ): # Resolved lazily in initialize() because dial_app_tooling contributes # to this multibinder only after _DialAppResolver runs. @@ -154,7 +154,7 @@ def __init__( self.__login_service: InteractiveLoginService = login_service self.__accept_language: ACCEPT_LANGUAGE = accept_language self.__app_config: ApplicationConfig = app_config - self.__deferred_context: _DeferredToolsContext = deferred_context + self.__deferred_context: DeferredToolsContext = deferred_context @staticmethod # todo add Title to config so that we could use it in stage name @@ -294,19 +294,14 @@ async def _load_tools( created_tools.append(mcp_tool) if created_tools: discovery_cfg = self.__app_config.orchestrator.tool_discovery - deferred_effective = ( - toolset_info.deferred is not False - and discovery_cfg is not None - and discovery_cfg.enabled - and len(created_tools) >= discovery_cfg.min_tools_for_deferral - ) - if deferred_effective: - register_tools_as_deferred(created_tools, self.__deferred_context) - logger.debug( - "Deferred %d tools from MCP toolset '%s' into DeferredToolsContext", - len(created_tools), - resolve_localized(resolved_toolset.name), - ) + if is_toolset_deferred(toolset_info, discovery_cfg, len(created_tools)): + self.__deferred_context.register_staged_tools(created_tools) + if logger.isEnabledFor(logging.DEBUG): + logger.debug( + "Deferred %d tools from MCP toolset '%s' into DeferredToolsContext", + len(created_tools), + resolve_localized(resolved_toolset.name), + ) self.__mcp_context.extend_tools(created_tools) async def _load_resources( diff --git a/src/quickapp/rest_api_tooling/rest_api_tooling_module.py b/src/quickapp/rest_api_tooling/rest_api_tooling_module.py index 10e13c74..5c72593b 100644 --- a/src/quickapp/rest_api_tooling/rest_api_tooling_module.py +++ b/src/quickapp/rest_api_tooling/rest_api_tooling_module.py @@ -10,9 +10,9 @@ from quickapp.config.application import ApplicationConfig from quickapp.config.tools.rest_api import RestApiTool from quickapp.config.toolsets.rest_api import RestApiToolSet -from quickapp.core.agent._deferred_tools_context import ( - _DeferredToolsContext, - register_tools_as_deferred, +from quickapp.tool_discovery._deferred_tools_context import ( + DeferredToolsContext, + is_toolset_deferred, ) from ._request_detail_builder import _RequestDetailsBuilder @@ -37,7 +37,7 @@ def __provide_rest_api_tools( app_config: ApplicationConfig, tool_builder: ClassAssistedBuilder[_RestApiTool], accept_language: ACCEPT_LANGUAGE, - deferred_context: _DeferredToolsContext, + deferred_context: DeferredToolsContext, ) -> list[StagedBaseTool]: result: list[StagedBaseTool] = [] for toolset_info in app_config.tool_sets: @@ -45,19 +45,14 @@ def __provide_rest_api_tools( toolset_stage_name = resolve_localized(toolset_info.name, accept_language) tools = self.__create_rest_api_tools(toolset_info, tool_builder, toolset_stage_name) discovery_cfg = app_config.orchestrator.tool_discovery - deferred_effective = ( - toolset_info.deferred is not False - and discovery_cfg is not None - and discovery_cfg.enabled - and len(tools) >= discovery_cfg.min_tools_for_deferral - ) - if deferred_effective: - register_tools_as_deferred(tools, deferred_context) - logger.debug( - "Deferred %d tools from REST toolset '%s' into DeferredToolsContext", - len(tools), - toolset_stage_name, - ) + if is_toolset_deferred(toolset_info, discovery_cfg, len(tools)): + deferred_context.register_staged_tools(tools) + if logger.isEnabledFor(logging.DEBUG): + logger.debug( + "Deferred %d tools from REST toolset '%s' into DeferredToolsContext", + len(tools), + toolset_stage_name, + ) result.extend(tools) return result diff --git a/src/quickapp/tool_discovery/_anonymous_agent.py b/src/quickapp/tool_discovery/_anonymous_agent.py index a27fe8a1..b9ba622a 100644 --- a/src/quickapp/tool_discovery/_anonymous_agent.py +++ b/src/quickapp/tool_discovery/_anonymous_agent.py @@ -41,9 +41,10 @@ async def route(self, query: str, catalog: list[dict[str, str]]) -> list[str]: return [] discovery = self.__config.orchestrator.tool_discovery + assert discovery is not None # only reachable when tool_discovery is enabled service_model = ( - discovery.service_model if discovery is not None else None - ) or self.__config.orchestrator.deployment.deployment_id + discovery.service_model or self.__config.orchestrator.deployment.deployment_id + ) catalog_text = "\n".join( f"- {entry['name']}: {entry.get('description', '')}" for entry in catalog @@ -60,7 +61,7 @@ async def route(self, query: str, catalog: list[dict[str, str]]) -> list[str]: stream=False, ) except openai.OpenAIError: - logger.exception("Anonymous agent routing call failed for query=%r", query) + logger.exception("Anonymous agent routing call failed") return [] raw = (response.choices[0].message.content or "").strip() @@ -69,5 +70,5 @@ async def route(self, query: str, catalog: list[dict[str, str]]) -> list[str]: if isinstance(names, list): return [n for n in names if isinstance(n, str)] except (json.JSONDecodeError, ValueError): - logger.warning("Anonymous agent returned non-JSON response: %r", raw) + logger.warning("Anonymous agent returned non-JSON response (length=%d)", len(raw)) return [] diff --git a/src/quickapp/tool_discovery/_deferred_tools_context.py b/src/quickapp/tool_discovery/_deferred_tools_context.py new file mode 100644 index 00000000..2e4474e2 --- /dev/null +++ b/src/quickapp/tool_discovery/_deferred_tools_context.py @@ -0,0 +1,61 @@ +from injector import inject + +from quickapp.common import StagedBaseTool +from quickapp.config.tools.base import BaseOpenAITool +from quickapp.config.toolsets.base import BaseToolSet +from quickapp.core.agent.models import OpenAiToolConfigDict +from quickapp.tool_discovery._tool_discovery_config import ToolDiscoveryConfig + + +@inject +class DeferredToolsContext: + """Request-scoped holder for tool catalog and full definitions of deferred toolsets.""" + + def __init__(self) -> None: + self._catalog: list[dict[str, str]] = [] + self._definitions: dict[str, OpenAiToolConfigDict] = {} + + def register_staged_tools(self, tools: list[StagedBaseTool]) -> None: + entries: list[tuple[BaseOpenAITool, str]] = [ + (t.tool_config, name) + for t in tools + if isinstance(t.tool_config, BaseOpenAITool) + and (name := t.tool_config.open_ai_tool.function.name) + ] + self._catalog.extend( + { + "name": name, + "description": tool_config.open_ai_tool.function.description or "", + } + for tool_config, name in entries + ) + self._definitions.update( + { + name: tool_config.open_ai_tool.model_dump(mode="json", exclude_none=True) + for tool_config, name in entries + } + ) + + @property + def deferred_names(self) -> frozenset[str]: + return frozenset(self._definitions.keys()) + + @property + def catalog(self) -> list[dict[str, str]]: + return list(self._catalog) + + def get_definition(self, name: str) -> OpenAiToolConfigDict | None: + return self._definitions.get(name) + + +def is_toolset_deferred( + toolset: BaseToolSet, + discovery_cfg: ToolDiscoveryConfig | None, + tool_count: int, +) -> bool: + return ( + toolset.deferred is not False + and discovery_cfg is not None + and discovery_cfg.enabled + and tool_count >= discovery_cfg.min_tools_for_deferral + ) diff --git a/src/quickapp/tool_discovery/_tool_search_tool.py b/src/quickapp/tool_discovery/_tool_search_tool.py index 3f830fe5..4bd42ba2 100644 --- a/src/quickapp/tool_discovery/_tool_search_tool.py +++ b/src/quickapp/tool_discovery/_tool_search_tool.py @@ -6,12 +6,13 @@ from quickapp.common import StagedBaseTool, ToolCallResult from quickapp.common.abstract.base_tool_argument_transformer import ToolArgumentTransformer +from quickapp.common.base_stage_wrapper import BaseStageWrapper from quickapp.common.perf_timer.perf_timer import PerformanceTimer from quickapp.config.application import StageDisplayLevel from quickapp.config.tools.internal import InternalTool -from quickapp.core.agent._deferred_tools_context import _DeferredToolsContext -from quickapp.core.agent._lazy_loaded_tools_holder import _LazyLoadedToolsHolder +from quickapp.core.agent.lazy_loaded_tools_holder import LazyLoadedToolsHolder from quickapp.tool_discovery._anonymous_agent import _AnonymousAgent +from quickapp.tool_discovery._deferred_tools_context import DeferredToolsContext from quickapp.tool_discovery._tool_search_stage_wrapper import _ToolSearchStageWrapper logger = logging.getLogger(__name__) @@ -26,8 +27,8 @@ def __init__( stage_wrapper_builder: AssistedBuilder[_ToolSearchStageWrapper], tool_config: InternalTool, perf_timer: PerformanceTimer, - deferred_context: _DeferredToolsContext, - lazy_holder: _LazyLoadedToolsHolder, + deferred_context: DeferredToolsContext, + lazy_holder: LazyLoadedToolsHolder, anonymous_agent: _AnonymousAgent, stage_display_level: StageDisplayLevel = StageDisplayLevel.INFO, argument_transformers: list[ToolArgumentTransformer] | None = None, @@ -47,7 +48,7 @@ def __init__( async def _run_in_stage_async( self, - stage_wrapper: Any = None, + stage_wrapper: BaseStageWrapper | None = None, tool_call_id: str | None = None, *args: Any, **kwargs: Any, diff --git a/src/tests/unit_tests/agent_tests/test_assistant_invoker.py b/src/tests/unit_tests/agent_tests/test_assistant_invoker.py index 0a75f784..c2e737ab 100644 --- a/src/tests/unit_tests/agent_tests/test_assistant_invoker.py +++ b/src/tests/unit_tests/agent_tests/test_assistant_invoker.py @@ -9,8 +9,8 @@ from quickapp.common.stage_close_registry import DeferredStageCloseRegistry from quickapp.core.agent import AssistantInvoker from quickapp.core.agent._chat_completion_config_builder import _ChatCompletionConfigBuilder -from quickapp.core.agent._lazy_loaded_tools_holder import _LazyLoadedToolsHolder from quickapp.core.agent._tool_choice_holder import _ToolChoiceHolder +from quickapp.core.agent.lazy_loaded_tools_holder import LazyLoadedToolsHolder def _presentation_settings(show_usage: bool): @@ -70,7 +70,7 @@ def _make_config_builder( pre_invocation_transformers=[mock_filter], presentation_settings=_presentation_settings(show_usage), forwarded_headers=forwarded_headers, - lazy_loaded_tools_holder=_LazyLoadedToolsHolder(), + lazy_loaded_tools_holder=LazyLoadedToolsHolder(), ) diff --git a/src/tests/unit_tests/agent_tests/test_tool_choice_config_builder.py b/src/tests/unit_tests/agent_tests/test_tool_choice_config_builder.py index 2b903559..df462657 100644 --- a/src/tests/unit_tests/agent_tests/test_tool_choice_config_builder.py +++ b/src/tests/unit_tests/agent_tests/test_tool_choice_config_builder.py @@ -5,8 +5,8 @@ from aidial_sdk.exceptions import InvalidRequestError from quickapp.core.agent._chat_completion_config_builder import _ChatCompletionConfigBuilder -from quickapp.core.agent._lazy_loaded_tools_holder import _LazyLoadedToolsHolder from quickapp.core.agent._tool_choice_holder import _ToolChoiceHolder +from quickapp.core.agent.lazy_loaded_tools_holder import LazyLoadedToolsHolder def _make_builder( @@ -25,7 +25,7 @@ def _make_builder( pre_invocation_transformers=[], presentation_settings=MagicMock(show_usage_statistics=False), forwarded_headers=None, - lazy_loaded_tools_holder=_LazyLoadedToolsHolder(), + lazy_loaded_tools_holder=LazyLoadedToolsHolder(), ) @@ -72,7 +72,7 @@ def test_tool_choice_consumed_only_on_first_build(self): pre_invocation_transformers=[], presentation_settings=MagicMock(show_usage_statistics=False), forwarded_headers=None, - lazy_loaded_tools_holder=_LazyLoadedToolsHolder(), + lazy_loaded_tools_holder=LazyLoadedToolsHolder(), ) first = builder.build([]) assert first["tool_choice"] == "required" From 544f5c14f04b27667e896c5f5699e89f4522bcdf Mon Sep 17 00:00:00 2001 From: Oleksandr Gubarets Date: Fri, 11 Sep 2026 16:31:27 +0300 Subject: [PATCH 06/10] code review fix. --- CONFIGURATION.md | 38 ++++ README.md | 203 ++++++++---------- src/quickapp/config/application.py | 2 +- .../tool_discovery.py} | 0 src/quickapp/core/agent/agent_module.py | 3 +- .../mcp_tooling/_mcp_tool_initializer.py | 16 +- .../rest_api_tooling_module.py | 16 +- src/quickapp/shared/__init__.py | 2 + .../shared/deferred_tools/__init__.py | 3 + .../_deferred_tools_context.py | 9 +- .../deferred_tools/deferred_tools_module.py | 16 ++ src/quickapp/tool_discovery/__init__.py | 3 - .../tool_discovery/_anonymous_agent.py | 7 +- .../tool_discovery/_tool_search_tool.py | 2 +- 14 files changed, 180 insertions(+), 140 deletions(-) rename src/quickapp/{tool_discovery/_tool_discovery_config.py => config/tool_discovery.py} (100%) create mode 100644 src/quickapp/shared/deferred_tools/__init__.py rename src/quickapp/{tool_discovery => shared/deferred_tools}/_deferred_tools_context.py (86%) create mode 100644 src/quickapp/shared/deferred_tools/deferred_tools_module.py diff --git a/CONFIGURATION.md b/CONFIGURATION.md index ceb9bcae..e141aece 100644 --- a/CONFIGURATION.md +++ b/CONFIGURATION.md @@ -257,6 +257,7 @@ The project contains predefined configs of application and predefined tools | deployment | Yes | Object | The DIAL deployment configuration. See [Deployment configuration](#deployment-configuration) | - | - | | system_prompt | Yes | Object | The configuration for the system prompt. See [System prompt configuration](#system-prompt-configuration) | - | - | | max_iterations | No | Integer | The max count of orchestrator(agent) operations. -1 value for infinite | Integer | 15 | +| tool_discovery | No | Object | `[Preview]` Dynamic tool discovery configuration. See [Tool discovery configuration](#tool-discovery-configuration) | - | `null` | #### Deployment configuration @@ -331,6 +332,38 @@ Custom system prompt: +#### Tool discovery configuration + +`[Preview]` Requires `ENABLE_PREVIEW_FEATURES=true`. When enabled, toolsets withheld from the initial LLM payload +(see the per-toolset `deferred` field in [Tool sets configuration](#tool-sets-configuration)) are surfaced on demand +via a `tool_search` meta-tool, which routes the query to the matching tool schemas through an isolated LLM call. + +| Field | Required | Type | Description | Available Values | Default Value | +|------------------------|----------|---------|-------------------------------------------------------------------------------------------------------------------------------------------------------------|-------------------|---------------| +| enabled | No | Boolean | Enable dynamic tool discovery. When `true`, toolsets with `deferred: true` are withheld from the initial LLM payload and surfaced via the `tool_search` meta-tool. | - | `false` | +| service_model | No | String | DIAL deployment used for the anonymous routing call inside `tool_search`. Falls back to the orchestrator's own deployment when omitted. | - | - | +| min_tools_for_deferral | No | Integer | Minimum number of tools in a toolset for deferral to apply. Toolsets smaller than this threshold are promoted to eager loading even when `deferred: true`. Deployment-wide default set by `MIN_TOOLS_FOR_DEFERRAL`. | - | `5` | + +
+Tool discovery configuration JSON sample + +```json +{ + "orchestrator": { + "deployment": { "name": "gpt-4o" }, + "tool_discovery": { + "enabled": true, + "service_model": "gpt-4o-mini", + "min_tools_for_deferral": 5 + } + } +} +``` + +
+ +See [Dynamic Tool Discovery design doc](docs/designs/dynamic_tool_discovery.md) for the full behavioral reference. + ### Contexts configuration | Field | Required | Type | Description | Available Values | Default Value | @@ -471,6 +504,11 @@ SSRF envelope, deployment dispatch table, error messages and agent retry behavio ### Tool sets configuration +Every toolset type also accepts a `deferred` field (Boolean, default `true`): `[Preview]` when true or unset, and +[Tool discovery configuration](#tool-discovery-configuration) is enabled, the toolset's tool schemas are withheld +from the initial LLM payload and discovered on demand via the `tool_search` meta-tool. Set to `false` to keep a +specific toolset always eager. + #### RestApiToolSet Configuration | Field | Required | Type | Description | Default Value | diff --git a/README.md b/README.md index 2460b80c..5fdbd7f9 100644 --- a/README.md +++ b/README.md @@ -40,8 +40,7 @@ Features in Preview are marked with a `[Preview]` tag in documentation. - [Configuration Reference](./CONFIGURATION.md) - Full configuration model, environment variables, and examples - [Agent Skills](docs/skills.md) - How to create and manage reusable agent skills -- [Config-Driven Hooks](docs/designs/config_driven_hooks.md) `[Preview]` - Declarative synthetic tool call injection at - orchestrator seams +- [Config-Driven Hooks](docs/designs/config_driven_hooks.md) `[Preview]` - Declarative synthetic tool call injection at orchestrator seams - [Technical Documentation](./docs/README.md) - Internal architecture and design documents ## Quick start (general) @@ -60,8 +59,7 @@ file: ### Hooks `[Preview]` -Hooks let you pre-populate the agent's message history with synthetic tool call results — without writing Python code. -Each hook fires at a named orchestrator seam and injects a `(ASSISTANT/tool_calls, TOOL)` message pair. +Hooks let you pre-populate the agent's message history with synthetic tool call results — without writing Python code. Each hook fires at a named orchestrator seam and injects a `(ASSISTANT/tool_calls, TOOL)` message pair. Enable with `ENABLE_PREVIEW_FEATURES=true`, then add a `hooks` array to the app manifest: @@ -73,9 +71,7 @@ Enable with `ENABLE_PREVIEW_FEATURES=true`, then add a `hooks` array to the app "event": "on_request_start", "toolset_name": "memory_server", "tool_name": "get_memories", - "arguments": { - "user_id": "123" - }, + "arguments": { "user_id": "123" }, "frequency": "always" } ] @@ -84,23 +80,20 @@ Enable with `ENABLE_PREVIEW_FEATURES=true`, then add a `hooks` array to the app Key fields: -| Field | Description | -|----------------|--------------------------------------------------------------------------------------------------------------------------------------| -| `kind` | Hook type. Only `"tool_call"` is supported today. | -| `event` | Orchestrator seam. Only `"on_request_start"` is wired today. | -| `toolset_name` | Toolset prefix for REST API / MCP tools. Omit for DIAL Deployment and Internal tools. | -| `tool_name` | Tool name within the toolset, or the exact function name when `toolset_name` is omitted. | -| `arguments` | Arguments forwarded to the tool call. | -| `frequency` | `"always"` — inject on every request. `"append_if_changed"` (default) — inject only when the result differs from the last injection. | +| Field | Description | +|---|---| +| `kind` | Hook type. Only `"tool_call"` is supported today. | +| `event` | Orchestrator seam. Only `"on_request_start"` is wired today. | +| `toolset_name` | Toolset prefix for REST API / MCP tools. Omit for DIAL Deployment and Internal tools. | +| `tool_name` | Tool name within the toolset, or the exact function name when `toolset_name` is omitted. | +| `arguments` | Arguments forwarded to the tool call. | +| `frequency` | `"always"` — inject on every request. `"append_if_changed"` (default) — inject only when the result differs from the last injection. | See [Config-Driven Hooks design doc](docs/designs/config_driven_hooks.md) for the full reference. ### Dynamic Tool Discovery `[Preview]` -Dynamic tool discovery defers large toolsets from the initial LLM payload and surfaces them -on demand via a `tool_search` meta-tool. The orchestrator calls `tool_search` with a natural-language -query when it needs a tool it hasn't seen yet; a lightweight anonymous LLM routing call selects -the relevant tool schemas and injects them into the next iteration. +Dynamic tool discovery defers large toolsets from the initial LLM payload and surfaces them on demand via a `tool_search` meta-tool. The orchestrator calls `tool_search` with a natural-language query when it needs a tool it hasn't seen yet; a lightweight anonymous LLM routing call selects the relevant tool schemas and injects them into the next iteration. Enable with `ENABLE_PREVIEW_FEATURES=true`, then add `orchestrator.tool_discovery` to the app manifest: @@ -127,8 +120,7 @@ Enable with `ENABLE_PREVIEW_FEATURES=true`, then add `orchestrator.tool_discover } ``` -Toolsets are deferred by default — omitting `deferred` or setting it to `true` both defer the -toolset. To keep a specific toolset always eager, set `"deferred": false` on that toolset: +Toolsets are deferred by default — omitting `deferred` or setting it to `true` both defer the toolset. To keep a specific toolset always eager, set `"deferred": false` on that toolset: ```json { @@ -141,17 +133,16 @@ toolset. To keep a specific toolset always eager, set `"deferred": false` on tha Key fields: -| Field | Default | Description | -|------------------------------------------------|---------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| `orchestrator.tool_discovery.enabled` | `false` | Activates dynamic discovery for this app. Must be `true` for deferral to take effect. | -| `orchestrator.tool_discovery.service_model` | — | DIAL deployment used for the anonymous routing call inside `tool_search`. Falls back to the orchestrator's own deployment when omitted. | -| `orchestrator.tool_discovery.min_tools_for_deferral` | `5` | Minimum number of tools in a toolset for deferral to apply. Toolsets smaller than this threshold are promoted to eager loading even when `deferred: true`. | -| `.deferred` | `true` | Per-toolset opt-out. Set to `false` to force a specific toolset into the initial payload regardless of `tool_discovery.enabled`. | +| Field | Default | Description | +|---|---|---| +| `orchestrator.tool_discovery.enabled` | `false` | Activates dynamic discovery for this app. Must be `true` for deferral to take effect. | +| `orchestrator.tool_discovery.service_model` | — | DIAL deployment used for the anonymous routing call inside `tool_search`. Falls back to the orchestrator's own deployment when omitted. | +| `orchestrator.tool_discovery.min_tools_for_deferral` | `5` | Minimum number of tools in a toolset for deferral to apply. Toolsets smaller than this threshold are promoted to eager loading even when `deferred: true`. | +| `.deferred` | `true` | Per-toolset opt-out. Set to `false` to force a specific toolset into the initial payload regardless of `tool_discovery.enabled`. | -The `MIN_TOOLS_FOR_DEFERRAL` environment variable sets the deployment-wide default for -`min_tools_for_deferral`; individual apps can override it in their manifest. +The `MIN_TOOLS_FOR_DEFERRAL` environment variable sets the deployment-wide default for `min_tools_for_deferral`; individual apps can override it in their manifest. -See [Dynamic Tool Discovery design doc](docs/designs/dynamic_tool_discovery.md) for the full reference. +See [Tool discovery configuration](./CONFIGURATION.md#tool-discovery-configuration) for the full field reference and the [Dynamic Tool Discovery design doc](docs/designs/dynamic_tool_discovery.md) for the behavioral design. ### Forwarding headers @@ -168,14 +159,13 @@ your gateways or downstream services expect. ### Stage display level -Controls which tool-execution stages are surfaced in the DIAL UI for each app. Set `features.stage_display.level` in the -app manifest: +Controls which tool-execution stages are surfaced in the DIAL UI for each app. Set `features.stage_display.level` in the app manifest: -| Value | Behavior | -|---------|----------------------------------------------------------------| -| `none` | No stages shown at all, not even for errors | -| `error` | Show stages only for failed tool calls | -| `info` | Show stages for regular tool calls and errors (default) | +| Value | Behavior | +|---|---| +| `none` | No stages shown at all, not even for errors | +| `error` | Show stages only for failed tool calls | +| `info` | Show stages for regular tool calls and errors (default) | | `debug` | Show stages for all tool calls, including internal/system ones | ```json @@ -190,77 +180,77 @@ app manifest: ### Environment Variables -| Variable | Default | Required | Description | -|------------------------------------------------|-----------------------------------------------------------------|----------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| **DIAL Core** | | | | -| `DIAL_URL` | — | Yes | URL of the DIAL Core API | -| `DIAL_API_VERSION` | `2025-01-01-preview` | No | API version for DIAL Core API | -| `APP_SCHEMA_ID` | `https://mydial.epam.com/custom_application_schemas/quickapps2` | No | Full application type schema `$id` emitted in the generated app schema. When unset, the built-in default is used. | -| **Proxy** | | | | -| `PROXY_LANGUAGE_HEADER` | `accept-language` | No | Name of the incoming HTTP request header that carries the locale for UI display (stage name localization). Override when a reverse proxy rewrites the standard `Accept-Language` header before forwarding the request. | -| **Logging** | | | | -| `DIAL_SDK_LOG_FORMAT` | `text` | No | Console log output format: `text` (human-readable) or `json` (escape-safe, one record per line). See [docs/logging.md](docs/logging.md). | -| `DIAL_SDK_TEXT_LOG_FORMAT` | [see docs/logging.md](docs/logging.md) | No | Custom `%`-style format string for `text` output. Unset (default) keeps the built-in format with the conditional OTEL trace block. | -| `DIAL_SDK_JSON_LOG_FORMAT` | [see docs/logging.md](docs/logging.md) | No | Custom template for `json` output — a JSON document whose string leaves are `%`-style format strings, values escaped via `json.dumps`. | -| `LOG_LEVEL` | `INFO` | No | Root logger level (all loggers except quickapp) | -| `QUICKAPP_LOG_LEVEL` | `INFO` | No | Log level for quickapp loggers | -| `LOG_PAYLOADS` | `false` | No | Emit payload content (message bodies, tool-call arguments, tool/LLM response bodies) at DEBUG. When `false`, no payload content is logged at **any** level and the payload-capable third-party loggers (`openai`/`httpx`/`httpcore`) are capped at INFO. **Local development only** — see [Payload Logging](#payload-logging). | -| `LOG_PAYLOADS_MAX_LENGTH` | `2000` | No | Per-field character cap applied to each payload value when `LOG_PAYLOADS=true`; longer values are truncated. Inert when `LOG_PAYLOADS=false`. | -| **Agent** | | | | -| `DEFAULT_AGENT_MAX_ITERATIONS` | `15` | No | Maximum number of orchestrator iterations (`-1` for infinite) | -| `DEFAULT_ORCHESTRATOR_DEPLOYMENT_ID` | — | No | Default DIAL deployment id used as the orchestrator model when a QuickApp manifest omits `orchestrator.deployment`. Also surfaces as the JSON-schema `default` for that field so DIAL Core can pre-fill new manifests. Apps can override per-app. | -| `SHOW_USAGE_STATISTICS` | `false` | No | Include usage statistics in chat completion stream | -| `SHOW_EXECUTION_TIME_STAGE` | `false` | No | Show execution time stage in the UI | -| **Python Interpreter** | | | | -| `PY_INTERPRETER_LOCAL_RUN` | `false` | No | Run PyInterpreter locally instead of via DIAL Core API | -| `PY_INTERPRETER_URL` | *(falls back to DIAL_URL)* | No | URL of the PyInterpreter service | -| `PY_INTERPRETER_API_KEY` | — | No | API key for local-run PyInterpreter | -| `PY_INTERPRETER_DEFAULT_SESSION_ID` | — | No | Default session ID for the PyInterpreter | -| `PY_INTERPRETER_CLIENT_MAX_RETRIES` | `3` | No | Max retries for PyInterpreter client requests | -| **Tool Timeouts** | | | | -| `DEFAULT_TOOL_TIMEOUT_SECONDS` | `300.0` | No | Deployment-wide default timeout (seconds, `0 < x ≤ 3600`) applied to every tool call (deployment, REST API, MCP, Python interpreter). Apps can override per-app via `tool_defaults.timeout_seconds`. | -| `DEFAULT_FILE_LOADING_SIZE_LIMIT` | `10485760` | No | Deployment-wide default maximum size (in bytes) for files the agent downloads. Apps can override per-app via `features.file_loading.size_limit`. | -| **Stage Display** | | | | -| `DEFAULT_STAGE_DISPLAY_LEVEL` | — | No | Deployment-wide override for stage visibility threshold (`none`, `error`, `info`, `debug`; case-insensitive). When set, wins over every app's `features.stage_display.level`. Unset (default) defers to the per-app config, which defaults to `info`. | -| **DIAL Files — Tool-Response Offload** | | | | -| `TOOL_CALL_RESULT_OFFLOAD__ENABLED_BY_DEFAULT` | `true` | No | Default value of the per-app `enabled` flag (`features.dial_files.tool_call_result_offload.enabled`). Apps override per-app; `enabled: false` disables offload for that app. | -| `TOOL_CALL_RESULT_OFFLOAD__SIZE_THRESHOLD` | `40000` | No | Default byte threshold above which a tool-call response is offloaded to a DIAL file. Apps override per-app via `features.dial_files.tool_call_result_offload.size_threshold`. | -| `TOOL_CALL_RESULT_OFFLOAD__EXCLUDED_TOOLS` | `[]` | No | Default JSON list of **additional** tool names exempt from offloading. The read-back tools (`internal_file_read_lines`, `internal_file_search`) are always excluded regardless of this value, so a large read-back slice is never re-offloaded. Apps add more per-app via `features.dial_files.tool_call_result_offload.excluded_tools`. | -| **External URL Egress** | | | | -| `EXTERNAL_URL_FETCH_ENABLED` | `false` | No | Admin cap on fetching external (non-DIAL) URLs. When `false` (default), no app may fetch external URLs regardless of its manifest; the deployment-handoff branch (deployments with `features.url_attachments`) is unaffected. Apps can opt out per-app via `features.external_url_fetch.enabled=false` even when the admin allows. | -| `EXTERNAL_URL_FETCH_HOST_ALLOWLIST` | — | No | Comma-separated allowlist of host patterns for external URL fetches. Unset (default) means no admin-level host restriction. Patterns: exact host (`example.com`) or `*.example.com` for any subdomain. Re-checked on every redirect hop. Per-app `features.external_url_fetch.host_allowlist` narrows further (intersection) but never expands. | -| `EXTERNAL_URL_FETCH_MAX_REDIRECTS` | `5` | No | Maximum HTTP redirects on external URL fetches. Each hop is SSRF-checked. Hard ceiling 10. | -| `EXTERNAL_URL_FETCH_CONNECT_TIMEOUT_SECONDS` | `5.0` | No | TCP connect timeout (seconds) for external URL fetches. Read/write/pool timeouts use the resolved tool timeout. | -| **Dynamic Tool Discovery** `[Preview]` | | | | -| `MIN_TOOLS_FOR_DEFERRAL` | `5` | No | Deployment-wide minimum toolset size for deferral to apply. Toolsets with fewer tools than this threshold are promoted to eager loading even when `deferred=true`. Apps override per-app via `orchestrator.tool_discovery.min_tools_for_deferral`. Requires `ENABLE_PREVIEW_FEATURES=true`. | -| **Feature Gating** | | | | -| `ENABLE_PREVIEW_FEATURES` | `false` | No | Enable preview features across the deployment (schema visibility + runtime activation) | -| **Templates** | | | | -| `PREDEFINED_EXTRA_PATHS` | — | No | JSON list of directories layered on top of built-in predefined content (later entries override earlier ones) | -| `CONFIG_PROMPT_MAPPING` | *(built-in mapping)* | No | JSON mapping of predefined system prompts to DIAL Core deployments | -| **Observability** | | | | -| `OTEL_SERVICE_NAME` | `quickapps` | No | Service name stamped on all exported telemetry (traces, metrics, logs) | -| `OTEL_TRACES_EXPORTER` | — | No | Set to `otlp` to enable tracing and export spans over OTLP/gRPC. Instruments the FastAPI server and outgoing HTTP clients (`httpx`, `requests`, `aiohttp`, `urllib`) and stamps trace context onto log records — see [docs/logging.md](docs/logging.md). | -| `OTEL_METRICS_EXPORTER` | — | No | Comma-separated metric exporters: `otlp` (push over OTLP/gRPC) and/or `prometheus` (serve a scrape endpoint). Enables FastAPI and system/process metrics. | -| `OTEL_LOGS_EXPORTER` | — | No | Set to `otlp` to export log records (INFO and above) over OTLP/gRPC alongside console output — see [docs/logging.md](docs/logging.md). | -| `OTEL_EXPORTER_OTLP_ENDPOINT` | `http://localhost:4317` | No | OTLP/gRPC collector endpoint shared by trace, metric, and log export. One of the [standard OpenTelemetry SDK variables](https://opentelemetry.io/docs/specs/otel/configuration/sdk-environment-variables/), which the underlying exporters honor as usual (per-signal endpoints, headers, timeouts, resource attributes, …). | -| `OTEL_EXPORTER_PROMETHEUS_PORT` | `9464` | No | Port of the Prometheus scrape endpoint (effective only with `prometheus` in `OTEL_METRICS_EXPORTER`) | -| **Scripts & Tests** | | | | -| `REMOTE_DIAL_URL` | — | No | URL of the remote DIAL Core, used only by `generate_dial_config` script and e2e/integration tests | -| `REMOTE_DIAL_API_KEY` | — | No | API key of the remote DIAL Core, used only by `generate_dial_config` script and e2e/integration tests | +| Variable | Default | Required | Description | +|--------------------------------------------|----------------------------|----------|----------------------------------------------------------------------------------------------------------------| +| **DIAL Core** | | | | +| `DIAL_URL` | — | Yes | URL of the DIAL Core API | +| `DIAL_API_VERSION` | `2025-01-01-preview` | No | API version for DIAL Core API | +| `APP_SCHEMA_ID` | `https://mydial.epam.com/custom_application_schemas/quickapps2` | No | Full application type schema `$id` emitted in the generated app schema. When unset, the built-in default is used. | +| **Proxy** | | | | +| `PROXY_LANGUAGE_HEADER` | `accept-language` | No | Name of the incoming HTTP request header that carries the locale for UI display (stage name localization). Override when a reverse proxy rewrites the standard `Accept-Language` header before forwarding the request. | +| **Logging** | | | | +| `DIAL_SDK_LOG_FORMAT` | `text` | No | Console log output format: `text` (human-readable) or `json` (escape-safe, one record per line). See [docs/logging.md](docs/logging.md). | +| `DIAL_SDK_TEXT_LOG_FORMAT` | [see docs/logging.md](docs/logging.md) | No | Custom `%`-style format string for `text` output. Unset (default) keeps the built-in format with the conditional OTEL trace block. | +| `DIAL_SDK_JSON_LOG_FORMAT` | [see docs/logging.md](docs/logging.md) | No | Custom template for `json` output — a JSON document whose string leaves are `%`-style format strings, values escaped via `json.dumps`. | +| `LOG_LEVEL` | `INFO` | No | Root logger level (all loggers except quickapp) | +| `QUICKAPP_LOG_LEVEL` | `INFO` | No | Log level for quickapp loggers | +| `LOG_PAYLOADS` | `false` | No | Emit payload content (message bodies, tool-call arguments, tool/LLM response bodies) at DEBUG. When `false`, no payload content is logged at **any** level and the payload-capable third-party loggers (`openai`/`httpx`/`httpcore`) are capped at INFO. **Local development only** — see [Payload Logging](#payload-logging). | +| `LOG_PAYLOADS_MAX_LENGTH` | `2000` | No | Per-field character cap applied to each payload value when `LOG_PAYLOADS=true`; longer values are truncated. Inert when `LOG_PAYLOADS=false`. | +| **Agent** | | | | +| `DEFAULT_AGENT_MAX_ITERATIONS` | `15` | No | Maximum number of orchestrator iterations (`-1` for infinite) | +| `DEFAULT_ORCHESTRATOR_DEPLOYMENT_ID` | — | No | Default DIAL deployment id used as the orchestrator model when a QuickApp manifest omits `orchestrator.deployment`. Also surfaces as the JSON-schema `default` for that field so DIAL Core can pre-fill new manifests. Apps can override per-app. | +| `SHOW_USAGE_STATISTICS` | `false` | No | Include usage statistics in chat completion stream | +| `SHOW_EXECUTION_TIME_STAGE` | `false` | No | Show execution time stage in the UI | +| **Python Interpreter** | | | | +| `PY_INTERPRETER_LOCAL_RUN` | `false` | No | Run PyInterpreter locally instead of via DIAL Core API | +| `PY_INTERPRETER_URL` | *(falls back to DIAL_URL)* | No | URL of the PyInterpreter service | +| `PY_INTERPRETER_API_KEY` | — | No | API key for local-run PyInterpreter | +| `PY_INTERPRETER_DEFAULT_SESSION_ID` | — | No | Default session ID for the PyInterpreter | +| `PY_INTERPRETER_CLIENT_MAX_RETRIES` | `3` | No | Max retries for PyInterpreter client requests | +| **Tool Timeouts** | | | | +| `DEFAULT_TOOL_TIMEOUT_SECONDS` | `300.0` | No | Deployment-wide default timeout (seconds, `0 < x ≤ 3600`) applied to every tool call (deployment, REST API, MCP, Python interpreter). Apps can override per-app via `tool_defaults.timeout_seconds`. | +| `DEFAULT_FILE_LOADING_SIZE_LIMIT` | `10485760` | No | Deployment-wide default maximum size (in bytes) for files the agent downloads. Apps can override per-app via `features.file_loading.size_limit`. | +| **Stage Display** | | | | +| `DEFAULT_STAGE_DISPLAY_LEVEL` | — | No | Deployment-wide override for stage visibility threshold (`none`, `error`, `info`, `debug`; case-insensitive). When set, wins over every app's `features.stage_display.level`. Unset (default) defers to the per-app config, which defaults to `info`. | +| **DIAL Files — Tool-Response Offload** | | | | +| `TOOL_CALL_RESULT_OFFLOAD__ENABLED_BY_DEFAULT` | `true` | No | Default value of the per-app `enabled` flag (`features.dial_files.tool_call_result_offload.enabled`). Apps override per-app; `enabled: false` disables offload for that app. | +| `TOOL_CALL_RESULT_OFFLOAD__SIZE_THRESHOLD` | `40000` | No | Default byte threshold above which a tool-call response is offloaded to a DIAL file. Apps override per-app via `features.dial_files.tool_call_result_offload.size_threshold`. | +| `TOOL_CALL_RESULT_OFFLOAD__EXCLUDED_TOOLS` | `[]` | No | Default JSON list of **additional** tool names exempt from offloading. The read-back tools (`internal_file_read_lines`, `internal_file_search`) are always excluded regardless of this value, so a large read-back slice is never re-offloaded. Apps add more per-app via `features.dial_files.tool_call_result_offload.excluded_tools`. | +| **External URL Egress** | | | | +| `EXTERNAL_URL_FETCH_ENABLED` | `false` | No | Admin cap on fetching external (non-DIAL) URLs. When `false` (default), no app may fetch external URLs regardless of its manifest; the deployment-handoff branch (deployments with `features.url_attachments`) is unaffected. Apps can opt out per-app via `features.external_url_fetch.enabled=false` even when the admin allows. | +| `EXTERNAL_URL_FETCH_HOST_ALLOWLIST` | — | No | Comma-separated allowlist of host patterns for external URL fetches. Unset (default) means no admin-level host restriction. Patterns: exact host (`example.com`) or `*.example.com` for any subdomain. Re-checked on every redirect hop. Per-app `features.external_url_fetch.host_allowlist` narrows further (intersection) but never expands. | +| `EXTERNAL_URL_FETCH_MAX_REDIRECTS` | `5` | No | Maximum HTTP redirects on external URL fetches. Each hop is SSRF-checked. Hard ceiling 10. | +| `EXTERNAL_URL_FETCH_CONNECT_TIMEOUT_SECONDS` | `5.0` | No | TCP connect timeout (seconds) for external URL fetches. Read/write/pool timeouts use the resolved tool timeout. | +| **Dynamic Tool Discovery** `[Preview]` | | | | +| `MIN_TOOLS_FOR_DEFERRAL` | `5` | No | Deployment-wide minimum toolset size for deferral to apply. Toolsets with fewer tools than this threshold are promoted to eager loading even when `deferred=true`. Apps override per-app via `orchestrator.tool_discovery.min_tools_for_deferral`. Requires `ENABLE_PREVIEW_FEATURES=true`. | +| **Feature Gating** | | | | +| `ENABLE_PREVIEW_FEATURES` | `false` | No | Enable preview features across the deployment (schema visibility + runtime activation) | +| **Templates** | | | | +| `PREDEFINED_EXTRA_PATHS` | — | No | JSON list of directories layered on top of built-in predefined content (later entries override earlier ones) | +| `CONFIG_PROMPT_MAPPING` | *(built-in mapping)* | No | JSON mapping of predefined system prompts to DIAL Core deployments | +| **Observability** | | | | +| `OTEL_SERVICE_NAME` | `quickapps` | No | Service name stamped on all exported telemetry (traces, metrics, logs) | +| `OTEL_TRACES_EXPORTER` | — | No | Set to `otlp` to enable tracing and export spans over OTLP/gRPC. Instruments the FastAPI server and outgoing HTTP clients (`httpx`, `requests`, `aiohttp`, `urllib`) and stamps trace context onto log records — see [docs/logging.md](docs/logging.md). | +| `OTEL_METRICS_EXPORTER` | — | No | Comma-separated metric exporters: `otlp` (push over OTLP/gRPC) and/or `prometheus` (serve a scrape endpoint). Enables FastAPI and system/process metrics. | +| `OTEL_LOGS_EXPORTER` | — | No | Set to `otlp` to export log records (INFO and above) over OTLP/gRPC alongside console output — see [docs/logging.md](docs/logging.md). | +| `OTEL_EXPORTER_OTLP_ENDPOINT` | `http://localhost:4317` | No | OTLP/gRPC collector endpoint shared by trace, metric, and log export. One of the [standard OpenTelemetry SDK variables](https://opentelemetry.io/docs/specs/otel/configuration/sdk-environment-variables/), which the underlying exporters honor as usual (per-signal endpoints, headers, timeouts, resource attributes, …). | +| `OTEL_EXPORTER_PROMETHEUS_PORT` | `9464` | No | Port of the Prometheus scrape endpoint (effective only with `prometheus` in `OTEL_METRICS_EXPORTER`) | +| **Scripts & Tests** | | | | +| `REMOTE_DIAL_URL` | — | No | URL of the remote DIAL Core, used only by `generate_dial_config` script and e2e/integration tests | +| `REMOTE_DIAL_API_KEY` | — | No | API key of the remote DIAL Core, used only by `generate_dial_config` script and e2e/integration tests | #### Deprecated Environment Variables > [!CAUTION] > These variables still work but will be removed in a future major version. -| Variable | Replacement | Description | -|---------------------------------|-------------------------------------------------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| `PREDEFINED_BASE_PATH` | `PREDEFINED_EXTRA_PATHS` | If set alone, treated as a single extra layer on top of the built-in content | -| `PY_INTERPRETER_CLIENT_TIMEOUT` | `DEFAULT_TOOL_TIMEOUT_SECONDS` or `tool_defaults.timeout_seconds` | When set, still controls the PyInterpreter client timeout (seconds, default `60.0`), but the unified tool-timeout settings are preferred. | -| `LOG_FORMAT` | `DIAL_SDK_TEXT_LOG_FORMAT` or `DIAL_SDK_LOG_FORMAT=json` | When set, still controls the `text` output format (and wins over the replacements); a warning is emitted at startup. See [docs/logging.md](docs/logging.md). | -| `LOG_DATE_FORMAT` | — | Still honored alongside `LOG_FORMAT`; going forward the timestamp format is fixed to `%Y-%m-%d %H:%M:%S` (the previous default). | -| `OTEL_PYTHON_LOG_CORRELATION` | — *(automatic)* | Deprecated by aidial-sdk; a warning is emitted at startup. Trace fields are stamped onto log records whenever tracing is enabled, so the switch is redundant — and setting it installs OTel's legacy root-logger format, which double-logs SDK records and bypasses this service's console formatting. See [docs/logging.md](docs/logging.md). | +| Variable | Replacement | Description | +|---------------------------------|--------------------------------------------------------------------|------------------------------------------------------------------------------------------------------------------------------| +| `PREDEFINED_BASE_PATH` | `PREDEFINED_EXTRA_PATHS` | If set alone, treated as a single extra layer on top of the built-in content | +| `PY_INTERPRETER_CLIENT_TIMEOUT` | `DEFAULT_TOOL_TIMEOUT_SECONDS` or `tool_defaults.timeout_seconds` | When set, still controls the PyInterpreter client timeout (seconds, default `60.0`), but the unified tool-timeout settings are preferred. | +| `LOG_FORMAT` | `DIAL_SDK_TEXT_LOG_FORMAT` or `DIAL_SDK_LOG_FORMAT=json` | When set, still controls the `text` output format (and wins over the replacements); a warning is emitted at startup. See [docs/logging.md](docs/logging.md). | +| `LOG_DATE_FORMAT` | — | Still honored alongside `LOG_FORMAT`; going forward the timestamp format is fixed to `%Y-%m-%d %H:%M:%S` (the previous default). | +| `OTEL_PYTHON_LOG_CORRELATION` | — *(automatic)* | Deprecated by aidial-sdk; a warning is emitted at startup. Trace fields are stamped onto log records whenever tracing is enabled, so the switch is redundant — and setting it installs OTel's legacy root-logger format, which double-logs SDK records and bypasses this service's console formatting. See [docs/logging.md](docs/logging.md). | **Notes:** @@ -288,8 +278,7 @@ content into the logs. `LOG_PAYLOADS=true` is the single, explicit exception: it re-enables the payload-bearing DEBUG records (message context, tool-call arguments, raw responses), each field truncated to `LOG_PAYLOADS_MAX_LENGTH`, and lifts the INFO cap on the wire-level third-party loggers (`openai`, `httpx`, `httpcore`). Every payload record is prefixed -with a `[payload]` marker so these lines can be found — or excluded — with a single filter. Forwarded header **values** -are +with a `[payload]` marker so these lines can be found — or excluded — with a single filter. Forwarded header **values** are never logged, even with the switch on. The switch is additive to the level — content appears only when `QUICKAPP_LOG_LEVEL=DEBUG` **and** `LOG_PAYLOADS=true`. @@ -431,8 +420,7 @@ never logged, even with the switch on. The switch is additive to the level — c - Notes: - If you want to run Quick Apps in Docker instead of on the host, update - [application-schemas.json](docker_compose_files/core/configuration/application-schemas.json) and change the - Quick + [application-schemas.json](docker_compose_files/core/configuration/application-schemas.json) and change the Quick Apps host from `host.docker.internal:5000` to `quick-apps:5000`. - When running via docker-compose the compose files set service hostnames (for example DIAL URL inside containers is http://core:8080). Those container-internal hostnames are not valid from your host machine — use @@ -513,8 +501,7 @@ never logged, even with the switch on. The switch is additive to the level — c ## E2E & Integration tests -Refer to [Testing Guide](./src/tests/integration_tests/README.md) for detailed instructions on setting up and running -tests. +Refer to [Testing Guide](./src/tests/integration_tests/README.md) for detailed instructions on setting up and running tests. ## More diff --git a/src/quickapp/config/application.py b/src/quickapp/config/application.py index 519d80d5..94a525be 100644 --- a/src/quickapp/config/application.py +++ b/src/quickapp/config/application.py @@ -22,9 +22,9 @@ from quickapp.config.skill import SkillConfig from quickapp.config.starters import ConversationStartersConfig from quickapp.config.timestamp import TimestampConfig, ToolCallTimestampConfig +from quickapp.config.tool_discovery import ToolDiscoveryConfig from quickapp.config.toolsets.toolset import ToolSet from quickapp.config.web_fetch import WebFetchConfig -from quickapp.tool_discovery import ToolDiscoveryConfig logger = logging.getLogger(__name__) diff --git a/src/quickapp/tool_discovery/_tool_discovery_config.py b/src/quickapp/config/tool_discovery.py similarity index 100% rename from src/quickapp/tool_discovery/_tool_discovery_config.py rename to src/quickapp/config/tool_discovery.py diff --git a/src/quickapp/core/agent/agent_module.py b/src/quickapp/core/agent/agent_module.py index ad3a022e..7114c5b4 100644 --- a/src/quickapp/core/agent/agent_module.py +++ b/src/quickapp/core/agent/agent_module.py @@ -63,7 +63,7 @@ OrchestratorDeploymentCacheService, ) from quickapp.core.application._request_context import _RequestContext -from quickapp.tool_discovery._deferred_tools_context import DeferredToolsContext +from quickapp.shared.deferred_tools import DeferredToolsContext DEFAULT_QUERY_PARAM = ConfigurableSchemaSimpleType( type=JsonTypeEnum.string, @@ -105,7 +105,6 @@ def configure(self, binder: Binder) -> None: ) binder.bind(AssistantInvoker, to=AssistantInvoker, scope=NoScope) binder.bind(_ChatCompletionConfigBuilder, to=_ChatCompletionConfigBuilder, scope=NoScope) - binder.bind(DeferredToolsContext, to=DeferredToolsContext, scope=request_scope) binder.bind(LazyLoadedToolsHolder, to=LazyLoadedToolsHolder, scope=request_scope) binder.bind(ChatStreamSinkFactory, to=ChatStreamSinkFactory, scope=NoScope) binder.bind(ChatCompletionStreamHandler, to=ChatCompletionStreamHandler, scope=NoScope) diff --git a/src/quickapp/mcp_tooling/_mcp_tool_initializer.py b/src/quickapp/mcp_tooling/_mcp_tool_initializer.py index 97c9391c..39dcf086 100644 --- a/src/quickapp/mcp_tooling/_mcp_tool_initializer.py +++ b/src/quickapp/mcp_tooling/_mcp_tool_initializer.py @@ -33,10 +33,7 @@ from quickapp.mcp_tooling._mcp_eager_resource import MCPEagerTextResource from quickapp.mcp_tooling._mcp_resource_meta import MCPResourceMeta from quickapp.mcp_tooling._mcp_server_capabilities import MCPServerCapabilities -from quickapp.tool_discovery._deferred_tools_context import ( - DeferredToolsContext, - is_toolset_deferred, -) +from quickapp.shared.deferred_tools import DeferredToolsContext, is_toolset_deferred from ._di_types import DialToolsetCacheService from ._mcp_tool import _MCPTool @@ -296,12 +293,11 @@ async def _load_tools( discovery_cfg = self.__app_config.orchestrator.tool_discovery if is_toolset_deferred(toolset_info, discovery_cfg, len(created_tools)): self.__deferred_context.register_staged_tools(created_tools) - if logger.isEnabledFor(logging.DEBUG): - logger.debug( - "Deferred %d tools from MCP toolset '%s' into DeferredToolsContext", - len(created_tools), - resolve_localized(resolved_toolset.name), - ) + logger.debug( + "Deferred %d tools from MCP toolset '%s' into DeferredToolsContext", + len(created_tools), + resolve_localized(resolved_toolset.name), + ) self.__mcp_context.extend_tools(created_tools) async def _load_resources( diff --git a/src/quickapp/rest_api_tooling/rest_api_tooling_module.py b/src/quickapp/rest_api_tooling/rest_api_tooling_module.py index 5c72593b..f73ea68c 100644 --- a/src/quickapp/rest_api_tooling/rest_api_tooling_module.py +++ b/src/quickapp/rest_api_tooling/rest_api_tooling_module.py @@ -10,10 +10,7 @@ from quickapp.config.application import ApplicationConfig from quickapp.config.tools.rest_api import RestApiTool from quickapp.config.toolsets.rest_api import RestApiToolSet -from quickapp.tool_discovery._deferred_tools_context import ( - DeferredToolsContext, - is_toolset_deferred, -) +from quickapp.shared.deferred_tools import DeferredToolsContext, is_toolset_deferred from ._request_detail_builder import _RequestDetailsBuilder from ._rest_api_stage_wrapper import _RestApiStageWrapper @@ -47,12 +44,11 @@ def __provide_rest_api_tools( discovery_cfg = app_config.orchestrator.tool_discovery if is_toolset_deferred(toolset_info, discovery_cfg, len(tools)): deferred_context.register_staged_tools(tools) - if logger.isEnabledFor(logging.DEBUG): - logger.debug( - "Deferred %d tools from REST toolset '%s' into DeferredToolsContext", - len(tools), - toolset_stage_name, - ) + logger.debug( + "Deferred %d tools from REST toolset '%s' into DeferredToolsContext", + len(tools), + toolset_stage_name, + ) result.extend(tools) return result diff --git a/src/quickapp/shared/__init__.py b/src/quickapp/shared/__init__.py index cdba8c85..da543296 100644 --- a/src/quickapp/shared/__init__.py +++ b/src/quickapp/shared/__init__.py @@ -1,6 +1,7 @@ from injector import Module from quickapp.shared.config_resolvers.config_resolvers_module import ConfigResolversModule +from quickapp.shared.deferred_tools.deferred_tools_module import DeferredToolsModule from quickapp.shared.external_fetch.external_fetch_module import ExternalFetchModule from quickapp.shared.home_path.home_path_module import HomePathModule @@ -9,6 +10,7 @@ # individually. shared_module: list[Module] = [ ConfigResolversModule(), + DeferredToolsModule(), ExternalFetchModule(), HomePathModule(), ] diff --git a/src/quickapp/shared/deferred_tools/__init__.py b/src/quickapp/shared/deferred_tools/__init__.py new file mode 100644 index 00000000..99fd006b --- /dev/null +++ b/src/quickapp/shared/deferred_tools/__init__.py @@ -0,0 +1,3 @@ +from ._deferred_tools_context import DeferredToolsContext, is_toolset_deferred + +__all__ = ["DeferredToolsContext", "is_toolset_deferred"] diff --git a/src/quickapp/tool_discovery/_deferred_tools_context.py b/src/quickapp/shared/deferred_tools/_deferred_tools_context.py similarity index 86% rename from src/quickapp/tool_discovery/_deferred_tools_context.py rename to src/quickapp/shared/deferred_tools/_deferred_tools_context.py index 2e4474e2..ead0737c 100644 --- a/src/quickapp/tool_discovery/_deferred_tools_context.py +++ b/src/quickapp/shared/deferred_tools/_deferred_tools_context.py @@ -1,10 +1,11 @@ +from typing import Any + from injector import inject from quickapp.common import StagedBaseTool +from quickapp.config.tool_discovery import ToolDiscoveryConfig from quickapp.config.tools.base import BaseOpenAITool from quickapp.config.toolsets.base import BaseToolSet -from quickapp.core.agent.models import OpenAiToolConfigDict -from quickapp.tool_discovery._tool_discovery_config import ToolDiscoveryConfig @inject @@ -13,7 +14,7 @@ class DeferredToolsContext: def __init__(self) -> None: self._catalog: list[dict[str, str]] = [] - self._definitions: dict[str, OpenAiToolConfigDict] = {} + self._definitions: dict[str, dict[str, Any]] = {} def register_staged_tools(self, tools: list[StagedBaseTool]) -> None: entries: list[tuple[BaseOpenAITool, str]] = [ @@ -44,7 +45,7 @@ def deferred_names(self) -> frozenset[str]: def catalog(self) -> list[dict[str, str]]: return list(self._catalog) - def get_definition(self, name: str) -> OpenAiToolConfigDict | None: + def get_definition(self, name: str) -> dict[str, Any] | None: return self._definitions.get(name) diff --git a/src/quickapp/shared/deferred_tools/deferred_tools_module.py b/src/quickapp/shared/deferred_tools/deferred_tools_module.py new file mode 100644 index 00000000..a8c783aa --- /dev/null +++ b/src/quickapp/shared/deferred_tools/deferred_tools_module.py @@ -0,0 +1,16 @@ +from fastapi_injector import request_scope +from injector import Binder, Module + +from quickapp.shared.deferred_tools._deferred_tools_context import DeferredToolsContext + + +class DeferredToolsModule(Module): + """DI binding for the shared deferred-tools catalog. + + Request-scoped holder shared between the REST API, MCP and core agent modules + so toolsets withheld from the initial LLM payload can be registered and later + surfaced via the tool_search meta-tool. + """ + + def configure(self, binder: Binder) -> None: + binder.bind(DeferredToolsContext, to=DeferredToolsContext, scope=request_scope) diff --git a/src/quickapp/tool_discovery/__init__.py b/src/quickapp/tool_discovery/__init__.py index 02859f96..e69de29b 100644 --- a/src/quickapp/tool_discovery/__init__.py +++ b/src/quickapp/tool_discovery/__init__.py @@ -1,3 +0,0 @@ -from quickapp.tool_discovery._tool_discovery_config import ToolDiscoveryConfig - -__all__ = ["ToolDiscoveryConfig"] diff --git a/src/quickapp/tool_discovery/_anonymous_agent.py b/src/quickapp/tool_discovery/_anonymous_agent.py index b9ba622a..adcc5c5c 100644 --- a/src/quickapp/tool_discovery/_anonymous_agent.py +++ b/src/quickapp/tool_discovery/_anonymous_agent.py @@ -41,7 +41,12 @@ async def route(self, query: str, catalog: list[dict[str, str]]) -> list[str]: return [] discovery = self.__config.orchestrator.tool_discovery - assert discovery is not None # only reachable when tool_discovery is enabled + if discovery is None: + logger.warning( + "Anonymous agent routing call invoked while tool_discovery is disabled — " + "returning no matches" + ) + return [] service_model = ( discovery.service_model or self.__config.orchestrator.deployment.deployment_id ) diff --git a/src/quickapp/tool_discovery/_tool_search_tool.py b/src/quickapp/tool_discovery/_tool_search_tool.py index 4bd42ba2..b356b65e 100644 --- a/src/quickapp/tool_discovery/_tool_search_tool.py +++ b/src/quickapp/tool_discovery/_tool_search_tool.py @@ -11,8 +11,8 @@ from quickapp.config.application import StageDisplayLevel from quickapp.config.tools.internal import InternalTool from quickapp.core.agent.lazy_loaded_tools_holder import LazyLoadedToolsHolder +from quickapp.shared.deferred_tools import DeferredToolsContext from quickapp.tool_discovery._anonymous_agent import _AnonymousAgent -from quickapp.tool_discovery._deferred_tools_context import DeferredToolsContext from quickapp.tool_discovery._tool_search_stage_wrapper import _ToolSearchStageWrapper logger = logging.getLogger(__name__) From 6ad122c3a08d02dcc371c7fc7758370877409bd7 Mon Sep 17 00:00:00 2001 From: Oleksandr Gubarets Date: Mon, 14 Sep 2026 19:23:48 +0300 Subject: [PATCH 07/10] code review fix. --- CONFIGURATION.md | 14 +- README.md | 4 +- docs/designs/dynamic_tool_discovery.md | 175 +++++++++++------- docs/generated-app-schema.json | 4 +- docs/generated-internal-tools.json | 13 ++ src/quickapp/config/tool_discovery.py | 8 +- src/quickapp/config/tools/base.py | 18 +- src/quickapp/core/agent/agent_module.py | 17 +- src/quickapp/core/agent/models.py | 4 +- .../internal_tooling_module.py | 16 +- .../deferred_tools/_deferred_tools_context.py | 26 ++- .../tool_discovery/tool_discovery_module.py | 2 +- src/scripts/dump_internal_tools.py | 2 + .../integration_tests/test_runner/config.py | 3 - .../test_runner/test_mcp_tool_deferred.json | 11 -- .../test_chat_completion_config_builder.py | 64 +++++++ .../test_internal_tooling_module.py | 87 +++++++++ .../tool_discovery_tests/__init__.py | 0 .../test_anonymous_agent.py | 125 +++++++++++++ .../test_deferred_tools_context.py | 170 +++++++++++++++++ 20 files changed, 635 insertions(+), 128 deletions(-) delete mode 100644 src/tests/integration_tests/test_runner/test_mcp_tool_deferred.json create mode 100644 src/tests/unit_tests/internal_tooling_tests/test_internal_tooling_module.py create mode 100644 src/tests/unit_tests/tool_discovery_tests/__init__.py create mode 100644 src/tests/unit_tests/tool_discovery_tests/test_anonymous_agent.py create mode 100644 src/tests/unit_tests/tool_discovery_tests/test_deferred_tools_context.py diff --git a/CONFIGURATION.md b/CONFIGURATION.md index 5f6100bb..73cb8d98 100644 --- a/CONFIGURATION.md +++ b/CONFIGURATION.md @@ -336,12 +336,13 @@ Custom system prompt: `[Preview]` Requires `ENABLE_PREVIEW_FEATURES=true`. When enabled, toolsets withheld from the initial LLM payload (see the per-toolset `deferred` field in [Tool sets configuration](#tool-sets-configuration)) are surfaced on demand -via a `tool_search` meta-tool, which routes the query to the matching tool schemas through an isolated LLM call. +via the `internal_tool_search` meta-tool (referred to as "tool search" below), which routes the query to the matching +tool schemas through an isolated LLM call. | Field | Required | Type | Description | Available Values | Default Value | |------------------------|----------|---------|-------------------------------------------------------------------------------------------------------------------------------------------------------------|-------------------|---------------| -| enabled | No | Boolean | Enable dynamic tool discovery. When `true`, toolsets with `deferred: true` are withheld from the initial LLM payload and surfaced via the `tool_search` meta-tool. | - | `false` | -| service_model | No | String | DIAL deployment used for the anonymous routing call inside `tool_search`. Falls back to the orchestrator's own deployment when omitted. | - | - | +| enabled | No | Boolean | Enable dynamic tool discovery. When `true`, toolsets with `deferred: true` are withheld from the initial LLM payload and surfaced via the `internal_tool_search` meta-tool. | - | `false` | +| service_model | No | String | DIAL deployment used for the anonymous routing call inside `internal_tool_search`. Falls back to the orchestrator's own deployment when omitted. | - | - | | min_tools_for_deferral | No | Integer | Minimum number of tools in a toolset for deferral to apply. Toolsets smaller than this threshold are promoted to eager loading even when `deferred: true`. Deployment-wide default set by `MIN_TOOLS_FOR_DEFERRAL`. | - | `5` |
@@ -504,10 +505,11 @@ SSRF envelope, deployment dispatch table, error messages and agent retry behavio ### Tool sets configuration -Every toolset type also accepts a `deferred` field (Boolean, default `true`): `[Preview]` when true or unset, and +Most toolset types also accept a `deferred` field (Boolean, default `true`): `[Preview]` when true or unset, and [Tool discovery configuration](#tool-discovery-configuration) is enabled, the toolset's tool schemas are withheld -from the initial LLM payload and discovered on demand via the `tool_search` meta-tool. Set to `false` to keep a -specific toolset always eager. +from the initial LLM payload and discovered on demand via the `internal_tool_search` meta-tool. Set to `false` to +keep a specific toolset always eager. Currently honored by REST API, MCP, and Internal toolsets; DIAL deployment and +DIAL app toolsets accept the field but do not yet act on it. #### RestApiToolSet Configuration diff --git a/README.md b/README.md index ffaff8ea..5b9e5020 100644 --- a/README.md +++ b/README.md @@ -93,7 +93,7 @@ See [Config-Driven Hooks design doc](docs/designs/config_driven_hooks.md) for th ### Dynamic Tool Discovery `[Preview]` -Dynamic tool discovery defers large toolsets from the initial LLM payload and surfaces them on demand via a `tool_search` meta-tool. The orchestrator calls `tool_search` with a natural-language query when it needs a tool it hasn't seen yet; a lightweight anonymous LLM routing call selects the relevant tool schemas and injects them into the next iteration. +Dynamic tool discovery defers large toolsets from the initial LLM payload and surfaces them on demand via the `internal_tool_search` meta-tool (referred to as "tool search" below). The orchestrator calls `internal_tool_search` with a natural-language query when it needs a tool it hasn't seen yet; a lightweight anonymous LLM routing call selects the relevant tool schemas and injects them into the next iteration. Enable with `ENABLE_PREVIEW_FEATURES=true`, then add `orchestrator.tool_discovery` to the app manifest: @@ -136,7 +136,7 @@ Key fields: | Field | Default | Description | |---|---|---| | `orchestrator.tool_discovery.enabled` | `false` | Activates dynamic discovery for this app. Must be `true` for deferral to take effect. | -| `orchestrator.tool_discovery.service_model` | — | DIAL deployment used for the anonymous routing call inside `tool_search`. Falls back to the orchestrator's own deployment when omitted. | +| `orchestrator.tool_discovery.service_model` | — | DIAL deployment used for the anonymous routing call inside `internal_tool_search`. Falls back to the orchestrator's own deployment when omitted. | | `orchestrator.tool_discovery.min_tools_for_deferral` | `5` | Minimum number of tools in a toolset for deferral to apply. Toolsets smaller than this threshold are promoted to eager loading even when `deferred: true`. | | `.deferred` | `true` | Per-toolset opt-out. Set to `false` to force a specific toolset into the initial payload regardless of `tool_discovery.enabled`. | diff --git a/docs/designs/dynamic_tool_discovery.md b/docs/designs/dynamic_tool_discovery.md index 3edd81f3..4eb7d034 100644 --- a/docs/designs/dynamic_tool_discovery.md +++ b/docs/designs/dynamic_tool_discovery.md @@ -278,37 +278,57 @@ registry, connect only when the model requests a server's capabilities. Requires --- -### Option 6 — `DeferredRequestContext` + anonymous agent search + orchestrator injection (Recommended) +### Option 6 — `DeferredToolsContext` + anonymous agent search + lazy schema injection (Recommended, chosen) **Mechanism:** -At request time, MCP and REST initializers split their output between the normal `RequestContext` -(eager tools) and a new `DeferredRequestContext` (deferred tools). A single `tool_search` meta-tool -is injected; when triggered it fires an isolated "anonymous agent" chat completion that consults -only the deferred catalog. The orchestrator intercepts the result and injects full schemas natively -into the next round. No Anthropic-specific API features are required. +At request time, toolset initializers (MCP, REST, and internal) split their output between the +normal `list[StagedBaseTool]` (eager tools) and a shared `DeferredToolsContext` (deferred tools). +A single `tool_search` meta-tool is injected; when triggered it fires an isolated "anonymous agent" +chat completion that consults only the deferred catalog, then stores the matched tools' full +schemas in a request-scoped `LazyLoadedToolsHolder`, which `_ChatCompletionConfigBuilder` merges +into `payload["tools"]` on the next build. No Anthropic-specific API features are required, and +**no orchestrator changes were needed** — the mechanism is entirely local to the `tool_search` +tool's own execution and the request-scoped holder it writes to. + +> **As built:** the sections below describe the mechanism as originally proposed. Where the +> landed implementation differs, an **As built** note calls it out. See the +> [Changes required](#changes-required) table for the final shape. #### Step 1 — Request initialisation -When a new chat completion request arrives, MCP and REST toolset initializers run as today and -build full `OpenAiToolConfig` definitions. They then apply the deferral decision: +When a new chat completion request arrives, toolset initializers run as today and build full +`OpenAiToolConfig` definitions. They then apply the deferral decision per toolset (the real +predicate, `is_toolset_deferred` — note the tri-state `deferred` field, see +[Deferral threshold](#deferral-threshold)): ``` -if toolset.deferred and len(tools) >= discovery.min_tools_for_deferral: - → push {name, description} entries + full definitions to DeferredRequestContext +if is_toolset_deferred(toolset, discovery_cfg, len(tools)): + → register {name, description} entries + full definitions with DeferredToolsContext + → the same tools are still appended to the module's own list[StagedBaseTool] + (they exist as StagedBaseTool instances throughout — deferral only withholds + their *schema* from the initial LLM payload, in AgentModule.provide_openai_tools) else: - → push full definitions to RequestContext (eager, as today) + → nothing extra happens; the tool is eager as today ``` -`DeferredRequestContext` holds two structures per toolset: +**As built:** REST API and MCP toolsets support deferral (`rest_api_tooling_module.py`, +`_mcp_tool_initializer.py`). Internal toolsets also support it. `dial-deployment` and `dial-app` +toolsets do not yet call `is_toolset_deferred` — setting `deferred` on those has no effect today +(tracked as a follow-up). + +`DeferredToolsContext` holds two structures across all deferred toolsets in the request (not +one instance per toolset — a single request-scoped context aggregates all of them): - **catalog**: `list[{name, description}]` — compact, never forwarded to the main LLM -- **definitions**: `dict[str, OpenAiToolConfigDict]` — full schemas, used by the lazy-initializer +- **definitions**: `dict[str, OpenAiToolConfigDict]` — full schemas (transformed the same way + as eager tools — const params stripped, `enrich_openai_tool_schema` applied), looked up by + `_ToolSearchTool` when the anonymous agent returns matches -Toolsets with `deferred: false`, or that fall below the tool-count threshold, go straight into -`RequestContext` unchanged. +Toolsets with `deferred: false`, or that fall below the tool-count threshold, stay eager — +`AgentModule.provide_openai_tools` includes them in `payload["tools"]` directly. #### Step 2 — Main orchestrator call -`payload["tools"]` is built from `RequestContext` (eager tools) plus the single `tool_search` +`payload["tools"]` is built from the eager `list[StagedBaseTool]` plus the single `tool_search` meta-tool injected by `AgentModule`. Deferred tools are **absent**. ``` @@ -329,7 +349,7 @@ history and no system prompt from the main request: model: service_model (config: defaults to orchestrator deployment) system: "You are a tool routing assistant. Given a user query, return the names of the tools from the following catalog that are most relevant. - Catalog: [{name, description}, ...]" ← injected from DeferredRequestContext + Catalog: [{name, description}, ...]" ← injected from DeferredToolsContext messages:[{"role": "user", "content": }] tools: none ``` @@ -340,36 +360,40 @@ history and no application system prompt, its token cost is bounded by the catal **Result returned to the main LLM:** `[{name, description}]` of matched tools, confirming what is now available to load. -#### Step 4 — Lazy-initializer builds OpenAI definitions +#### Step 4 — `_ToolSearchTool` builds OpenAI definitions -The `tool_search` handler passes the matched tool names to an internal **lazy-initializer**, -which looks up each name in `DeferredRequestContext.definitions` and returns the corresponding -`OpenAiToolConfigDict` objects. No LLM call is made at this step. +Rather than a separate lazy-initializer module, `_ToolSearchTool` (a `StagedBaseTool`) itself +looks up each matched name in `DeferredToolsContext.get_definition(...)` and passes the +corresponding `OpenAiToolConfigDict` objects to `LazyLoadedToolsHolder.add(...)`. No LLM call is +made at this step. -#### Step 5 — Orchestrator injection (Path A) +#### Step 5 — Lazy schema injection (no orchestrator involvement) -The orchestrator intercepts the `tool_search` result and: -1. Reads the list of matched tool names from the result. -2. Calls the lazy-initializer to retrieve their full `OpenAiToolConfigDict`s. -3. Accumulates them in `_lazy_loaded_tools: dict[str, OpenAiToolConfigDict]` (persists across - iterations within the turn). -4. On the **next** `_ChatCompletionConfigBuilder.build()` call, `_lazy_loaded_tools` is merged - into `payload["tools"]`. +**As built, this differs from the original proposal:** there is no orchestrator-side interception +or `_lazy_loaded_tools` state inside `orchestrator.py`. Instead: +1. `LazyLoadedToolsHolder` is a request-scoped holder (`core/agent/lazy_loaded_tools_holder.py`) + that `_ToolSearchTool` writes into directly during its own tool-call execution. +2. On every subsequent `_ChatCompletionConfigBuilder.build()` call within the same request, + the builder reads `lazy_loaded_tools_holder.get_all()` and merges those definitions into + `payload["tools"]`, de-duplicated by function name against the eager tools. +3. The orchestrator loop is completely unaware of tool discovery — it just re-builds the payload + each iteration as it always did, and the holder's contents are picked up automatically. The main LLM now sees the discovered tools natively alongside `tool_search` and the eager tools, and calls them with proper schema-based argument generation. -Optionally, discovered tool names are serialised into `custom_content.state["lazy_loaded_tools"]` -so they survive across conversation turns, avoiding rediscovery on the next user message. +Cross-turn persistence (serialising discovered tool names into +`custom_content.state["lazy_loaded_tools"]` so rediscovery is skipped on the next user message) +was **not implemented** — see [Out of Scope](#out-of-scope-mvp). #### Flow diagram ``` New request arrives │ - ├─ MCP initializer: len(tools) >= threshold → DeferredRequestContext - │ (catalog + definitions) - ├─ REST initializer: len(tools) < threshold → RequestContext (eager) + ├─ MCP/REST/internal initializers: len(tools) >= threshold → DeferredToolsContext + │ (catalog + definitions) + │ len(tools) < threshold → eager list[StagedBaseTool] │ ▼ Orchestrator — iteration 1 @@ -378,16 +402,17 @@ Orchestrator — iteration 1 │ └─ Main LLM calls tool_search("I need to query Salesforce contacts") │ - └─ AnonymousAgent (isolated chat completion): + └─ _ToolSearchTool → _AnonymousAgent (isolated chat completion): model = service_model - system = routing prompt + catalog from DeferredRequestContext + system = routing prompt + catalog from DeferredToolsContext message = "I need to query Salesforce contacts" → ["sf_query_contacts", "sf_list_contacts"] - Lazy-initializer: names → full OpenAiToolConfigDicts - Orchestrator stores in _lazy_loaded_tools + _ToolSearchTool: names → full OpenAiToolConfigDicts, written into + LazyLoadedToolsHolder (request-scoped; orchestrator is not involved) Result returned to main LLM: [{name, description}, ...] Orchestrator — iteration 2 + _ChatCompletionConfigBuilder reads LazyLoadedToolsHolder.get_all() and merges: payload["tools"] = [tool_search, ...eager tools, sf_query_contacts ←injected, sf_list_contacts ←injected] @@ -397,14 +422,22 @@ Orchestrator — iteration 2 #### Deferral threshold -A toolset with `deferred: true` is only placed into `DeferredRequestContext` if its tool count -meets the minimum. Below the threshold it is loaded eagerly — no discovery overhead: +A toolset is only placed into `DeferredToolsContext` if `is_toolset_deferred` returns true. +`deferred` is a tri-state field (`bool | None`, default `None`/unset) — unset **or** `true` +both defer; only an explicit `false` forces eager regardless of count: -``` -deferred_effective = toolset.deferred and len(tools) >= discovery.min_tools_for_deferral +```python +def is_toolset_deferred(toolset, discovery_cfg, tool_count) -> bool: + return ( + toolset.deferred is not False + and discovery_cfg is not None + and discovery_cfg.enabled + and tool_count >= discovery_cfg.min_tools_for_deferral + ) ``` -`deferred: false` always means eager, regardless of count. +Below the threshold — or when `tool_discovery` is disabled/unset entirely — a toolset is loaded +eagerly, no discovery overhead. #### Configuration @@ -433,27 +466,26 @@ deferred_effective = toolset.deferred and len(tools) >= discovery.min_tools_for_ } ``` -- `deferred: true` (or unset/`null`) opts the toolset into `DeferredRequestContext`. Default: `true` (omitting the field defers by default). +- `deferred: true` (or unset/`null`) opts the toolset into `DeferredToolsContext`. Default: `true` (omitting the field defers by default). - `service_model` names the DIAL deployment used for the `AnonymousAgent` chat completion. When omitted, falls back to the orchestrator's own deployment. - `min_tools_for_deferral` is the tool-count guard below which a deferred toolset is silently promoted to eager. Default: `5`. - Non-deferred and below-threshold toolsets populate `RequestContext` immediately, as today. -#### Changes required +#### Changes required (as built) | Area | Change | |---|---| -| `BaseToolSet` | Add `deferred: bool = False` field | -| `OrchestratorConfig` | Add `tool_discovery: ToolDiscoveryConfig` sub-config (`enabled`, `service_model`, `min_tools_for_deferral`) | -| `DeferredRequestContext` | New DI-scoped object: holds per-toolset `catalog` list and `definitions` dict; populated by initializers during request setup | -| MCP & REST initializer modules | After building tool definitions, evaluate `deferred_effective`; route to `DeferredRequestContext` or `RequestContext` accordingly | -| `AnonymousAgent` | New module: fires a single isolated `chat.completions.create` call (no history, no app system prompt); takes `service_model`, a system prompt with the catalog, and a user query; returns matched tool names | -| `tool_search` (`StagedBaseTool`) | New internal tool injected via `AgentModule @multiprovider`; calls `AnonymousAgent`, passes results to the lazy-initializer, returns `[{name, description}]` to the main LLM | -| Lazy-initializer | Thin helper: given a list of tool names, looks up `DeferredRequestContext.definitions` and returns `list[OpenAiToolConfigDict]` | -| `orchestrator.py` | After each iteration, detect `tool_search` results; call lazy-initializer; accumulate in `_lazy_loaded_tools`; pass to `_ChatCompletionConfigBuilder` on next call | -| `_ChatCompletionConfigBuilder` | Accept `lazy_tool_dicts: list[OpenAiToolConfigDict]`; merge into `payload["tools"]` | -| State serialisation (optional) | Persist `_lazy_loaded_tools` names in `custom_content.state["lazy_loaded_tools"]` for cross-turn reuse | +| `BaseToolSet` (`config/toolsets/base.py`) | Add `deferred: bool \| None` field (tri-state, default `None`/unset — unset behaves as deferred, see [Deferral threshold](#deferral-threshold)) | +| `config/tool_discovery.py` | New `ToolDiscoveryConfig` (`enabled`, `service_model`, `min_tools_for_deferral`), referenced by `OrchestratorConfig.tool_discovery` | +| `shared/deferred_tools/` (`DeferredToolsContext`, `is_toolset_deferred`) | Request-scoped shared object aggregating `catalog`/`definitions` across all deferred toolsets in the request; `is_toolset_deferred` is the pure threshold predicate. Bound via its own `DeferredToolsModule`, spliced into `shared_module` | +| REST, MCP, internal toolset modules | After building each toolset's tools, evaluate `is_toolset_deferred`; register with `DeferredToolsContext` or leave in the eager `list[StagedBaseTool]` accordingly. **`dial-deployment`/`dial-app` toolsets do not yet do this** — follow-up | +| `tool_discovery/_anonymous_agent.py` (`_AnonymousAgent`) | Fires a single isolated `chat.completions.create` call (no history, no app system prompt); takes the catalog and a user query; returns matched tool names | +| `tool_discovery/_tool_search_tool.py` (`_ToolSearchTool`) | Internal `tool_search` (registered name: `internal_tool_search`) tool injected via `ToolDiscoveryModule`'s own `@multiprovider` (preview-gated); calls `_AnonymousAgent`, looks up matched names in `DeferredToolsContext`, writes results into `LazyLoadedToolsHolder`, returns `[{name, description}]` to the main LLM | +| `core/agent/lazy_loaded_tools_holder.py` (`LazyLoadedToolsHolder`) | Request-scoped holder of discovered `OpenAiToolConfigDict`s — replaces the originally-proposed lazy-initializer + orchestrator-side `_lazy_loaded_tools` state | +| `_chat_completion_config_builder.py` | Reads `LazyLoadedToolsHolder.get_all()` on every build and merges into `payload["tools"]`, de-duplicated against eager tool names — **`orchestrator.py` itself was not changed** | +| Cross-turn state persistence | **Not implemented** — see [Out of Scope](#out-of-scope-mvp) | **Round-trip cost:** +1 turn before first native tool use (search + inject → tool call). The anonymous agent call happens inside the `tool_search` tool execution, not as a separate @@ -466,7 +498,7 @@ with ~20-token descriptions each, this is ~4 K tokens regardless of how long the **Pros:** - Fully model-agnostic: works with any DIAL deployment as orchestrator. -- `DeferredRequestContext` cleanly separates eager and deferred tool state at the DI layer — +- `DeferredToolsContext` cleanly separates eager and deferred tool state at the DI layer — no orchestrator logic needed to decide what to defer. - Anonymous agent isolates search cost from main conversation tokens; scales with catalog size, not conversation length. @@ -481,8 +513,8 @@ with ~20-token descriptions each, this is ~4 K tokens regardless of how long the **Cons:** - +1 orchestrator iteration before first native use of a deferred tool. - `AnonymousAgent` introduces a new code path for isolated completions. -- `_lazy_loaded_tools` state in the orchestrator; optional cross-turn persistence requires - state serialisation. +- Discovered tools only live for the current turn (`LazyLoadedToolsHolder` is request-scoped); + cross-turn persistence would require state serialisation (not implemented, see Out of Scope). - Search quality depends on service model and catalog description quality. --- @@ -492,7 +524,7 @@ with ~20-token descriptions each, this is ~4 K tokens regardless of how long the | | Option 1 | Option 2 | Option 3 | Option 4 | Option 5 | **Option 6** | |---|---|---|---|---|---|---| | Model-agnostic | Yes | Yes | Yes | No (Anthropic 4.5+) | Yes | **Yes** | -| Orchestrator changes | None | None | Significant | Medium | Medium | **Medium** | +| Orchestrator changes | None | None | Significant | Medium | Medium | **None** (as built) | | Native schema on discovered tool call | No | No | Yes | Yes | Depends | **Yes** | | Full name-space visible upfront | No | Yes | No | No | No | **No** | | Prompt cache preserved | — | — | — | Yes (by design) | — | **Partial** ¹ | @@ -526,23 +558,22 @@ top of Option 6 incrementally. 1. **Granularity:** `deferred` on `BaseToolSet` (per-toolset) or also settable per-tool within a toolset? Per-toolset covers MCP servers and REST API groups cleanly; per-tool is needed - only for mixed toolsets where some tools are always-on. + only for mixed toolsets where some tools are always-on. **Still open** — per-toolset is what + shipped; per-tool granularity remains a possible future extension. -2. **`service_model` default:** Should it fall back to the orchestrator deployment, or require - explicit configuration? Defaulting to the orchestrator deployment is the simplest path but - misses the cost-saving opportunity of routing to a cheaper model. +2. ~~**`service_model` default**~~ — **Resolved, as built:** falls back to the orchestrator's + own deployment when omitted (`_AnonymousAgent.route`). -3. **Search implementation in MVP:** Pure LLM routing via `AnonymousAgent` from the start, or - offer a keyword-only fallback that skips the anonymous agent call entirely? Keyword match - has zero latency and no model dependency; LLM routing handles synonyms and fuzzy intent. +3. ~~**Search implementation in MVP**~~ — **Resolved, as built:** pure LLM routing via + `_AnonymousAgent`, no keyword-only fallback. Not currently planned. -4. **Multi-turn persistence:** Serialise `_lazy_loaded_tools` names into - `custom_content.state["lazy_loaded_tools"]` so rediscovery is skipped on subsequent turns, - or always rediscover? Persistence saves round-trips but grows state size. +4. ~~**Multi-turn persistence**~~ — **Resolved, as built: not implemented.** Discovered tools + are always rediscovered each turn; see [Out of Scope](#out-of-scope-mvp). 5. **Always-on threshold:** Should there be a heuristic that automatically promotes a frequently-discovered tool to eager loading (e.g. seen in last N turns), or is that always - explicit config? + explicit config? **Still open** — not implemented; today it's always explicit config + (`deferred: false` per toolset). --- @@ -550,10 +581,12 @@ top of Option 6 incrementally. | Item | Reason | |---|---| -| Embedding-based search in `AnonymousAgent` | Swap-in strategy; `AnonymousAgent` interface is the extension point | +| Embedding-based search in `_AnonymousAgent` | Swap-in strategy; `_AnonymousAgent` interface is the extension point | | Dynamic MCP server connection/disconnection | Significant lifecycle change; Option 5 follow-on | | Per-tool granularity within a toolset | Per-toolset is sufficient for the initial use case | | Automatic threshold based on token count | Tool-count threshold is simpler and good enough for MVP | +| Cross-turn discovery persistence (`custom_content.state["lazy_loaded_tools"]`) | Not implemented — `LazyLoadedToolsHolder` is request-scoped only; discovered tools are rediscovered every turn | +| `dial-deployment` / `dial-app` toolset deferral | REST, MCP, and internal toolsets support `deferred`; deployment/app toolsets do not yet call `is_toolset_deferred` — tracked as a follow-up | --- diff --git a/docs/generated-app-schema.json b/docs/generated-app-schema.json index 6e17496b..2aa2453a 100644 --- a/docs/generated-app-schema.json +++ b/docs/generated-app-schema.json @@ -3494,7 +3494,7 @@ "properties": { "enabled": { "default": false, - "description": "Enable dynamic tool discovery. When true, toolsets with deferred=true are withheld from the initial LLM payload and surfaced via the tool_search meta-tool.", + "description": "Enable dynamic tool discovery. When true, toolsets with deferred=true are withheld from the initial LLM payload and surfaced via the internal_tool_search meta-tool.", "title": "Enabled", "type": "boolean" }, @@ -3508,7 +3508,7 @@ } ], "default": null, - "description": "DIAL deployment used for the anonymous routing call inside tool_search. When omitted, falls back to the orchestrator's own deployment.", + "description": "DIAL deployment used for the anonymous routing call inside internal_tool_search. When omitted, falls back to the orchestrator's own deployment.", "title": "Service Model" }, "min_tools_for_deferral": { diff --git a/docs/generated-internal-tools.json b/docs/generated-internal-tools.json index 383c187c..098d2a66 100644 --- a/docs/generated-internal-tools.json +++ b/docs/generated-internal-tools.json @@ -385,6 +385,19 @@ } } }, + { + "name": "internal_tool_search", + "description": "Search for additional tools available to this assistant. Use this when you need a capability that is not listed in the current tool list. Returns the names and descriptions of matching tools; those tools will be available to call immediately after.", + "properties": { + "query": { + "type": "string", + "description": "A natural-language description of the capability you need." + } + }, + "required": [ + "query" + ] + }, { "name": "internal_web_fetch", "description": "Fetch a resource from an external http(s) URL (e.g. a README, a source file, a documentation page). Without save_path it returns the text inline in a single call — text only: binary content (images, PDFs, archives) is rejected, re-call with a save_path instead. Text larger than the inline cap is returned truncated to its head with a notice stating the total size; the head is often enough, otherwise re-call with a save_path. With save_path it persists the resource (any content type) at that workspace-relative path under the agent home and returns the saved path (+ a short preview for text), so other available tools can process the full content. DIAL file paths (files/...) are not fetched here.", diff --git a/src/quickapp/config/tool_discovery.py b/src/quickapp/config/tool_discovery.py index df43f208..2c39a4b2 100644 --- a/src/quickapp/config/tool_discovery.py +++ b/src/quickapp/config/tool_discovery.py @@ -1,9 +1,11 @@ from pydantic import BaseModel, ConfigDict, Field from pydantic.fields import FieldInfo -from pydantic_settings import BaseSettings +from pydantic_settings import BaseSettings, SettingsConfigDict class ToolDiscoverySettings(BaseSettings): + model_config = SettingsConfigDict() + min_tools_for_deferral: int = Field( default=5, ge=1, @@ -32,12 +34,12 @@ class ToolDiscoveryConfig(BaseModel): enabled: bool = Field( default=False, - description="Enable dynamic tool discovery. When true, toolsets with deferred=true are withheld from the initial LLM payload and surfaced via the tool_search meta-tool.", + description="Enable dynamic tool discovery. When true, toolsets with deferred=true are withheld from the initial LLM payload and surfaced via the internal_tool_search meta-tool.", ) service_model: str | None = Field( default=None, description=( - "DIAL deployment used for the anonymous routing call inside tool_search. " + "DIAL deployment used for the anonymous routing call inside internal_tool_search. " "When omitted, falls back to the orchestrator's own deployment." ), ) diff --git a/src/quickapp/config/tools/base.py b/src/quickapp/config/tools/base.py index e1e46a08..a91fb199 100644 --- a/src/quickapp/config/tools/base.py +++ b/src/quickapp/config/tools/base.py @@ -1,5 +1,6 @@ +import copy from enum import Enum -from typing import Annotated, Any, Generic, Literal, TypeVar, Union +from typing import Annotated, Any, Generic, Literal, TypeAlias, TypeVar, Union from pydantic import BaseModel, Field, field_validator @@ -196,6 +197,21 @@ class OpenAiToolConfig( ] +OpenAiToolConfigDict: TypeAlias = dict[str, Any] + + +def remove_const_schema_params(open_ai_tool: OpenAiToolConfig) -> OpenAiToolConfig: + """Strip const-valued parameters (fixed values hidden from the LLM) from a tool's JSON schema.""" + tool_copy = copy.deepcopy(open_ai_tool) + props = tool_copy.function.parameters.properties + + for prop_name in list(props.keys()): + if issubclass(type(props[prop_name]), JsonSchemaConst): + del props[prop_name] + + return tool_copy + + class BaseTool(BaseModel): attachment: AttachmentConfig = Field( default_factory=AttachmentConfig, description="Configuration for tool attachments." diff --git a/src/quickapp/core/agent/agent_module.py b/src/quickapp/core/agent/agent_module.py index 7114c5b4..40d95a48 100644 --- a/src/quickapp/core/agent/agent_module.py +++ b/src/quickapp/core/agent/agent_module.py @@ -1,5 +1,3 @@ -import copy - from aidial_sdk.chat_completion.request import StaticTool from aidial_sdk.exceptions import InvalidRequestError from fastapi_injector import request_scope @@ -35,10 +33,10 @@ BaseOpenAITool, ConfigurableSchemaArray, ConfigurableSchemaSimpleType, - JsonSchemaConst, JsonSchemaSimpleType, JsonTypeEnum, OpenAiToolConfig, + remove_const_schema_params, ) from quickapp.config.tools.deployment import DialDeploymentTool from quickapp.config.tools.display.paramenter import ( @@ -179,7 +177,7 @@ def provide_openai_tools( open_ai_tool: OpenAiToolConfig = tool.tool_config.open_ai_tool if open_ai_tool.function.name in deferred_names: continue - open_ai_tool = self._remove_const_params(open_ai_tool) + open_ai_tool = remove_const_schema_params(open_ai_tool) if isinstance(tool.tool_config, DialDeploymentTool): open_ai_tool = self._append_default_props(open_ai_tool) open_ai_tool = tool.enrich_openai_tool_schema(open_ai_tool) @@ -227,17 +225,6 @@ def provide_tool_names(self, context: _RequestContext) -> EXTERNAL_TOOL_NAMES: t.function.name for t in context.extra_tools if t.function and t.function.name ) - @staticmethod - def _remove_const_params(open_ai_tool): - tool_copy = copy.deepcopy(open_ai_tool) - props = tool_copy.function.parameters.properties - - for prop_name in list(props.keys()): - if issubclass(type(props[prop_name]), JsonSchemaConst): - del props[prop_name] - - return tool_copy - @staticmethod def _append_default_props(converted_open_ai_tool: OpenAiToolConfig): if "query" not in converted_open_ai_tool.function.parameters.properties: diff --git a/src/quickapp/core/agent/models.py b/src/quickapp/core/agent/models.py index 843e7b92..547ea3b7 100644 --- a/src/quickapp/core/agent/models.py +++ b/src/quickapp/core/agent/models.py @@ -1,6 +1,6 @@ -from typing import Any, TypeAlias +from quickapp.config.tools.base import OpenAiToolConfigDict TOOL_EXECUTION_HISTORY: str = "tool_execution_history" STATE_KEY_ORCHESTRATOR: str = "orchestrator_state" -OpenAiToolConfigDict: TypeAlias = dict[str, Any] +__all__ = ["TOOL_EXECUTION_HISTORY", "STATE_KEY_ORCHESTRATOR", "OpenAiToolConfigDict"] diff --git a/src/quickapp/internal_tooling/internal_tooling_module.py b/src/quickapp/internal_tooling/internal_tooling_module.py index 851ce572..0e9405d2 100644 --- a/src/quickapp/internal_tooling/internal_tooling_module.py +++ b/src/quickapp/internal_tooling/internal_tooling_module.py @@ -5,6 +5,7 @@ from quickapp.common import DIAL_API_KEY, StagedBaseTool from quickapp.common.dial_settings import DialSettings +from quickapp.common.localized_string import resolve_localized from quickapp.common.tool_names import INTERNAL_CODE_EXECUTION_PYTHON_INTERPRETER_TOOL_NAME from quickapp.config.application import ApplicationConfig from quickapp.config.tools.predefined import PredefinedTool @@ -25,6 +26,7 @@ ) from quickapp.internal_tooling.py_interpreter_tooling.handlers.session_manager import SessionManager from quickapp.shared.config_resolvers.tool_timeout_resolver import ToolTimeoutResolver +from quickapp.shared.deferred_tools import DeferredToolsContext, is_toolset_deferred logger = logging.getLogger(__name__) @@ -43,11 +45,13 @@ def _provide_internal_tools( self, app_config: ApplicationConfig, py_builder: AssistedBuilder[_PyInterpreterTool], + deferred_context: DeferredToolsContext, ) -> list[StagedBaseTool]: tools: list[StagedBaseTool] = [] for tool_set in app_config.tool_sets: if isinstance(tool_set, InternalToolSet): + toolset_tools: list[StagedBaseTool] = [] for tool_config in tool_set.tools: if tool_config.enabled: if isinstance(tool_config, PredefinedTool): @@ -58,7 +62,7 @@ def _provide_internal_tools( INTERNAL_CODE_EXECUTION_PYTHON_INTERPRETER_TOOL_NAME ): # TODO: remove this filtering by name, the user may configure any name of the tool. - tools.append( + toolset_tools.append( py_builder.build( tool_config=tool_config, name=tool_config.open_ai_tool.function.name, @@ -66,6 +70,16 @@ def _provide_internal_tools( ) ) + discovery_cfg = app_config.orchestrator.tool_discovery + if is_toolset_deferred(tool_set, discovery_cfg, len(toolset_tools)): + deferred_context.register_staged_tools(toolset_tools) + logger.debug( + "Deferred %d tools from internal toolset '%s' into DeferredToolsContext", + len(toolset_tools), + resolve_localized(tool_set.name), + ) + tools.extend(toolset_tools) + return tools @singleton diff --git a/src/quickapp/shared/deferred_tools/_deferred_tools_context.py b/src/quickapp/shared/deferred_tools/_deferred_tools_context.py index ead0737c..96c52d1a 100644 --- a/src/quickapp/shared/deferred_tools/_deferred_tools_context.py +++ b/src/quickapp/shared/deferred_tools/_deferred_tools_context.py @@ -1,10 +1,12 @@ -from typing import Any - from injector import inject from quickapp.common import StagedBaseTool from quickapp.config.tool_discovery import ToolDiscoveryConfig -from quickapp.config.tools.base import BaseOpenAITool +from quickapp.config.tools.base import ( + BaseOpenAITool, + OpenAiToolConfigDict, + remove_const_schema_params, +) from quickapp.config.toolsets.base import BaseToolSet @@ -14,11 +16,11 @@ class DeferredToolsContext: def __init__(self) -> None: self._catalog: list[dict[str, str]] = [] - self._definitions: dict[str, dict[str, Any]] = {} + self._definitions: dict[str, OpenAiToolConfigDict] = {} def register_staged_tools(self, tools: list[StagedBaseTool]) -> None: - entries: list[tuple[BaseOpenAITool, str]] = [ - (t.tool_config, name) + entries: list[tuple[StagedBaseTool, BaseOpenAITool, str]] = [ + (t, t.tool_config, name) for t in tools if isinstance(t.tool_config, BaseOpenAITool) and (name := t.tool_config.open_ai_tool.function.name) @@ -28,12 +30,16 @@ def register_staged_tools(self, tools: list[StagedBaseTool]) -> None: "name": name, "description": tool_config.open_ai_tool.function.description or "", } - for tool_config, name in entries + for _, tool_config, name in entries ) self._definitions.update( { - name: tool_config.open_ai_tool.model_dump(mode="json", exclude_none=True) - for tool_config, name in entries + # Apply the same transform pipeline as the eager path (AgentModule.provide_openai_tools) + # so a discovered tool's schema matches what it would have looked like loaded eagerly. + name: tool.enrich_openai_tool_schema( + remove_const_schema_params(tool_config.open_ai_tool) + ).model_dump(mode="json", exclude_none=True) + for tool, tool_config, name in entries } ) @@ -45,7 +51,7 @@ def deferred_names(self) -> frozenset[str]: def catalog(self) -> list[dict[str, str]]: return list(self._catalog) - def get_definition(self, name: str) -> dict[str, Any] | None: + def get_definition(self, name: str) -> OpenAiToolConfigDict | None: return self._definitions.get(name) diff --git a/src/quickapp/tool_discovery/tool_discovery_module.py b/src/quickapp/tool_discovery/tool_discovery_module.py index b2df8206..19d9e7e7 100644 --- a/src/quickapp/tool_discovery/tool_discovery_module.py +++ b/src/quickapp/tool_discovery/tool_discovery_module.py @@ -23,7 +23,7 @@ def configure(self, binder: Binder) -> None: binder.bind(_ToolSearchStageWrapper, to=_ToolSearchStageWrapper) @multiprovider - def _provide_tool_search_tool( + def _provide_tool_search_tools( self, config: ApplicationConfig, tool_builder: AssistedBuilder[_ToolSearchTool], diff --git a/src/scripts/dump_internal_tools.py b/src/scripts/dump_internal_tools.py index 7e4a4b46..ae1d6860 100644 --- a/src/scripts/dump_internal_tools.py +++ b/src/scripts/dump_internal_tools.py @@ -43,6 +43,7 @@ from quickapp.config.orchestrator_attachment_strategy import LazyOnDemandAttachmentStrategy from quickapp.config.prompt import CustomSystemPromptConfig from quickapp.config.timestamp import ToolCallTimestampConfig +from quickapp.config.tool_discovery import ToolDiscoveryConfig from quickapp.config.tools.const import ALL_MIME_TYPES from quickapp.config.web_fetch import WebFetchConfig from quickapp.core.agent import OrchestratorCapabilities @@ -71,6 +72,7 @@ def build_dump_application_config() -> ApplicationConfig: variables={}, ), attachment_strategy=LazyOnDemandAttachmentStrategy(), + tool_discovery=ToolDiscoveryConfig(enabled=True), ), contexts=[ FileContextConfig( diff --git a/src/tests/integration_tests/test_runner/config.py b/src/tests/integration_tests/test_runner/config.py index ce9b8fb7..31d6fd3f 100644 --- a/src/tests/integration_tests/test_runner/config.py +++ b/src/tests/integration_tests/test_runner/config.py @@ -23,9 +23,6 @@ "integration_simple": ["test_tool_set_chat_hub"], "e2e": ["test_tool_set_chat_hub", "test_tool_set_py_interpreter"], "lazy_admin_context": [], - # Preview: MCP toolset with deferred=true for dynamic tool discovery tests. - # Requires ENABLE_PREVIEW_FEATURES=true and orchestrator.tool_discovery.enabled=true. - "tool_discovery": ["test_mcp_tool_deferred"], } diff --git a/src/tests/integration_tests/test_runner/test_mcp_tool_deferred.json b/src/tests/integration_tests/test_runner/test_mcp_tool_deferred.json deleted file mode 100644 index 0fc125eb..00000000 --- a/src/tests/integration_tests/test_runner/test_mcp_tool_deferred.json +++ /dev/null @@ -1,11 +0,0 @@ -{ - "name": "mcp-local-toolset", - "description": "Set with MCP tools (deferred)", - "type": "mcp", - "deferred": true, - "mcp_server_info": { - "url": "http://localhost:8003/mcp", - "protocol": "streamable_http", - "authorization": null - } -} diff --git a/src/tests/unit_tests/agent_tests/test_chat_completion_config_builder.py b/src/tests/unit_tests/agent_tests/test_chat_completion_config_builder.py index 6c1ec7f0..f54bcee8 100644 --- a/src/tests/unit_tests/agent_tests/test_chat_completion_config_builder.py +++ b/src/tests/unit_tests/agent_tests/test_chat_completion_config_builder.py @@ -1,7 +1,71 @@ +from unittest.mock import MagicMock + +from aidial_sdk.chat_completion import Message, Role + from quickapp.core.agent._chat_completion_config_builder import _ChatCompletionConfigBuilder +from quickapp.core.agent.lazy_loaded_tools_holder import LazyLoadedToolsHolder from quickapp.core.agent.models import STATE_KEY_ORCHESTRATOR as ORCH +def _make_tool_dict(name: str) -> dict: + return {"type": "function", "function": {"name": name, "description": "", "parameters": {}}} + + +def _make_builder( + tools: list[dict], lazy_holder: LazyLoadedToolsHolder +) -> _ChatCompletionConfigBuilder: + config = MagicMock() + config.orchestrator.deployment.parameters.model_dump.return_value = {} + config.orchestrator.deployment.deployment_id = "orchestrator-model" + tool_choice_holder = MagicMock() + tool_choice_holder.consume.return_value = None + presentation_settings = MagicMock() + presentation_settings.show_usage_statistics = False + return _ChatCompletionConfigBuilder( + config=config, + tools=tools, + response_format=None, + tool_choice_holder=tool_choice_holder, + pre_invocation_transformers=[], + presentation_settings=presentation_settings, + forwarded_headers=None, + lazy_loaded_tools_holder=lazy_holder, + ) + + +def test_build_merges_lazy_tools_after_eager_tools(): + """Discovered (lazy) tools are appended to the eager tools in the outgoing payload.""" + lazy_holder = LazyLoadedToolsHolder() + lazy_holder.add([_make_tool_dict("discovered_tool")]) + builder = _make_builder([_make_tool_dict("eager_tool")], lazy_holder) + + payload = builder.build([Message(role=Role.USER, content="hi")]) + + names = [t["function"]["name"] for t in payload["tools"]] + assert names == ["eager_tool", "discovered_tool"] + + +def test_build_dedupes_lazy_tools_against_eager_names(): + """A lazy tool whose name collides with an eager tool is dropped, not duplicated.""" + lazy_holder = LazyLoadedToolsHolder() + lazy_holder.add([_make_tool_dict("shared_name"), _make_tool_dict("discovered_tool")]) + builder = _make_builder([_make_tool_dict("shared_name")], lazy_holder) + + payload = builder.build([Message(role=Role.USER, content="hi")]) + + names = [t["function"]["name"] for t in payload["tools"]] + assert names == ["shared_name", "discovered_tool"] + + +def test_build_with_no_lazy_tools_returns_eager_tools_only(): + lazy_holder = LazyLoadedToolsHolder() + builder = _make_builder([_make_tool_dict("eager_tool")], lazy_holder) + + payload = builder.build([Message(role=Role.USER, content="hi")]) + + assert [t["function"]["name"] for t in payload["tools"]] == ["eager_tool"] + + def test_promote_orchestrator_state_to_top_level(): """Before the next orchestrator call, state.orchestrator (response state only) is promoted to top-level.""" msg = { diff --git a/src/tests/unit_tests/internal_tooling_tests/test_internal_tooling_module.py b/src/tests/unit_tests/internal_tooling_tests/test_internal_tooling_module.py new file mode 100644 index 00000000..230018d8 --- /dev/null +++ b/src/tests/unit_tests/internal_tooling_tests/test_internal_tooling_module.py @@ -0,0 +1,87 @@ +from unittest.mock import MagicMock + +from quickapp.common.tool_names import INTERNAL_CODE_EXECUTION_PYTHON_INTERPRETER_TOOL_NAME +from quickapp.config.tool_discovery import ToolDiscoveryConfig +from quickapp.config.tools.base import ( + OpenAiToolConfig, + OpenAiToolFunction, + OpenAiToolFunctionParameters, +) +from quickapp.config.tools.internal import InternalTool +from quickapp.config.toolsets.internal import InternalToolSet +from quickapp.internal_tooling.internal_tooling_module import InternalToolModule +from quickapp.shared.deferred_tools import DeferredToolsContext + + +def _make_internal_tool_config( + name: str = INTERNAL_CODE_EXECUTION_PYTHON_INTERPRETER_TOOL_NAME, +) -> InternalTool: + return InternalTool( + open_ai_tool=OpenAiToolConfig( + function=OpenAiToolFunction( + name=name, + description="Runs python code", + parameters=OpenAiToolFunctionParameters(type="object", properties={}), + ) + ) + ) + + +def _make_app_config( + toolset: InternalToolSet, discovery_cfg: ToolDiscoveryConfig | None +) -> MagicMock: + app_config = MagicMock() + app_config.tool_sets = [toolset] + app_config.orchestrator.tool_discovery = discovery_cfg + return app_config + + +def _make_py_builder(tool_config: InternalTool) -> MagicMock: + staged_tool = MagicMock() + staged_tool.tool_config = tool_config + builder = MagicMock() + builder.build.return_value = staged_tool + return staged_tool, builder + + +class TestProvideInternalTools: + def test_registers_deferred_toolset_with_deferred_context(self): + tool_config = _make_internal_tool_config() + toolset = InternalToolSet(name="internal", deferred=True, tools=[tool_config]) + discovery_cfg = ToolDiscoveryConfig(enabled=True, min_tools_for_deferral=1) + app_config = _make_app_config(toolset, discovery_cfg) + staged_tool, py_builder = _make_py_builder(tool_config) + deferred_context = MagicMock(spec=DeferredToolsContext) + + module = InternalToolModule() + result = module._provide_internal_tools(app_config, py_builder, deferred_context) + + assert result == [staged_tool] + deferred_context.register_staged_tools.assert_called_once_with([staged_tool]) + + def test_does_not_defer_below_threshold(self): + tool_config = _make_internal_tool_config() + toolset = InternalToolSet(name="internal", deferred=True, tools=[tool_config]) + discovery_cfg = ToolDiscoveryConfig(enabled=True, min_tools_for_deferral=5) + app_config = _make_app_config(toolset, discovery_cfg) + staged_tool, py_builder = _make_py_builder(tool_config) + deferred_context = MagicMock(spec=DeferredToolsContext) + + module = InternalToolModule() + result = module._provide_internal_tools(app_config, py_builder, deferred_context) + + assert result == [staged_tool] + deferred_context.register_staged_tools.assert_not_called() + + def test_does_not_defer_when_discovery_disabled(self): + tool_config = _make_internal_tool_config() + toolset = InternalToolSet(name="internal", deferred=True, tools=[tool_config]) + app_config = _make_app_config(toolset, discovery_cfg=None) + staged_tool, py_builder = _make_py_builder(tool_config) + deferred_context = MagicMock(spec=DeferredToolsContext) + + module = InternalToolModule() + result = module._provide_internal_tools(app_config, py_builder, deferred_context) + + assert result == [staged_tool] + deferred_context.register_staged_tools.assert_not_called() diff --git a/src/tests/unit_tests/tool_discovery_tests/__init__.py b/src/tests/unit_tests/tool_discovery_tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/tests/unit_tests/tool_discovery_tests/test_anonymous_agent.py b/src/tests/unit_tests/tool_discovery_tests/test_anonymous_agent.py new file mode 100644 index 00000000..d2f1d6a1 --- /dev/null +++ b/src/tests/unit_tests/tool_discovery_tests/test_anonymous_agent.py @@ -0,0 +1,125 @@ +from unittest.mock import AsyncMock, MagicMock + +import openai +import pytest + +from quickapp.tool_discovery._anonymous_agent import _AnonymousAgent + + +def _make_config( + tool_discovery=None, service_model=None, orchestrator_deployment_id="orchestrator-model" +): + config = MagicMock() + config.orchestrator.deployment.deployment_id = orchestrator_deployment_id + if tool_discovery is None: + config.orchestrator.tool_discovery = None + else: + discovery = MagicMock() + discovery.service_model = service_model + config.orchestrator.tool_discovery = discovery + return config + + +def _make_response(content: str): + response = MagicMock() + response.choices = [MagicMock()] + response.choices[0].message.content = content + return response + + +CATALOG = [{"name": "tool_a", "description": "Does A"}, {"name": "tool_b", "description": "Does B"}] + + +@pytest.mark.asyncio +async def test_route_returns_empty_list_for_empty_catalog(): + client = AsyncMock() + agent = _AnonymousAgent(client=client, config=_make_config()) + + result = await agent.route("find a tool", []) + + assert result == [] + client.chat.completions.create.assert_not_called() + + +@pytest.mark.asyncio +async def test_route_fails_open_when_tool_discovery_is_none(): + client = AsyncMock() + agent = _AnonymousAgent(client=client, config=_make_config(tool_discovery=None)) + + result = await agent.route("find a tool", CATALOG) + + assert result == [] + client.chat.completions.create.assert_not_called() + + +@pytest.mark.asyncio +async def test_route_returns_matched_names_on_valid_json_response(): + client = AsyncMock() + client.chat.completions.create.return_value = _make_response('["tool_a", "tool_b"]') + agent = _AnonymousAgent(client=client, config=_make_config(tool_discovery=True)) + + result = await agent.route("find a tool", CATALOG) + + assert result == ["tool_a", "tool_b"] + + +@pytest.mark.asyncio +async def test_route_uses_service_model_when_configured(): + client = AsyncMock() + client.chat.completions.create.return_value = _make_response("[]") + agent = _AnonymousAgent( + client=client, config=_make_config(tool_discovery=True, service_model="cheap-router-model") + ) + + await agent.route("find a tool", CATALOG) + + assert client.chat.completions.create.call_args.kwargs["model"] == "cheap-router-model" + + +@pytest.mark.asyncio +async def test_route_falls_back_to_orchestrator_deployment_when_service_model_unset(): + client = AsyncMock() + client.chat.completions.create.return_value = _make_response("[]") + agent = _AnonymousAgent( + client=client, + config=_make_config( + tool_discovery=True, service_model=None, orchestrator_deployment_id="orchestrator-model" + ), + ) + + await agent.route("find a tool", CATALOG) + + assert client.chat.completions.create.call_args.kwargs["model"] == "orchestrator-model" + + +@pytest.mark.asyncio +async def test_route_returns_empty_list_on_non_json_response(): + client = AsyncMock() + client.chat.completions.create.return_value = _make_response("not json at all") + agent = _AnonymousAgent(client=client, config=_make_config(tool_discovery=True)) + + result = await agent.route("find a tool", CATALOG) + + assert result == [] + + +@pytest.mark.asyncio +async def test_route_filters_out_non_string_entries_from_response(): + client = AsyncMock() + client.chat.completions.create.return_value = _make_response('["tool_a", 123, null]') + agent = _AnonymousAgent(client=client, config=_make_config(tool_discovery=True)) + + result = await agent.route("find a tool", CATALOG) + + assert result == ["tool_a"] + + +@pytest.mark.asyncio +async def test_route_returns_empty_list_on_openai_error(): + client = AsyncMock() + client.chat.completions.create.side_effect = openai.APIConnectionError(request=MagicMock()) + agent = _AnonymousAgent(client=client, config=_make_config(tool_discovery=True)) + + result = await agent.route("find a tool", CATALOG) + + assert result == [] diff --git a/src/tests/unit_tests/tool_discovery_tests/test_deferred_tools_context.py b/src/tests/unit_tests/tool_discovery_tests/test_deferred_tools_context.py new file mode 100644 index 00000000..072800ba --- /dev/null +++ b/src/tests/unit_tests/tool_discovery_tests/test_deferred_tools_context.py @@ -0,0 +1,170 @@ +from unittest.mock import MagicMock + +from quickapp.config.tool_discovery import ToolDiscoveryConfig +from quickapp.config.tools.base import ( + OpenAiToolConfig, + OpenAiToolFunction, + OpenAiToolFunctionParameters, +) +from quickapp.config.tools.rest_api import ( + RestApiEndpointConstParam, + RestApiEndpointHeaderParamInfo, + RestApiEndpointMethodInfo, + RestApiEndpointSimpleTypeParam, + RestApiTool, + ToolEndpointInfoMethodType, + ToolEndpointParamType, +) +from quickapp.config.toolsets.rest_api import RestApiToolSet +from quickapp.shared.deferred_tools._deferred_tools_context import ( + DeferredToolsContext, + is_toolset_deferred, +) + + +def _make_toolset(deferred: bool | None = None) -> RestApiToolSet: + return RestApiToolSet(name="my-toolset", deferred=deferred, tools=[]) + + +def _make_discovery_config( + enabled: bool = True, min_tools_for_deferral: int = 5 +) -> ToolDiscoveryConfig: + return ToolDiscoveryConfig(enabled=enabled, min_tools_for_deferral=min_tools_for_deferral) + + +class TestIsToolsetDeferred: + def test_deferred_true_above_threshold_is_deferred(self): + assert ( + is_toolset_deferred(_make_toolset(True), _make_discovery_config(), tool_count=5) is True + ) + + def test_unset_defaults_to_deferred(self): + """`deferred` unset (None) behaves the same as `deferred: true`.""" + assert ( + is_toolset_deferred(_make_toolset(None), _make_discovery_config(), tool_count=5) is True + ) + + def test_explicit_false_is_never_deferred_even_above_threshold(self): + assert ( + is_toolset_deferred(_make_toolset(False), _make_discovery_config(), tool_count=50) + is False + ) + + def test_below_threshold_is_not_deferred(self): + assert ( + is_toolset_deferred(_make_toolset(True), _make_discovery_config(), tool_count=4) + is False + ) + + def test_at_threshold_boundary_is_deferred(self): + cfg = _make_discovery_config(min_tools_for_deferral=5) + assert is_toolset_deferred(_make_toolset(True), cfg, tool_count=5) is True + + def test_discovery_disabled_is_never_deferred(self): + cfg = _make_discovery_config(enabled=False) + assert is_toolset_deferred(_make_toolset(True), cfg, tool_count=50) is False + + def test_discovery_config_none_is_never_deferred(self): + assert is_toolset_deferred(_make_toolset(True), None, tool_count=50) is False + + +def _make_rest_api_tool_with_const_param(name: str = "test_tool") -> RestApiTool: + return RestApiTool( + rest_api_method_info=RestApiEndpointMethodInfo( + method_url="https://example.com", method_type=ToolEndpointInfoMethodType.get + ), + open_ai_tool=OpenAiToolConfig( + function=OpenAiToolFunction( + name=name, + description="A test tool", + parameters=OpenAiToolFunctionParameters( + type="object", + properties={ + "query": RestApiEndpointSimpleTypeParam( + type="string", + description="Query param", + parameter_info=RestApiEndpointHeaderParamInfo( + type=ToolEndpointParamType.query, key="query" + ), + ), + "api_key": RestApiEndpointConstParam( + type=None, + const="secret-value", + parameter_info=RestApiEndpointHeaderParamInfo( + type=ToolEndpointParamType.header, key="X-Api-Key" + ), + ), + }, + ), + ) + ), + ) + + +def _make_staged_tool(tool_config, enrich_side_effect=None) -> MagicMock: + staged_tool = MagicMock() + staged_tool.tool_config = tool_config + staged_tool.enrich_openai_tool_schema.side_effect = enrich_side_effect or (lambda t: t) + return staged_tool + + +class TestRegisterStagedTools: + def test_catalog_contains_name_and_description(self): + context = DeferredToolsContext() + tool_config = _make_rest_api_tool_with_const_param() + context.register_staged_tools([_make_staged_tool(tool_config)]) + + assert context.catalog == [{"name": "test_tool", "description": "A test tool"}] + + def test_deferred_names_reflects_registered_tools(self): + context = DeferredToolsContext() + context.register_staged_tools( + [_make_staged_tool(_make_rest_api_tool_with_const_param("tool_a"))] + ) + + assert context.deferred_names == frozenset({"tool_a"}) + + def test_definition_strips_const_params_same_as_eager_path(self): + """Regression: a discovered tool's schema must match the eager path — const params + (fixed values hidden from the LLM) must not leak into the schema surfaced via tool_search. + """ + context = DeferredToolsContext() + tool_config = _make_rest_api_tool_with_const_param() + context.register_staged_tools([_make_staged_tool(tool_config)]) + + definition = context.get_definition("test_tool") + + assert definition is not None + properties = definition["function"]["parameters"]["properties"] + assert "query" in properties + assert "api_key" not in properties + + def test_definition_applies_enrich_openai_tool_schema(self): + """The per-tool enrich_openai_tool_schema hook runs for deferred tools too.""" + + def enrich(open_ai_tool: OpenAiToolConfig) -> OpenAiToolConfig: + enriched = open_ai_tool.model_copy(deep=True) + enriched.function.description = "enriched" + return enriched + + context = DeferredToolsContext() + tool_config = _make_rest_api_tool_with_const_param() + context.register_staged_tools([_make_staged_tool(tool_config, enrich_side_effect=enrich)]) + + definition = context.get_definition("test_tool") + assert definition is not None + assert definition["function"]["description"] == "enriched" + + def test_get_definition_returns_none_for_unknown_name(self): + context = DeferredToolsContext() + assert context.get_definition("does_not_exist") is None + + def test_ignores_tools_without_openai_tool_config(self): + context = DeferredToolsContext() + non_openai_staged_tool = MagicMock() + non_openai_staged_tool.tool_config = MagicMock() # not a BaseOpenAITool instance + + context.register_staged_tools([non_openai_staged_tool]) + + assert context.catalog == [] + assert context.deferred_names == frozenset() From 328909c6fbe838114d578a90b6ab19177fcfc096 Mon Sep 17 00:00:00 2001 From: Oleksandr Gubarets Date: Tue, 15 Sep 2026 14:31:05 +0300 Subject: [PATCH 08/10] code review fix. --- config/predefined/toolset/weather.json | 4 +- docs/designs/dynamic_tool_discovery.md | 11 ++- docs/generated-internal-tools.json | 2 +- src/quickapp/app_factory.py | 5 +- src/quickapp/common/deferred_tool_types.py | 14 +++ .../deferred_tools_accumulator.py} | 30 ++++-- src/quickapp/core/agent/agent_module.py | 6 +- .../_internal_deferred_tools_context.py | 5 + .../internal_tooling_module.py | 46 ++++++++- .../mcp_tooling/_mcp_tool_initializer.py | 8 +- .../mcp_tooling/_mcp_tooling_context.py | 6 +- .../mcp_tooling/mcp_tooling_module.py | 34 +++++++ .../_rest_api_deferred_tools_context.py | 5 + .../rest_api_tooling_module.py | 46 ++++++++- src/quickapp/shared/__init__.py | 2 - .../shared/deferred_tools/__init__.py | 3 - .../deferred_tools/deferred_tools_module.py | 16 ---- src/quickapp/tool_discovery/_tool_configs.py | 28 ++++-- .../_tool_search_hint_prompt_provider.py | 34 +++++++ .../tool_discovery/_tool_search_tool.py | 34 ++++++- .../tool_discovery/_toolset_summary_format.py | 11 +++ .../tool_discovery/tool_discovery_module.py | 23 ++++- .../mcp_server/mcp_http_test_server.py | 8 +- .../test_deferred_tools_accumulator.py} | 94 +++++++++++++++---- .../test_internal_tooling_module.py | 14 +-- .../test_mcp_initializer_interactive_login.py | 1 - .../test_mcp_tool_initializer.py | 4 - .../test_rest_api_tool.py | 18 ++-- .../test_tool_discovery_module.py | 60 ++++++++++++ .../test_tool_search_hint_prompt_provider.py | 43 +++++++++ .../test_tool_search_tool.py | 87 +++++++++++++++++ .../test_toolset_summary_format.py | 32 +++++++ 32 files changed, 632 insertions(+), 102 deletions(-) create mode 100644 src/quickapp/common/deferred_tool_types.py rename src/quickapp/{shared/deferred_tools/_deferred_tools_context.py => common/deferred_tools_accumulator.py} (67%) create mode 100644 src/quickapp/internal_tooling/_internal_deferred_tools_context.py create mode 100644 src/quickapp/rest_api_tooling/_rest_api_deferred_tools_context.py delete mode 100644 src/quickapp/shared/deferred_tools/__init__.py delete mode 100644 src/quickapp/shared/deferred_tools/deferred_tools_module.py create mode 100644 src/quickapp/tool_discovery/_tool_search_hint_prompt_provider.py create mode 100644 src/quickapp/tool_discovery/_toolset_summary_format.py rename src/tests/unit_tests/{tool_discovery_tests/test_deferred_tools_context.py => common/test_deferred_tools_accumulator.py} (63%) create mode 100644 src/tests/unit_tests/tool_discovery_tests/test_tool_discovery_module.py create mode 100644 src/tests/unit_tests/tool_discovery_tests/test_tool_search_hint_prompt_provider.py create mode 100644 src/tests/unit_tests/tool_discovery_tests/test_tool_search_tool.py create mode 100644 src/tests/unit_tests/tool_discovery_tests/test_toolset_summary_format.py diff --git a/config/predefined/toolset/weather.json b/config/predefined/toolset/weather.json index 7519159e..7b6e5613 100644 --- a/config/predefined/toolset/weather.json +++ b/config/predefined/toolset/weather.json @@ -56,9 +56,7 @@ }, "required": [ "latitude", - "longitude", - "current", - "format" + "longitude" ] } } diff --git a/docs/designs/dynamic_tool_discovery.md b/docs/designs/dynamic_tool_discovery.md index 4eb7d034..dbc43bae 100644 --- a/docs/designs/dynamic_tool_discovery.md +++ b/docs/designs/dynamic_tool_discovery.md @@ -339,6 +339,14 @@ payload["messages"] = full conversation history The description of `tool_search` explicitly states that additional tools are available and can be discovered on demand, so the model knows to search before assuming a capability is missing. +**As built:** the description also carries a dynamic, per-request section listing every currently +deferred toolset by name, its tool count, and its own `description` (when set) — e.g. "Additional +toolsets available for discovery: - salesforce. Available tools: 12. Query and update Salesforce +records". This is built in `_ToolSearchTool.enrich_openai_tool_schema` from +`DeferredToolsContext.toolset_summaries`, giving the model a hint about *what* (and how much) is +hidden, not just that *something* is discoverable — without the per-tool schema cost that listing +every tool upfront would incur. + #### Step 3 — `tool_search` execution: anonymous agent When the orchestrator calls `tool_search(query)`, its handler delegates to a new @@ -482,8 +490,9 @@ eagerly, no discovery overhead. | `shared/deferred_tools/` (`DeferredToolsContext`, `is_toolset_deferred`) | Request-scoped shared object aggregating `catalog`/`definitions` across all deferred toolsets in the request; `is_toolset_deferred` is the pure threshold predicate. Bound via its own `DeferredToolsModule`, spliced into `shared_module` | | REST, MCP, internal toolset modules | After building each toolset's tools, evaluate `is_toolset_deferred`; register with `DeferredToolsContext` or leave in the eager `list[StagedBaseTool]` accordingly. **`dial-deployment`/`dial-app` toolsets do not yet do this** — follow-up | | `tool_discovery/_anonymous_agent.py` (`_AnonymousAgent`) | Fires a single isolated `chat.completions.create` call (no history, no app system prompt); takes the catalog and a user query; returns matched tool names | -| `tool_discovery/_tool_search_tool.py` (`_ToolSearchTool`) | Internal `tool_search` (registered name: `internal_tool_search`) tool injected via `ToolDiscoveryModule`'s own `@multiprovider` (preview-gated); calls `_AnonymousAgent`, looks up matched names in `DeferredToolsContext`, writes results into `LazyLoadedToolsHolder`, returns `[{name, description}]` to the main LLM | +| `tool_discovery/_tool_search_tool.py` (`_ToolSearchTool`) | Internal `tool_search` (registered name: `internal_tool_search`) tool injected via `ToolDiscoveryModule`'s own `@multiprovider` (preview-gated); calls `_AnonymousAgent`, looks up matched names in `DeferredToolsContext`, writes results into `LazyLoadedToolsHolder`, returns `[{name, description}]` to the main LLM. Its own `enrich_openai_tool_schema` override appends a dynamic list of deferred toolset names/descriptions (`DeferredToolsContext.toolset_summaries`) to the static tool description | | `core/agent/lazy_loaded_tools_holder.py` (`LazyLoadedToolsHolder`) | Request-scoped holder of discovered `OpenAiToolConfigDict`s — replaces the originally-proposed lazy-initializer + orchestrator-side `_lazy_loaded_tools` state | +| `tool_discovery/_tool_search_hint_prompt_provider.py` (`_ToolSearchHintPromptProvider`) | System-prompt-level reminder to call `internal_tool_search` before declaring a limitation, plus the same deferred-toolset summary list (name, tool count, description — via the shared `_toolset_summary_format.format_toolset_summaries`) that `_ToolSearchTool.enrich_openai_tool_schema` appends to the tool's own description. Contributed by `ToolDiscoveryModule`'s own `_provide_prompt_parts` (preview-gated, same condition as the tool itself). `ToolDiscoveryModule` is registered in `app_factory.py` **before** `SkillsModule` specifically so this hint lands immediately ahead of the `` block in the aggregated system prompt | | `_chat_completion_config_builder.py` | Reads `LazyLoadedToolsHolder.get_all()` on every build and merges into `payload["tools"]`, de-duplicated against eager tool names — **`orchestrator.py` itself was not changed** | | Cross-turn state persistence | **Not implemented** — see [Out of Scope](#out-of-scope-mvp) | diff --git a/docs/generated-internal-tools.json b/docs/generated-internal-tools.json index 098d2a66..2ee98442 100644 --- a/docs/generated-internal-tools.json +++ b/docs/generated-internal-tools.json @@ -387,7 +387,7 @@ }, { "name": "internal_tool_search", - "description": "Search for additional tools available to this assistant. Use this when you need a capability that is not listed in the current tool list. Returns the names and descriptions of matching tools; those tools will be available to call immediately after.", + "description": "\nPurpose: Discover additional tools and capabilities.\n\nYou MUST call this tool before stating any limitation or saying you cannot fulfill a user request.\n\nIn addition, you MUST call this tool whenever:\n- The user’s request is open-ended or underspecified.\n- The request might involve capabilities beyond your currently listed tools.\n- The request could plausibly be served by a specialized tool or by returning a resource, even if you believe you can respond text-only.\n\nIf a user request might involve capabilities beyond your current listed tools, you are REQUIRED to:\n1) Call internal_tool_search with a brief description of the needed capability.\n2) Inspect any returned tools.\n\nYou may NOT:\n- Assume that the initially listed tools are exhaustive.\n- Say \"I can't\", \"I don't have access\", or express similar limitations until you have called internal_tool_search in this conversation turn.\n\nOnly if internal_tool_search returns no suitable tools, or all relevant tools fail, may you tell the user you cannot do it.\n\nThis tool dynamically discovers additional toolsets (including hidden or MCP tools) that may provide additional capabilities:\n", "properties": { "query": { "type": "string", diff --git a/src/quickapp/app_factory.py b/src/quickapp/app_factory.py index 31a2f30c..0c32e3d2 100644 --- a/src/quickapp/app_factory.py +++ b/src/quickapp/app_factory.py @@ -58,11 +58,14 @@ def build_di_modules() -> list[Module]: FileTransferModule(), AttachmentProcessingModule(), LazyOnDemandStrategyModule(), + # ToolDiscoveryModule is registered before SkillsModule so its tool_search prompt + # hint (when active) lands immediately ahead of the block in the + # aggregated system prompt (list[PromptPartProvider] preserves module registration order). + ToolDiscoveryModule(), SkillsModule(), DialPromptSkillsModule(), DialSkillsModule(), TimestampModule(), - ToolDiscoveryModule(), AgentHooksModule(), DialFilesToolingModule(), WebToolingModule(), diff --git a/src/quickapp/common/deferred_tool_types.py b/src/quickapp/common/deferred_tool_types.py new file mode 100644 index 00000000..d25fd2e0 --- /dev/null +++ b/src/quickapp/common/deferred_tool_types.py @@ -0,0 +1,14 @@ +from typing import Annotated + +from pydantic import BaseModel + +from quickapp.config.tools.base import OpenAiToolConfigDict + +DeferredToolName = Annotated[str, "DeferredToolName"] +DeferredToolCatalogEntry = Annotated[dict[str, str], "DeferredToolCatalogEntry"] +DeferredToolsetSummary = Annotated[dict[str, str | int | None], "DeferredToolsetSummary"] + + +class DeferredToolDefinition(BaseModel): + name: str + definition: OpenAiToolConfigDict diff --git a/src/quickapp/shared/deferred_tools/_deferred_tools_context.py b/src/quickapp/common/deferred_tools_accumulator.py similarity index 67% rename from src/quickapp/shared/deferred_tools/_deferred_tools_context.py rename to src/quickapp/common/deferred_tools_accumulator.py index 96c52d1a..dab62e31 100644 --- a/src/quickapp/shared/deferred_tools/_deferred_tools_context.py +++ b/src/quickapp/common/deferred_tools_accumulator.py @@ -1,6 +1,5 @@ -from injector import inject - -from quickapp.common import StagedBaseTool +from quickapp.common.localized_string import resolve_localized +from quickapp.common.staged_base_tool import StagedBaseTool from quickapp.config.tool_discovery import ToolDiscoveryConfig from quickapp.config.tools.base import ( BaseOpenAITool, @@ -10,15 +9,19 @@ from quickapp.config.toolsets.base import BaseToolSet -@inject -class DeferredToolsContext: - """Request-scoped holder for tool catalog and full definitions of deferred toolsets.""" +class DeferredToolsAccumulator: + """Base contract for a tooling module's own deferred-tool registry. + + Concrete subclasses are bound request-scoped by their owning module (mirrors + `ToolingContextBase`), not by a Module of their own. + """ def __init__(self) -> None: self._catalog: list[dict[str, str]] = [] self._definitions: dict[str, OpenAiToolConfigDict] = {} + self._toolset_summaries: list[dict[str, str | int | None]] = [] - def register_staged_tools(self, tools: list[StagedBaseTool]) -> None: + def register_deferred_tools(self, toolset: BaseToolSet, tools: list[StagedBaseTool]) -> None: entries: list[tuple[StagedBaseTool, BaseOpenAITool, str]] = [ (t, t.tool_config, name) for t in tools @@ -42,6 +45,15 @@ def register_staged_tools(self, tools: list[StagedBaseTool]) -> None: for tool, tool_config, name in entries } ) + self._toolset_summaries.append( + { + "name": resolve_localized(toolset.name), + "description": ( + resolve_localized(toolset.description) if toolset.description else None + ), + "tool_count": len(entries), + } + ) @property def deferred_names(self) -> frozenset[str]: @@ -51,6 +63,10 @@ def deferred_names(self) -> frozenset[str]: def catalog(self) -> list[dict[str, str]]: return list(self._catalog) + @property + def toolset_summaries(self) -> list[dict[str, str | int | None]]: + return list(self._toolset_summaries) + def get_definition(self, name: str) -> OpenAiToolConfigDict | None: return self._definitions.get(name) diff --git a/src/quickapp/core/agent/agent_module.py b/src/quickapp/core/agent/agent_module.py index 40d95a48..eb2c330e 100644 --- a/src/quickapp/core/agent/agent_module.py +++ b/src/quickapp/core/agent/agent_module.py @@ -23,6 +23,7 @@ from quickapp.common.chat_completion_recovery import ChatCompletionRecoveryService from quickapp.common.chat_completion_stream.chat_stream_sink_factory import ChatStreamSinkFactory from quickapp.common.chat_completion_stream.handler import ChatCompletionStreamHandler +from quickapp.common.deferred_tool_types import DeferredToolName from quickapp.common.dial_settings import DialSettings from quickapp.common.request_async_close_registry import RequestAsyncCloseRegistry from quickapp.common.stage_close_registry import DeferredStageCloseRegistry @@ -61,7 +62,6 @@ OrchestratorDeploymentCacheService, ) from quickapp.core.application._request_context import _RequestContext -from quickapp.shared.deferred_tools import DeferredToolsContext DEFAULT_QUERY_PARAM = ConfigurableSchemaSimpleType( type=JsonTypeEnum.string, @@ -168,9 +168,9 @@ def provide_openai_tools( self, tools: list[StagedBaseTool], static_tools: list[StaticTool], - deferred_context: DeferredToolsContext, + deferred_tool_names: list[DeferredToolName], ) -> list[OpenAiToolConfigDict]: - deferred_names = deferred_context.deferred_names + deferred_names = frozenset(deferred_tool_names) openai_functions = [] for tool in tools: if isinstance(tool.tool_config, BaseOpenAITool): diff --git a/src/quickapp/internal_tooling/_internal_deferred_tools_context.py b/src/quickapp/internal_tooling/_internal_deferred_tools_context.py new file mode 100644 index 00000000..6ad3d066 --- /dev/null +++ b/src/quickapp/internal_tooling/_internal_deferred_tools_context.py @@ -0,0 +1,5 @@ +from quickapp.common.deferred_tools_accumulator import DeferredToolsAccumulator + + +class _InternalDeferredToolsContext(DeferredToolsAccumulator): + """Request-scoped deferred-tool registry owned by internal tooling.""" diff --git a/src/quickapp/internal_tooling/internal_tooling_module.py b/src/quickapp/internal_tooling/internal_tooling_module.py index 0e9405d2..7ef1152e 100644 --- a/src/quickapp/internal_tooling/internal_tooling_module.py +++ b/src/quickapp/internal_tooling/internal_tooling_module.py @@ -4,12 +4,20 @@ from injector import AssistedBuilder, Binder, Module, multiprovider, provider, singleton from quickapp.common import DIAL_API_KEY, StagedBaseTool +from quickapp.common.deferred_tool_types import ( + DeferredToolCatalogEntry, + DeferredToolDefinition, + DeferredToolName, + DeferredToolsetSummary, +) +from quickapp.common.deferred_tools_accumulator import is_toolset_deferred from quickapp.common.dial_settings import DialSettings from quickapp.common.localized_string import resolve_localized from quickapp.common.tool_names import INTERNAL_CODE_EXECUTION_PYTHON_INTERPRETER_TOOL_NAME from quickapp.config.application import ApplicationConfig from quickapp.config.tools.predefined import PredefinedTool from quickapp.config.toolsets.internal import InternalToolSet +from quickapp.internal_tooling._internal_deferred_tools_context import _InternalDeferredToolsContext from quickapp.internal_tooling.py_interpreter_tooling._py_interpreter_client import ( _PyInterpreterClient, ) @@ -26,7 +34,6 @@ ) from quickapp.internal_tooling.py_interpreter_tooling.handlers.session_manager import SessionManager from quickapp.shared.config_resolvers.tool_timeout_resolver import ToolTimeoutResolver -from quickapp.shared.deferred_tools import DeferredToolsContext, is_toolset_deferred logger = logging.getLogger(__name__) @@ -38,6 +45,9 @@ def configure(self, binder: Binder) -> None: binder.bind(SessionManager, to=SessionManager, scope=request_scope) binder.bind(_PyInterpreterTool, to=_PyInterpreterTool, scope=request_scope) binder.bind(InputFileHandler, to=InputFileHandler, scope=request_scope) + binder.bind( + _InternalDeferredToolsContext, to=_InternalDeferredToolsContext, scope=request_scope + ) logger.debug("InternalTooling module configuration completed") @multiprovider @@ -45,7 +55,7 @@ def _provide_internal_tools( self, app_config: ApplicationConfig, py_builder: AssistedBuilder[_PyInterpreterTool], - deferred_context: DeferredToolsContext, + deferred_context: _InternalDeferredToolsContext, ) -> list[StagedBaseTool]: tools: list[StagedBaseTool] = [] @@ -72,9 +82,9 @@ def _provide_internal_tools( discovery_cfg = app_config.orchestrator.tool_discovery if is_toolset_deferred(tool_set, discovery_cfg, len(toolset_tools)): - deferred_context.register_staged_tools(toolset_tools) + deferred_context.register_deferred_tools(tool_set, toolset_tools) logger.debug( - "Deferred %d tools from internal toolset '%s' into DeferredToolsContext", + "Deferred %d tools from internal toolset '%s' into the deferred tools registry", len(toolset_tools), resolve_localized(tool_set.name), ) @@ -82,6 +92,34 @@ def _provide_internal_tools( return tools + @multiprovider + def _provide_deferred_tool_names( + self, deferred_context: _InternalDeferredToolsContext + ) -> list[DeferredToolName]: + return list(deferred_context.deferred_names) + + @multiprovider + def _provide_deferred_catalog_entries( + self, deferred_context: _InternalDeferredToolsContext + ) -> list[DeferredToolCatalogEntry]: + return deferred_context.catalog + + @multiprovider + def _provide_deferred_tool_definitions( + self, deferred_context: _InternalDeferredToolsContext + ) -> list[DeferredToolDefinition]: + return [ + DeferredToolDefinition(name=name, definition=definition) + for name in deferred_context.deferred_names + if (definition := deferred_context.get_definition(name)) is not None + ] + + @multiprovider + def _provide_deferred_toolset_summaries( + self, deferred_context: _InternalDeferredToolsContext + ) -> list[DeferredToolsetSummary]: + return deferred_context.toolset_summaries + @singleton @provider def _provide_py_interpreter_settings( diff --git a/src/quickapp/mcp_tooling/_mcp_tool_initializer.py b/src/quickapp/mcp_tooling/_mcp_tool_initializer.py index 39dcf086..d20192d3 100644 --- a/src/quickapp/mcp_tooling/_mcp_tool_initializer.py +++ b/src/quickapp/mcp_tooling/_mcp_tool_initializer.py @@ -11,6 +11,7 @@ from quickapp.common import ACCEPT_LANGUAGE, DIAL_API_KEY, StagedBaseTool from quickapp.common.base_initializer import CompletionInitializer +from quickapp.common.deferred_tools_accumulator import is_toolset_deferred from quickapp.common.dial_settings import DialSettings from quickapp.common.exceptions import ToolInitializationException from quickapp.common.json_schema_converter import JsonSchemaConverter @@ -33,7 +34,6 @@ from quickapp.mcp_tooling._mcp_eager_resource import MCPEagerTextResource from quickapp.mcp_tooling._mcp_resource_meta import MCPResourceMeta from quickapp.mcp_tooling._mcp_server_capabilities import MCPServerCapabilities -from quickapp.shared.deferred_tools import DeferredToolsContext, is_toolset_deferred from ._di_types import DialToolsetCacheService from ._mcp_tool import _MCPTool @@ -134,7 +134,6 @@ def __init__( login_service: InteractiveLoginService, accept_language: ACCEPT_LANGUAGE, app_config: ApplicationConfig, - deferred_context: DeferredToolsContext, ): # Resolved lazily in initialize() because dial_app_tooling contributes # to this multibinder only after _DialAppResolver runs. @@ -151,7 +150,6 @@ def __init__( self.__login_service: InteractiveLoginService = login_service self.__accept_language: ACCEPT_LANGUAGE = accept_language self.__app_config: ApplicationConfig = app_config - self.__deferred_context: DeferredToolsContext = deferred_context @staticmethod # todo add Title to config so that we could use it in stage name @@ -292,9 +290,9 @@ async def _load_tools( if created_tools: discovery_cfg = self.__app_config.orchestrator.tool_discovery if is_toolset_deferred(toolset_info, discovery_cfg, len(created_tools)): - self.__deferred_context.register_staged_tools(created_tools) + self.__mcp_context.register_deferred_tools(toolset_info, created_tools) logger.debug( - "Deferred %d tools from MCP toolset '%s' into DeferredToolsContext", + "Deferred %d tools from MCP toolset '%s' into the deferred tools registry", len(created_tools), resolve_localized(resolved_toolset.name), ) diff --git a/src/quickapp/mcp_tooling/_mcp_tooling_context.py b/src/quickapp/mcp_tooling/_mcp_tooling_context.py index 974ef571..ccba8221 100644 --- a/src/quickapp/mcp_tooling/_mcp_tooling_context.py +++ b/src/quickapp/mcp_tooling/_mcp_tooling_context.py @@ -1,3 +1,4 @@ +from quickapp.common.deferred_tools_accumulator import DeferredToolsAccumulator from quickapp.common.tooling_context_base import ToolingContextBase from quickapp.mcp_tooling._mcp_eager_resource import MCPEagerResource from quickapp.mcp_tooling._mcp_resource_meta import MCPResourceMeta @@ -5,9 +6,10 @@ from quickapp.mcp_tooling._mcp_toolset_client import _MCPToolsetClient -class _MCPToolingContext(ToolingContextBase): +class _MCPToolingContext(ToolingContextBase, DeferredToolsAccumulator): def __init__(self) -> None: - super().__init__() + ToolingContextBase.__init__(self) + DeferredToolsAccumulator.__init__(self) self._resource_metas: list[MCPResourceMeta] = [] self._eager_resources: list[MCPEagerResource] = [] self._server_capabilities: list[MCPServerCapabilities] = [] diff --git a/src/quickapp/mcp_tooling/mcp_tooling_module.py b/src/quickapp/mcp_tooling/mcp_tooling_module.py index dc4e9bed..c86473da 100644 --- a/src/quickapp/mcp_tooling/mcp_tooling_module.py +++ b/src/quickapp/mcp_tooling/mcp_tooling_module.py @@ -7,6 +7,12 @@ from quickapp.common.abstract.base_prompt_provider import PromptPartProvider from quickapp.common.abstract.base_transformer import MessagesTransformer from quickapp.common.base_initializer import CompletionInitializer +from quickapp.common.deferred_tool_types import ( + DeferredToolCatalogEntry, + DeferredToolDefinition, + DeferredToolName, + DeferredToolsetSummary, +) from quickapp.common.exceptions import InitializationException from quickapp.common.tool_names import INTERNAL_MCP_READ_RESOURCE_TOOL_NAME from quickapp.config.application import ApplicationConfig @@ -74,6 +80,34 @@ def __provide_initializers( def _provide_mcp_tools(self, mcp_context: _MCPToolingContext) -> list[StagedBaseTool]: return mcp_context.tools + @multiprovider + def _provide_deferred_tool_names( + self, mcp_context: _MCPToolingContext + ) -> list[DeferredToolName]: + return list(mcp_context.deferred_names) + + @multiprovider + def _provide_deferred_catalog_entries( + self, mcp_context: _MCPToolingContext + ) -> list[DeferredToolCatalogEntry]: + return mcp_context.catalog + + @multiprovider + def _provide_deferred_tool_definitions( + self, mcp_context: _MCPToolingContext + ) -> list[DeferredToolDefinition]: + return [ + DeferredToolDefinition(name=name, definition=definition) + for name in mcp_context.deferred_names + if (definition := mcp_context.get_definition(name)) is not None + ] + + @multiprovider + def _provide_deferred_toolset_summaries( + self, mcp_context: _MCPToolingContext + ) -> list[DeferredToolsetSummary]: + return mcp_context.toolset_summaries + @multiprovider def __provide_initialization_exceptions( self, context: _MCPToolingContext diff --git a/src/quickapp/rest_api_tooling/_rest_api_deferred_tools_context.py b/src/quickapp/rest_api_tooling/_rest_api_deferred_tools_context.py new file mode 100644 index 00000000..cdff893f --- /dev/null +++ b/src/quickapp/rest_api_tooling/_rest_api_deferred_tools_context.py @@ -0,0 +1,5 @@ +from quickapp.common.deferred_tools_accumulator import DeferredToolsAccumulator + + +class _RestApiDeferredToolsContext(DeferredToolsAccumulator): + """Request-scoped deferred-tool registry owned by REST API tooling.""" diff --git a/src/quickapp/rest_api_tooling/rest_api_tooling_module.py b/src/quickapp/rest_api_tooling/rest_api_tooling_module.py index f73ea68c..a84ac793 100644 --- a/src/quickapp/rest_api_tooling/rest_api_tooling_module.py +++ b/src/quickapp/rest_api_tooling/rest_api_tooling_module.py @@ -4,15 +4,22 @@ from injector import Binder, ClassAssistedBuilder, Module, multiprovider from quickapp.common import ACCEPT_LANGUAGE, StagedBaseTool +from quickapp.common.deferred_tool_types import ( + DeferredToolCatalogEntry, + DeferredToolDefinition, + DeferredToolName, + DeferredToolsetSummary, +) +from quickapp.common.deferred_tools_accumulator import is_toolset_deferred from quickapp.common.localized_string import resolve_localized from quickapp.common.oauth_token_fetcher import OAuthTokenFetcher from quickapp.common.utils import sanitize_toolname from quickapp.config.application import ApplicationConfig from quickapp.config.tools.rest_api import RestApiTool from quickapp.config.toolsets.rest_api import RestApiToolSet -from quickapp.shared.deferred_tools import DeferredToolsContext, is_toolset_deferred from ._request_detail_builder import _RequestDetailsBuilder +from ._rest_api_deferred_tools_context import _RestApiDeferredToolsContext from ._rest_api_stage_wrapper import _RestApiStageWrapper from ._rest_api_tool import _RestApiTool @@ -26,6 +33,9 @@ def configure(self, binder: Binder) -> None: binder.bind(_RestApiTool, to=_RestApiTool, scope=request_scope) binder.bind(_RequestDetailsBuilder, to=_RequestDetailsBuilder) binder.bind(OAuthTokenFetcher, to=OAuthTokenFetcher) + binder.bind( + _RestApiDeferredToolsContext, to=_RestApiDeferredToolsContext, scope=request_scope + ) logger.debug("RestApiTooling module configuration completed") @multiprovider @@ -34,7 +44,7 @@ def __provide_rest_api_tools( app_config: ApplicationConfig, tool_builder: ClassAssistedBuilder[_RestApiTool], accept_language: ACCEPT_LANGUAGE, - deferred_context: DeferredToolsContext, + deferred_context: _RestApiDeferredToolsContext, ) -> list[StagedBaseTool]: result: list[StagedBaseTool] = [] for toolset_info in app_config.tool_sets: @@ -43,15 +53,43 @@ def __provide_rest_api_tools( tools = self.__create_rest_api_tools(toolset_info, tool_builder, toolset_stage_name) discovery_cfg = app_config.orchestrator.tool_discovery if is_toolset_deferred(toolset_info, discovery_cfg, len(tools)): - deferred_context.register_staged_tools(tools) + deferred_context.register_deferred_tools(toolset_info, tools) logger.debug( - "Deferred %d tools from REST toolset '%s' into DeferredToolsContext", + "Deferred %d tools from REST toolset '%s' into the deferred tools registry", len(tools), toolset_stage_name, ) result.extend(tools) return result + @multiprovider + def _provide_deferred_tool_names( + self, deferred_context: _RestApiDeferredToolsContext + ) -> list[DeferredToolName]: + return list(deferred_context.deferred_names) + + @multiprovider + def _provide_deferred_catalog_entries( + self, deferred_context: _RestApiDeferredToolsContext + ) -> list[DeferredToolCatalogEntry]: + return deferred_context.catalog + + @multiprovider + def _provide_deferred_tool_definitions( + self, deferred_context: _RestApiDeferredToolsContext + ) -> list[DeferredToolDefinition]: + return [ + DeferredToolDefinition(name=name, definition=definition) + for name in deferred_context.deferred_names + if (definition := deferred_context.get_definition(name)) is not None + ] + + @multiprovider + def _provide_deferred_toolset_summaries( + self, deferred_context: _RestApiDeferredToolsContext + ) -> list[DeferredToolsetSummary]: + return deferred_context.toolset_summaries + @staticmethod def __create_rest_api_tools( rest_api_toolset: RestApiToolSet, diff --git a/src/quickapp/shared/__init__.py b/src/quickapp/shared/__init__.py index da543296..cdba8c85 100644 --- a/src/quickapp/shared/__init__.py +++ b/src/quickapp/shared/__init__.py @@ -1,7 +1,6 @@ from injector import Module from quickapp.shared.config_resolvers.config_resolvers_module import ConfigResolversModule -from quickapp.shared.deferred_tools.deferred_tools_module import DeferredToolsModule from quickapp.shared.external_fetch.external_fetch_module import ExternalFetchModule from quickapp.shared.home_path.home_path_module import HomePathModule @@ -10,7 +9,6 @@ # individually. shared_module: list[Module] = [ ConfigResolversModule(), - DeferredToolsModule(), ExternalFetchModule(), HomePathModule(), ] diff --git a/src/quickapp/shared/deferred_tools/__init__.py b/src/quickapp/shared/deferred_tools/__init__.py deleted file mode 100644 index 99fd006b..00000000 --- a/src/quickapp/shared/deferred_tools/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -from ._deferred_tools_context import DeferredToolsContext, is_toolset_deferred - -__all__ = ["DeferredToolsContext", "is_toolset_deferred"] diff --git a/src/quickapp/shared/deferred_tools/deferred_tools_module.py b/src/quickapp/shared/deferred_tools/deferred_tools_module.py deleted file mode 100644 index a8c783aa..00000000 --- a/src/quickapp/shared/deferred_tools/deferred_tools_module.py +++ /dev/null @@ -1,16 +0,0 @@ -from fastapi_injector import request_scope -from injector import Binder, Module - -from quickapp.shared.deferred_tools._deferred_tools_context import DeferredToolsContext - - -class DeferredToolsModule(Module): - """DI binding for the shared deferred-tools catalog. - - Request-scoped holder shared between the REST API, MCP and core agent modules - so toolsets withheld from the initial LLM payload can be registered and later - surfaced via the tool_search meta-tool. - """ - - def configure(self, binder: Binder) -> None: - binder.bind(DeferredToolsContext, to=DeferredToolsContext, scope=request_scope) diff --git a/src/quickapp/tool_discovery/_tool_configs.py b/src/quickapp/tool_discovery/_tool_configs.py index 5223778d..9cbcb3e7 100644 --- a/src/quickapp/tool_discovery/_tool_configs.py +++ b/src/quickapp/tool_discovery/_tool_configs.py @@ -13,12 +13,28 @@ open_ai_tool=OpenAiToolConfig( function=OpenAiToolFunction( name=INTERNAL_TOOL_SEARCH_TOOL_NAME, - description=( - "Search for additional tools available to this assistant. " - "Use this when you need a capability that is not listed in the current tool list. " - "Returns the names and descriptions of matching tools; " - "those tools will be available to call immediately after." - ), + description=(""" +Purpose: Discover additional tools and capabilities. + +You MUST call this tool before stating any limitation or saying you cannot fulfill a user request. + +In addition, you MUST call this tool whenever: +- The user’s request is open-ended or underspecified. +- The request might involve capabilities beyond your currently listed tools. +- The request could plausibly be served by a specialized tool or by returning a resource, even if you believe you can respond text-only. + +If a user request might involve capabilities beyond your current listed tools, you are REQUIRED to: +1) Call internal_tool_search with a brief description of the needed capability. +2) Inspect any returned tools. + +You may NOT: +- Assume that the initially listed tools are exhaustive. +- Say "I can't", "I don't have access", or express similar limitations until you have called internal_tool_search in this conversation turn. + +Only if internal_tool_search returns no suitable tools, or all relevant tools fail, may you tell the user you cannot do it. + +This tool dynamically discovers additional toolsets (including hidden or MCP tools) that may provide additional capabilities: +"""), parameters=OpenAiToolFunctionParameters( type=JsonTypeEnum.object, properties={ diff --git a/src/quickapp/tool_discovery/_tool_search_hint_prompt_provider.py b/src/quickapp/tool_discovery/_tool_search_hint_prompt_provider.py new file mode 100644 index 00000000..3fdd12a4 --- /dev/null +++ b/src/quickapp/tool_discovery/_tool_search_hint_prompt_provider.py @@ -0,0 +1,34 @@ +from injector import inject + +from quickapp.common.abstract.base_prompt_provider import PromptPartProvider +from quickapp.common.deferred_tool_types import DeferredToolsetSummary +from quickapp.common.tool_names import INTERNAL_TOOL_SEARCH_TOOL_NAME +from quickapp.tool_discovery._toolset_summary_format import format_toolset_summaries + +_TOOL_SEARCH_HINT = ( + "If a user request might involve capabilities beyond your current listed tools, you are " + "REQUIRED to:\n" + f"1) Call `{INTERNAL_TOOL_SEARCH_TOOL_NAME}` with a brief description of the needed capability.\n" + "2) Inspect any returned tools." +) + + +@inject +class _ToolSearchHintPromptProvider(PromptPartProvider): + """Reminds the orchestrator to try tool discovery before declaring a limitation, and lists + the toolsets currently withheld from the initial payload (name, tool count, description). + """ + + def __init__(self, toolset_summaries: list[DeferredToolsetSummary]) -> None: + self.__toolset_summaries = toolset_summaries + + async def get_prompt_part(self) -> str: + summaries = self.__toolset_summaries + if not summaries: + return _TOOL_SEARCH_HINT + + return ( + _TOOL_SEARCH_HINT + + "\n\nAdditional toolsets available for discovery:\n" + + format_toolset_summaries(summaries) + ) diff --git a/src/quickapp/tool_discovery/_tool_search_tool.py b/src/quickapp/tool_discovery/_tool_search_tool.py index b356b65e..0398745f 100644 --- a/src/quickapp/tool_discovery/_tool_search_tool.py +++ b/src/quickapp/tool_discovery/_tool_search_tool.py @@ -7,13 +7,19 @@ from quickapp.common import StagedBaseTool, ToolCallResult from quickapp.common.abstract.base_tool_argument_transformer import ToolArgumentTransformer from quickapp.common.base_stage_wrapper import BaseStageWrapper +from quickapp.common.deferred_tool_types import ( + DeferredToolCatalogEntry, + DeferredToolDefinition, + DeferredToolsetSummary, +) from quickapp.common.perf_timer.perf_timer import PerformanceTimer from quickapp.config.application import StageDisplayLevel +from quickapp.config.tools.base import OpenAiToolConfig, OpenAiToolConfigDict from quickapp.config.tools.internal import InternalTool from quickapp.core.agent.lazy_loaded_tools_holder import LazyLoadedToolsHolder -from quickapp.shared.deferred_tools import DeferredToolsContext from quickapp.tool_discovery._anonymous_agent import _AnonymousAgent from quickapp.tool_discovery._tool_search_stage_wrapper import _ToolSearchStageWrapper +from quickapp.tool_discovery._toolset_summary_format import format_toolset_summaries logger = logging.getLogger(__name__) @@ -27,7 +33,9 @@ def __init__( stage_wrapper_builder: AssistedBuilder[_ToolSearchStageWrapper], tool_config: InternalTool, perf_timer: PerformanceTimer, - deferred_context: DeferredToolsContext, + catalog: list[DeferredToolCatalogEntry], + definitions: list[DeferredToolDefinition], + toolset_summaries: list[DeferredToolsetSummary], lazy_holder: LazyLoadedToolsHolder, anonymous_agent: _AnonymousAgent, stage_display_level: StageDisplayLevel = StageDisplayLevel.INFO, @@ -42,10 +50,26 @@ def __init__( argument_transformers=argument_transformers, **kwargs, ) - self.__deferred_context = deferred_context + self.__catalog = catalog + self.__toolset_summaries = toolset_summaries + self.__definitions_by_name: dict[str, OpenAiToolConfigDict] = { + definition.name: definition.definition for definition in definitions + } self.__lazy_holder = lazy_holder self.__anonymous_agent = anonymous_agent + def enrich_openai_tool_schema(self, open_ai_tool: OpenAiToolConfig) -> OpenAiToolConfig: + summaries = self.__toolset_summaries + if not summaries: + return open_ai_tool + + open_ai_tool.function.description = ( + open_ai_tool.function.description + + "\n\nAdditional toolsets available for discovery:\n" + + format_toolset_summaries(summaries) + ) + return open_ai_tool + async def _run_in_stage_async( self, stage_wrapper: BaseStageWrapper | None = None, @@ -54,7 +78,7 @@ async def _run_in_stage_async( **kwargs: Any, ) -> ToolCallResult: query: str = kwargs.get("query", "") - catalog = self.__deferred_context.catalog + catalog = self.__catalog if not catalog: result = ToolCallResult( @@ -70,7 +94,7 @@ async def _run_in_stage_async( discovered: list[dict[str, str]] = [] new_definitions = [] for name in matched_names: - definition = self.__deferred_context.get_definition(name) + definition = self.__definitions_by_name.get(name) if definition is None: logger.warning("tool_search matched unknown tool name %r — skipping", name) continue diff --git a/src/quickapp/tool_discovery/_toolset_summary_format.py b/src/quickapp/tool_discovery/_toolset_summary_format.py new file mode 100644 index 00000000..852fd4ff --- /dev/null +++ b/src/quickapp/tool_discovery/_toolset_summary_format.py @@ -0,0 +1,11 @@ +def format_toolset_summaries(summaries: list[dict[str, str | int | None]]) -> str: + """Render deferred-toolset summaries as a bullet list: name, tool count, description.""" + lines = [_format_one(summary) for summary in summaries] + return "\n".join(lines) + + +def _format_one(summary: dict[str, str | int | None]) -> str: + line = f"- {summary['name']}. Available tools: {summary['tool_count']}." + if summary["description"]: + line += f" {summary['description']}" + return line diff --git a/src/quickapp/tool_discovery/tool_discovery_module.py b/src/quickapp/tool_discovery/tool_discovery_module.py index 19d9e7e7..16f8c461 100644 --- a/src/quickapp/tool_discovery/tool_discovery_module.py +++ b/src/quickapp/tool_discovery/tool_discovery_module.py @@ -4,10 +4,12 @@ from injector import AssistedBuilder, Binder, Module, multiprovider from quickapp.common import StagedBaseTool +from quickapp.common.abstract.base_prompt_provider import PromptPartProvider from quickapp.common.preview import preview_module from quickapp.config.application import ApplicationConfig from quickapp.tool_discovery._anonymous_agent import _AnonymousAgent from quickapp.tool_discovery._tool_configs import TOOL_SEARCH_TOOL_CONFIG +from quickapp.tool_discovery._tool_search_hint_prompt_provider import _ToolSearchHintPromptProvider from quickapp.tool_discovery._tool_search_stage_wrapper import _ToolSearchStageWrapper from quickapp.tool_discovery._tool_search_tool import _ToolSearchTool @@ -21,6 +23,15 @@ def configure(self, binder: Binder) -> None: binder.bind(_AnonymousAgent, to=_AnonymousAgent, scope=request_scope) binder.bind(_ToolSearchTool, to=_ToolSearchTool, scope=request_scope) binder.bind(_ToolSearchStageWrapper, to=_ToolSearchStageWrapper) + binder.bind( + _ToolSearchHintPromptProvider, to=_ToolSearchHintPromptProvider, scope=request_scope + ) + + @staticmethod + def _is_enabled(config: ApplicationConfig) -> bool: + return bool( + config.orchestrator.tool_discovery and config.orchestrator.tool_discovery.enabled + ) @multiprovider def _provide_tool_search_tools( @@ -28,7 +39,7 @@ def _provide_tool_search_tools( config: ApplicationConfig, tool_builder: AssistedBuilder[_ToolSearchTool], ) -> list[StagedBaseTool]: - if not config.orchestrator.tool_discovery or not config.orchestrator.tool_discovery.enabled: + if not self._is_enabled(config): return [] tool = tool_builder.build( @@ -36,3 +47,13 @@ def _provide_tool_search_tools( ) logger.debug("ToolDiscoveryModule: tool_search meta-tool registered") return [tool] + + @multiprovider + def _provide_prompt_parts( + self, + config: ApplicationConfig, + tool_search_hint: _ToolSearchHintPromptProvider, + ) -> list[PromptPartProvider]: + if not self._is_enabled(config): + return [] + return [tool_search_hint] diff --git a/src/tests/integration_tests/test_runner/mcp_server/mcp_http_test_server.py b/src/tests/integration_tests/test_runner/mcp_server/mcp_http_test_server.py index e3fdecdc..f08b3cdb 100644 --- a/src/tests/integration_tests/test_runner/mcp_server/mcp_http_test_server.py +++ b/src/tests/integration_tests/test_runner/mcp_server/mcp_http_test_server.py @@ -65,7 +65,7 @@ async def sum_integers(incoming: list[int]): @mcp.tool(description="Returns a predefined small picture") async def get_small_picture() -> Image: - return Image(path="auto.jpg") + return Image(path="./auto.jpg") def __get_file_data(file_path: str) -> str: @@ -87,7 +87,7 @@ async def get_test_pdf() -> EmbeddedResource: return EmbeddedResource( type="resource", resource=BlobResourceContents( - blob=__get_file_data("mcp_pdf.pdf"), + blob=__get_file_data("./mcp_pdf.pdf"), uri=AnyUrl("file://test/test.pdf"), mimeType="application/pdf", ), @@ -99,7 +99,7 @@ async def get_test_plotly() -> EmbeddedResource: return EmbeddedResource( type="resource", resource=BlobResourceContents( - blob=__get_file_data("plotly.json"), + blob=__get_file_data("./plotly.json"), uri=AnyUrl("file://test/plotly.json"), mimeType="application/vnd.plotly.v1+json", ), @@ -124,4 +124,6 @@ def get_config() -> dict: if __name__ == "__main__": + # result = asyncio.run(get_small_picture()) + # print(result) mcp.run(transport="streamable-http", host="0.0.0.0", port=8003, log_level="debug") diff --git a/src/tests/unit_tests/tool_discovery_tests/test_deferred_tools_context.py b/src/tests/unit_tests/common/test_deferred_tools_accumulator.py similarity index 63% rename from src/tests/unit_tests/tool_discovery_tests/test_deferred_tools_context.py rename to src/tests/unit_tests/common/test_deferred_tools_accumulator.py index 072800ba..83313670 100644 --- a/src/tests/unit_tests/tool_discovery_tests/test_deferred_tools_context.py +++ b/src/tests/unit_tests/common/test_deferred_tools_accumulator.py @@ -1,5 +1,6 @@ from unittest.mock import MagicMock +from quickapp.common.deferred_tools_accumulator import DeferredToolsAccumulator, is_toolset_deferred from quickapp.config.tool_discovery import ToolDiscoveryConfig from quickapp.config.tools.base import ( OpenAiToolConfig, @@ -16,10 +17,6 @@ ToolEndpointParamType, ) from quickapp.config.toolsets.rest_api import RestApiToolSet -from quickapp.shared.deferred_tools._deferred_tools_context import ( - DeferredToolsContext, - is_toolset_deferred, -) def _make_toolset(deferred: bool | None = None) -> RestApiToolSet: @@ -108,18 +105,18 @@ def _make_staged_tool(tool_config, enrich_side_effect=None) -> MagicMock: return staged_tool -class TestRegisterStagedTools: +class TestRegisterDeferredTools: def test_catalog_contains_name_and_description(self): - context = DeferredToolsContext() + context = DeferredToolsAccumulator() tool_config = _make_rest_api_tool_with_const_param() - context.register_staged_tools([_make_staged_tool(tool_config)]) + context.register_deferred_tools(_make_toolset(), [_make_staged_tool(tool_config)]) assert context.catalog == [{"name": "test_tool", "description": "A test tool"}] def test_deferred_names_reflects_registered_tools(self): - context = DeferredToolsContext() - context.register_staged_tools( - [_make_staged_tool(_make_rest_api_tool_with_const_param("tool_a"))] + context = DeferredToolsAccumulator() + context.register_deferred_tools( + _make_toolset(), [_make_staged_tool(_make_rest_api_tool_with_const_param("tool_a"))] ) assert context.deferred_names == frozenset({"tool_a"}) @@ -128,9 +125,9 @@ def test_definition_strips_const_params_same_as_eager_path(self): """Regression: a discovered tool's schema must match the eager path — const params (fixed values hidden from the LLM) must not leak into the schema surfaced via tool_search. """ - context = DeferredToolsContext() + context = DeferredToolsAccumulator() tool_config = _make_rest_api_tool_with_const_param() - context.register_staged_tools([_make_staged_tool(tool_config)]) + context.register_deferred_tools(_make_toolset(), [_make_staged_tool(tool_config)]) definition = context.get_definition("test_tool") @@ -147,24 +144,87 @@ def enrich(open_ai_tool: OpenAiToolConfig) -> OpenAiToolConfig: enriched.function.description = "enriched" return enriched - context = DeferredToolsContext() + context = DeferredToolsAccumulator() tool_config = _make_rest_api_tool_with_const_param() - context.register_staged_tools([_make_staged_tool(tool_config, enrich_side_effect=enrich)]) + context.register_deferred_tools( + _make_toolset(), [_make_staged_tool(tool_config, enrich_side_effect=enrich)] + ) definition = context.get_definition("test_tool") assert definition is not None assert definition["function"]["description"] == "enriched" def test_get_definition_returns_none_for_unknown_name(self): - context = DeferredToolsContext() + context = DeferredToolsAccumulator() assert context.get_definition("does_not_exist") is None def test_ignores_tools_without_openai_tool_config(self): - context = DeferredToolsContext() + context = DeferredToolsAccumulator() non_openai_staged_tool = MagicMock() non_openai_staged_tool.tool_config = MagicMock() # not a BaseOpenAITool instance - context.register_staged_tools([non_openai_staged_tool]) + context.register_deferred_tools(_make_toolset(), [non_openai_staged_tool]) assert context.catalog == [] assert context.deferred_names == frozenset() + + +class TestToolsetSummaries: + def test_summary_includes_name_description_and_tool_count(self): + context = DeferredToolsAccumulator() + toolset = RestApiToolSet( + name="salesforce", description="Query Salesforce records", tools=[] + ) + context.register_deferred_tools( + toolset, [_make_staged_tool(_make_rest_api_tool_with_const_param())] + ) + + assert context.toolset_summaries == [ + {"name": "salesforce", "description": "Query Salesforce records", "tool_count": 1} + ] + + def test_summary_is_name_only_when_description_is_none(self): + context = DeferredToolsAccumulator() + toolset = RestApiToolSet(name="salesforce", description=None, tools=[]) + context.register_deferred_tools( + toolset, [_make_staged_tool(_make_rest_api_tool_with_const_param())] + ) + + assert context.toolset_summaries == [ + {"name": "salesforce", "description": None, "tool_count": 1} + ] + + def test_tool_count_reflects_number_of_registered_tools(self): + context = DeferredToolsAccumulator() + toolset = RestApiToolSet( + name="salesforce", description="Query Salesforce records", tools=[] + ) + context.register_deferred_tools( + toolset, + [ + _make_staged_tool(_make_rest_api_tool_with_const_param("tool_a")), + _make_staged_tool(_make_rest_api_tool_with_const_param("tool_b")), + _make_staged_tool(_make_rest_api_tool_with_const_param("tool_c")), + ], + ) + + assert context.toolset_summaries[0]["tool_count"] == 3 + + def test_summaries_accumulate_across_multiple_toolsets(self): + context = DeferredToolsAccumulator() + context.register_deferred_tools( + RestApiToolSet(name="toolset_a", description="Does A", tools=[]), + [_make_staged_tool(_make_rest_api_tool_with_const_param("tool_a"))], + ) + context.register_deferred_tools( + RestApiToolSet(name="toolset_b", description=None, tools=[]), + [ + _make_staged_tool(_make_rest_api_tool_with_const_param("tool_b1")), + _make_staged_tool(_make_rest_api_tool_with_const_param("tool_b2")), + ], + ) + + assert context.toolset_summaries == [ + {"name": "toolset_a", "description": "Does A", "tool_count": 1}, + {"name": "toolset_b", "description": None, "tool_count": 2}, + ] diff --git a/src/tests/unit_tests/internal_tooling_tests/test_internal_tooling_module.py b/src/tests/unit_tests/internal_tooling_tests/test_internal_tooling_module.py index 230018d8..f767f03d 100644 --- a/src/tests/unit_tests/internal_tooling_tests/test_internal_tooling_module.py +++ b/src/tests/unit_tests/internal_tooling_tests/test_internal_tooling_module.py @@ -9,8 +9,8 @@ ) from quickapp.config.tools.internal import InternalTool from quickapp.config.toolsets.internal import InternalToolSet +from quickapp.internal_tooling._internal_deferred_tools_context import _InternalDeferredToolsContext from quickapp.internal_tooling.internal_tooling_module import InternalToolModule -from quickapp.shared.deferred_tools import DeferredToolsContext def _make_internal_tool_config( @@ -51,13 +51,13 @@ def test_registers_deferred_toolset_with_deferred_context(self): discovery_cfg = ToolDiscoveryConfig(enabled=True, min_tools_for_deferral=1) app_config = _make_app_config(toolset, discovery_cfg) staged_tool, py_builder = _make_py_builder(tool_config) - deferred_context = MagicMock(spec=DeferredToolsContext) + deferred_context = MagicMock(spec=_InternalDeferredToolsContext) module = InternalToolModule() result = module._provide_internal_tools(app_config, py_builder, deferred_context) assert result == [staged_tool] - deferred_context.register_staged_tools.assert_called_once_with([staged_tool]) + deferred_context.register_deferred_tools.assert_called_once_with(toolset, [staged_tool]) def test_does_not_defer_below_threshold(self): tool_config = _make_internal_tool_config() @@ -65,23 +65,23 @@ def test_does_not_defer_below_threshold(self): discovery_cfg = ToolDiscoveryConfig(enabled=True, min_tools_for_deferral=5) app_config = _make_app_config(toolset, discovery_cfg) staged_tool, py_builder = _make_py_builder(tool_config) - deferred_context = MagicMock(spec=DeferredToolsContext) + deferred_context = MagicMock(spec=_InternalDeferredToolsContext) module = InternalToolModule() result = module._provide_internal_tools(app_config, py_builder, deferred_context) assert result == [staged_tool] - deferred_context.register_staged_tools.assert_not_called() + deferred_context.register_deferred_tools.assert_not_called() def test_does_not_defer_when_discovery_disabled(self): tool_config = _make_internal_tool_config() toolset = InternalToolSet(name="internal", deferred=True, tools=[tool_config]) app_config = _make_app_config(toolset, discovery_cfg=None) staged_tool, py_builder = _make_py_builder(tool_config) - deferred_context = MagicMock(spec=DeferredToolsContext) + deferred_context = MagicMock(spec=_InternalDeferredToolsContext) module = InternalToolModule() result = module._provide_internal_tools(app_config, py_builder, deferred_context) assert result == [staged_tool] - deferred_context.register_staged_tools.assert_not_called() + deferred_context.register_deferred_tools.assert_not_called() diff --git a/src/tests/unit_tests/mcp_tool_tests/test_mcp_initializer_interactive_login.py b/src/tests/unit_tests/mcp_tool_tests/test_mcp_initializer_interactive_login.py index eb18827a..4ad2059c 100644 --- a/src/tests/unit_tests/mcp_tool_tests/test_mcp_initializer_interactive_login.py +++ b/src/tests/unit_tests/mcp_tool_tests/test_mcp_initializer_interactive_login.py @@ -129,7 +129,6 @@ def _make_initializer( login_service=login_service, accept_language=None, app_config=MagicMock(), - deferred_context=MagicMock(), ) return initializer, mcp_context, login_service diff --git a/src/tests/unit_tests/mcp_tool_tests/test_mcp_tool_initializer.py b/src/tests/unit_tests/mcp_tool_tests/test_mcp_tool_initializer.py index daab01b2..d0a13e22 100644 --- a/src/tests/unit_tests/mcp_tool_tests/test_mcp_tool_initializer.py +++ b/src/tests/unit_tests/mcp_tool_tests/test_mcp_tool_initializer.py @@ -235,7 +235,6 @@ def _create(protocol: MCPProtocol, allowed_tools=None, name="test_toolset"): MagicMock(), # login_service None, # accept_language _make_app_config_mock(), # app_config - MagicMock(), # deferred_context ) return initializer, mcp_context @@ -324,7 +323,6 @@ async def test_initialize_multiple_toolsets(tool1, tool2, builder_mock): MagicMock(), # login_service None, # accept_language _make_app_config_mock(), # app_config - MagicMock(), # deferred_context ) await initializer.initialize() @@ -415,7 +413,6 @@ async def test_no_exception_if_toolset_list_is_empty(): MagicMock(), # login_service None, # accept_language _make_app_config_mock(), # app_config - MagicMock(), # deferred_context ) await initializer.initialize() mcp_context.append_tool.assert_not_called() @@ -608,7 +605,6 @@ async def test_initialize_surfaces_session_terminated_through_nested_exception_g MagicMock(), # login_service None, # accept_language _make_app_config_mock(), # app_config - MagicMock(), # deferred_context ) await initializer.initialize() diff --git a/src/tests/unit_tests/rest_api_tooling_tests/test_rest_api_tool.py b/src/tests/unit_tests/rest_api_tooling_tests/test_rest_api_tool.py index 1e253fd1..11c79df7 100644 --- a/src/tests/unit_tests/rest_api_tooling_tests/test_rest_api_tool.py +++ b/src/tests/unit_tests/rest_api_tooling_tests/test_rest_api_tool.py @@ -6,7 +6,7 @@ from aidial_sdk.chat_completion import Attachment, Stage from fastapi_injector import Injected from httpx import QueryParams -from injector import Binder, Injector, InstanceProvider +from injector import Binder, InstanceProvider from pydantic import SecretStr from starlette.testclient import TestClient @@ -434,12 +434,18 @@ def configure(binder: Binder): binder.bind(ACCEPT_LANGUAGE, to=InstanceProvider(None)) binder.multibind(list[ToolArgumentTransformer], to=[]) - injector = Injector(modules=[RestApiToolingModule, configure]) - tools = injector.get(list[StagedBaseTool]) + app = create_test_app([RestApiToolingModule, configure]) + + @app.get("/") + async def get_method(tools: list[StagedBaseTool] = Injected(list[StagedBaseTool])): + assert len(tools) == 1 + tool_config: BaseOpenAITool = tools[0].tool_config + assert tool_config.open_ai_tool.function.name == f"{toolset_name}_{tool_name}" + return {} - assert len(tools) == 1 - tool_config: BaseOpenAITool = tools[0].tool_config - assert tool_config.open_ai_tool.function.name == f"{toolset_name}_{tool_name}" + client = TestClient(app) + response = client.get("/") + assert response.status_code == 200 @pytest.mark.asyncio diff --git a/src/tests/unit_tests/tool_discovery_tests/test_tool_discovery_module.py b/src/tests/unit_tests/tool_discovery_tests/test_tool_discovery_module.py new file mode 100644 index 00000000..e11506bd --- /dev/null +++ b/src/tests/unit_tests/tool_discovery_tests/test_tool_discovery_module.py @@ -0,0 +1,60 @@ +from unittest.mock import MagicMock + +from quickapp.tool_discovery._tool_search_hint_prompt_provider import _ToolSearchHintPromptProvider +from quickapp.tool_discovery.tool_discovery_module import ToolDiscoveryModule + + +def _make_config(enabled: bool | None) -> MagicMock: + config = MagicMock() + if enabled is None: + config.orchestrator.tool_discovery = None + else: + config.orchestrator.tool_discovery.enabled = enabled + return config + + +class TestProvidePromptParts: + def test_includes_hint_when_discovery_enabled(self): + module = ToolDiscoveryModule() + tool_search_hint = MagicMock(spec=_ToolSearchHintPromptProvider) + + result = module._provide_prompt_parts(_make_config(True), tool_search_hint) + + assert result == [tool_search_hint] + + def test_omits_hint_when_discovery_disabled(self): + module = ToolDiscoveryModule() + tool_search_hint = MagicMock(spec=_ToolSearchHintPromptProvider) + + result = module._provide_prompt_parts(_make_config(False), tool_search_hint) + + assert result == [] + + def test_omits_hint_when_discovery_unset(self): + module = ToolDiscoveryModule() + tool_search_hint = MagicMock(spec=_ToolSearchHintPromptProvider) + + result = module._provide_prompt_parts(_make_config(None), tool_search_hint) + + assert result == [] + + +class TestProvideToolSearchTools: + def test_returns_empty_list_when_discovery_disabled(self): + module = ToolDiscoveryModule() + tool_builder = MagicMock() + + result = module._provide_tool_search_tools(_make_config(False), tool_builder) + + assert result == [] + tool_builder.build.assert_not_called() + + def test_builds_tool_when_discovery_enabled(self): + module = ToolDiscoveryModule() + tool_builder = MagicMock() + built_tool = MagicMock() + tool_builder.build.return_value = built_tool + + result = module._provide_tool_search_tools(_make_config(True), tool_builder) + + assert result == [built_tool] diff --git a/src/tests/unit_tests/tool_discovery_tests/test_tool_search_hint_prompt_provider.py b/src/tests/unit_tests/tool_discovery_tests/test_tool_search_hint_prompt_provider.py new file mode 100644 index 00000000..65785f7c --- /dev/null +++ b/src/tests/unit_tests/tool_discovery_tests/test_tool_search_hint_prompt_provider.py @@ -0,0 +1,43 @@ +import pytest + +from quickapp.tool_discovery._tool_search_hint_prompt_provider import _ToolSearchHintPromptProvider + + +def _make_provider( + toolset_summaries: list[dict[str, str | int | None]], +) -> _ToolSearchHintPromptProvider: + return _ToolSearchHintPromptProvider(toolset_summaries) + + +@pytest.mark.asyncio +async def test_get_prompt_part_mentions_internal_tool_search(): + provider = _make_provider([]) + + part = await provider.get_prompt_part() + + assert "internal_tool_search" in part + + +@pytest.mark.asyncio +async def test_get_prompt_part_omits_toolset_list_when_nothing_deferred(): + provider = _make_provider([]) + + part = await provider.get_prompt_part() + + assert "Additional toolsets available for discovery" not in part + + +@pytest.mark.asyncio +async def test_get_prompt_part_appends_deferred_toolset_summaries(): + provider = _make_provider( + [ + {"name": "salesforce", "description": "Query Salesforce records", "tool_count": 12}, + {"name": "internal-utils", "description": None, "tool_count": 1}, + ] + ) + + part = await provider.get_prompt_part() + + assert "Additional toolsets available for discovery:" in part + assert "- salesforce. Available tools: 12. Query Salesforce records" in part + assert "- internal-utils. Available tools: 1." in part diff --git a/src/tests/unit_tests/tool_discovery_tests/test_tool_search_tool.py b/src/tests/unit_tests/tool_discovery_tests/test_tool_search_tool.py new file mode 100644 index 00000000..a7f48a2b --- /dev/null +++ b/src/tests/unit_tests/tool_discovery_tests/test_tool_search_tool.py @@ -0,0 +1,87 @@ +from unittest.mock import MagicMock + +from quickapp.config.tools.base import ( + OpenAiToolConfig, + OpenAiToolFunction, + OpenAiToolFunctionParameters, +) +from quickapp.tool_discovery._tool_search_tool import _ToolSearchTool + + +def _make_open_ai_tool(description: str = "Search for additional tools.") -> OpenAiToolConfig: + return OpenAiToolConfig( + function=OpenAiToolFunction( + name="internal_tool_search", + description=description, + parameters=OpenAiToolFunctionParameters(type="object", properties={}), + ) + ) + + +def _make_tool_search_tool(toolset_summaries: list[dict[str, str | int | None]]) -> _ToolSearchTool: + return _ToolSearchTool( + stage_wrapper_builder=MagicMock(), + tool_config=MagicMock(), + perf_timer=MagicMock(), + catalog=[], + definitions=[], + toolset_summaries=toolset_summaries, + lazy_holder=MagicMock(), + anonymous_agent=MagicMock(), + ) + + +class TestEnrichOpenAiToolSchema: + def test_no_op_when_no_deferred_toolsets(self): + tool = _make_tool_search_tool([]) + open_ai_tool = _make_open_ai_tool() + + result = tool.enrich_openai_tool_schema(open_ai_tool) + + assert result.function.description == "Search for additional tools." + + def test_appends_toolset_with_description_and_tool_count(self): + tool = _make_tool_search_tool( + [{"name": "salesforce", "description": "Query Salesforce records", "tool_count": 12}] + ) + open_ai_tool = _make_open_ai_tool("Search for additional tools.") + + result = tool.enrich_openai_tool_schema(open_ai_tool) + + assert result.function.description == ( + "Search for additional tools.\n\n" + "Additional toolsets available for discovery:\n" + "- salesforce. Available tools: 12. Query Salesforce records" + ) + + def test_appends_toolset_name_and_count_only_when_description_is_none(self): + tool = _make_tool_search_tool( + [{"name": "internal-utils", "description": None, "tool_count": 1}] + ) + open_ai_tool = _make_open_ai_tool("Search for additional tools.") + + result = tool.enrich_openai_tool_schema(open_ai_tool) + + assert result.function.description == ( + "Search for additional tools.\n\n" + "Additional toolsets available for discovery:\n" + "- internal-utils. Available tools: 1." + ) + + def test_appends_multiple_toolsets_in_order(self): + tool = _make_tool_search_tool( + [ + {"name": "toolset_a", "description": "Does A", "tool_count": 3}, + {"name": "toolset_b", "description": None, "tool_count": 7}, + ] + ) + open_ai_tool = _make_open_ai_tool("Search for additional tools.") + + result = tool.enrich_openai_tool_schema(open_ai_tool) + + assert result.function.description == ( + "Search for additional tools.\n\n" + "Additional toolsets available for discovery:\n" + "- toolset_a. Available tools: 3. Does A\n" + "- toolset_b. Available tools: 7." + ) diff --git a/src/tests/unit_tests/tool_discovery_tests/test_toolset_summary_format.py b/src/tests/unit_tests/tool_discovery_tests/test_toolset_summary_format.py new file mode 100644 index 00000000..1e6eff36 --- /dev/null +++ b/src/tests/unit_tests/tool_discovery_tests/test_toolset_summary_format.py @@ -0,0 +1,32 @@ +from quickapp.tool_discovery._toolset_summary_format import format_toolset_summaries + + +def test_formats_summary_with_description(): + result = format_toolset_summaries( + [{"name": "salesforce", "description": "Query Salesforce records", "tool_count": 12}] + ) + + assert result == "- salesforce. Available tools: 12. Query Salesforce records" + + +def test_formats_summary_without_description(): + result = format_toolset_summaries( + [{"name": "internal-utils", "description": None, "tool_count": 1}] + ) + + assert result == "- internal-utils. Available tools: 1." + + +def test_formats_multiple_summaries_in_order(): + result = format_toolset_summaries( + [ + {"name": "toolset_a", "description": "Does A", "tool_count": 3}, + {"name": "toolset_b", "description": None, "tool_count": 7}, + ] + ) + + assert result == "- toolset_a. Available tools: 3. Does A\n- toolset_b. Available tools: 7." + + +def test_formats_empty_list(): + assert format_toolset_summaries([]) == "" From df03e633a60bb259ccf7319262fcaa654938dc97 Mon Sep 17 00:00:00 2001 From: Oleksandr Gubarets Date: Tue, 15 Sep 2026 14:38:55 +0300 Subject: [PATCH 09/10] code review fix. --- CONFIGURATION.md | 2 +- src/quickapp/config/tool_discovery.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/CONFIGURATION.md b/CONFIGURATION.md index 73cb8d98..2d9d4f62 100644 --- a/CONFIGURATION.md +++ b/CONFIGURATION.md @@ -343,7 +343,7 @@ tool schemas through an isolated LLM call. |------------------------|----------|---------|-------------------------------------------------------------------------------------------------------------------------------------------------------------|-------------------|---------------| | enabled | No | Boolean | Enable dynamic tool discovery. When `true`, toolsets with `deferred: true` are withheld from the initial LLM payload and surfaced via the `internal_tool_search` meta-tool. | - | `false` | | service_model | No | String | DIAL deployment used for the anonymous routing call inside `internal_tool_search`. Falls back to the orchestrator's own deployment when omitted. | - | - | -| min_tools_for_deferral | No | Integer | Minimum number of tools in a toolset for deferral to apply. Toolsets smaller than this threshold are promoted to eager loading even when `deferred: true`. Deployment-wide default set by `MIN_TOOLS_FOR_DEFERRAL`. | - | `5` | +| min_tools_for_deferral | No | Integer | Minimum number of tools in a toolset for deferral to apply. Toolsets smaller than this threshold are promoted to eager loading even when `deferred: true`. Deployment-wide default set by `MIN_TOOLS_FOR_DEFERRAL`. | - | `10` |
Tool discovery configuration JSON sample diff --git a/src/quickapp/config/tool_discovery.py b/src/quickapp/config/tool_discovery.py index 2c39a4b2..ea26603f 100644 --- a/src/quickapp/config/tool_discovery.py +++ b/src/quickapp/config/tool_discovery.py @@ -7,7 +7,7 @@ class ToolDiscoverySettings(BaseSettings): model_config = SettingsConfigDict() min_tools_for_deferral: int = Field( - default=5, + default=10, ge=1, description="Minimum toolset size for deferral to apply deployment-wide.", alias="MIN_TOOLS_FOR_DEFERRAL", @@ -19,11 +19,11 @@ def _min_tools_for_deferral_field() -> FieldInfo: "Minimum number of tools in a toolset for deferral to apply. " "Toolsets with fewer tools than this threshold are promoted to eager loading " "even when deferred=true, avoiding discovery overhead for small toolsets. " - "Default: 5 (or the value of MIN_TOOLS_FOR_DEFERRAL env var)" + "Default: 10 (or the value of MIN_TOOLS_FOR_DEFERRAL env var)" ) return Field( # type: ignore[return-value] default_factory=lambda: ToolDiscoverySettings().min_tools_for_deferral, - json_schema_extra={"default": 5}, + json_schema_extra={"default": 10}, ge=1, description=description, ) From a51b548fbe61d8c219426b525e70a7cad7a30ea4 Mon Sep 17 00:00:00 2001 From: Oleksandr Gubarets Date: Tue, 15 Sep 2026 14:54:07 +0300 Subject: [PATCH 10/10] code review fix. --- README.md | 132 +++++++++--------- docs/generated-app-schema.json | 4 +- .../agent/_chat_completion_config_builder.py | 62 ++++---- 3 files changed, 106 insertions(+), 92 deletions(-) diff --git a/README.md b/README.md index 00b89545..9ade1bfb 100644 --- a/README.md +++ b/README.md @@ -134,11 +134,11 @@ Toolsets are deferred by default — omitting `deferred` or setting it to `true` Key fields: | Field | Default | Description | -|---|---|---| +|---|---------|---| | `orchestrator.tool_discovery.enabled` | `false` | Activates dynamic discovery for this app. Must be `true` for deferral to take effect. | -| `orchestrator.tool_discovery.service_model` | — | DIAL deployment used for the anonymous routing call inside `internal_tool_search`. Falls back to the orchestrator's own deployment when omitted. | -| `orchestrator.tool_discovery.min_tools_for_deferral` | `5` | Minimum number of tools in a toolset for deferral to apply. Toolsets smaller than this threshold are promoted to eager loading even when `deferred: true`. | -| `.deferred` | `true` | Per-toolset opt-out. Set to `false` to force a specific toolset into the initial payload regardless of `tool_discovery.enabled`. | +| `orchestrator.tool_discovery.service_model` | — | DIAL deployment used for the anonymous routing call inside `internal_tool_search`. Falls back to the orchestrator's own deployment when omitted. | +| `orchestrator.tool_discovery.min_tools_for_deferral` | `10` | Minimum number of tools in a toolset for deferral to apply. Toolsets smaller than this threshold are promoted to eager loading even when `deferred: true`. | +| `.deferred` | `true` | Per-toolset opt-out. Set to `false` to force a specific toolset into the initial payload regardless of `tool_discovery.enabled`. | The `MIN_TOOLS_FOR_DEFERRAL` environment variable sets the deployment-wide default for `min_tools_for_deferral`; individual apps can override it in their manifest. @@ -180,69 +180,69 @@ Controls which tool-execution stages are surfaced in the DIAL UI for each app. S ### Environment Variables -| Variable | Default | Required | Description | -|--------------------------------------------|----------------------------|----------|----------------------------------------------------------------------------------------------------------------| -| **DIAL Core** | | | | -| `DIAL_URL` | — | Yes | URL of the DIAL Core API | -| `DIAL_API_VERSION` | `2025-01-01-preview` | No | API version for DIAL Core API | +| Variable | Default | Required | Description | +|--------------------------------------------|-----------------------------------------------------------------|----------|----------------------------------------------------------------------------------------------------------------| +| **DIAL Core** | | | | +| `DIAL_URL` | — | Yes | URL of the DIAL Core API | +| `DIAL_API_VERSION` | `2025-01-01-preview` | No | API version for DIAL Core API | | `APP_SCHEMA_ID` | `https://mydial.epam.com/custom_application_schemas/quickapps2` | No | Full application type schema `$id` emitted in the generated app schema. When unset, the built-in default is used. | -| **Proxy** | | | | -| `PROXY_LANGUAGE_HEADER` | `accept-language` | No | Name of the incoming HTTP request header that carries the locale for UI display (stage name localization). Override when a reverse proxy rewrites the standard `Accept-Language` header before forwarding the request. | -| **Logging** | | | | -| `DIAL_SDK_LOG_FORMAT` | `text` | No | Console log output format: `text` (human-readable) or `json` (escape-safe, one record per line). See [docs/logging.md](docs/logging.md). | -| `DIAL_SDK_TEXT_LOG_FORMAT` | [see docs/logging.md](docs/logging.md) | No | Custom `%`-style format string for `text` output. Unset (default) keeps the built-in format with the conditional OTEL trace block. | -| `DIAL_SDK_JSON_LOG_FORMAT` | [see docs/logging.md](docs/logging.md) | No | Custom template for `json` output — a JSON document whose string leaves are `%`-style format strings, values escaped via `json.dumps`. | -| `LOG_LEVEL` | `INFO` | No | Root logger level (all loggers except quickapp) | -| `QUICKAPP_LOG_LEVEL` | `INFO` | No | Log level for quickapp loggers | -| `LOG_PAYLOADS` | `false` | No | Emit payload content (message bodies, tool-call arguments, tool/LLM response bodies) at DEBUG. When `false`, no payload content is logged at **any** level and the payload-capable third-party loggers (`openai`/`httpx`/`httpcore`) are capped at INFO. **Local development only** — see [Payload Logging](#payload-logging). | -| `LOG_PAYLOADS_MAX_LENGTH` | `2000` | No | Per-field character cap applied to each payload value when `LOG_PAYLOADS=true`; longer values are truncated. Inert when `LOG_PAYLOADS=false`. | -| **Agent** | | | | -| `DEFAULT_AGENT_MAX_ITERATIONS` | `15` | No | Maximum number of orchestrator iterations (`-1` for infinite) | -| `DEFAULT_ORCHESTRATOR_DEPLOYMENT_ID` | — | No | Default DIAL deployment id used as the orchestrator model when a QuickApp manifest omits `orchestrator.deployment`. Also surfaces as the JSON-schema `default` for that field so DIAL Core can pre-fill new manifests. Apps can override per-app. | -| `SHOW_USAGE_STATISTICS` | `false` | No | Include usage statistics in chat completion stream | -| `SHOW_EXECUTION_TIME_STAGE` | `false` | No | Show execution time stage in the UI | -| **Python Interpreter** | | | | -| `PY_INTERPRETER_LOCAL_RUN` | `false` | No | Run PyInterpreter locally instead of via DIAL Core API | -| `PY_INTERPRETER_URL` | *(falls back to DIAL_URL)* | No | URL of the PyInterpreter service | -| `PY_INTERPRETER_API_KEY` | — | No | API key for local-run PyInterpreter | -| `PY_INTERPRETER_DEFAULT_SESSION_ID` | — | No | Default session ID for the PyInterpreter | -| `PY_INTERPRETER_CLIENT_MAX_RETRIES` | `3` | No | Max retries for PyInterpreter client requests | -| **Tool Timeouts** | | | | -| `DEFAULT_TOOL_TIMEOUT_SECONDS` | `300.0` | No | Deployment-wide default timeout (seconds, `0 < x ≤ 3600`) applied to every tool call (deployment, REST API, MCP, Python interpreter). Apps can override per-app via `tool_defaults.timeout_seconds`. | -| `DEFAULT_FILE_LOADING_SIZE_LIMIT` | `10485760` | No | Deployment-wide default maximum size (in bytes) for files the agent downloads. Apps can override per-app via `features.file_loading.size_limit`. | -| **Stage Display** | | | | -| `DEFAULT_STAGE_DISPLAY_LEVEL` | — | No | Deployment-wide override for stage visibility threshold (`none`, `error`, `info`, `debug`; case-insensitive). When set, wins over every app's `features.stage_display.level`. Unset (default) defers to the per-app config, which defaults to `info`. | -| **DIAL Files — Tool-Response Offload** | | | | -| `TOOL_CALL_RESULT_OFFLOAD__ENABLED_BY_DEFAULT` | `true` | No | Default value of the per-app `enabled` flag (`features.dial_files.tool_call_result_offload.enabled`). Apps override per-app; `enabled: false` disables offload for that app. | -| `TOOL_CALL_RESULT_OFFLOAD__SIZE_THRESHOLD` | `40000` | No | Default byte threshold above which a tool-call response is offloaded to a DIAL file. Apps override per-app via `features.dial_files.tool_call_result_offload.size_threshold`. | -| `TOOL_CALL_RESULT_OFFLOAD__EXCLUDED_TOOLS` | `[]` | No | Default JSON list of **additional** tool names exempt from offloading. The read-back tools (`internal_file_read_lines`, `internal_file_search`) are always excluded regardless of this value, so a large read-back slice is never re-offloaded. Apps add more per-app via `features.dial_files.tool_call_result_offload.excluded_tools`. | -| **External URL Egress** | | | | -| `EXTERNAL_URL_FETCH_ENABLED` | `false` | No | Admin cap on fetching external (non-DIAL) URLs. When `false` (default), no app may fetch external URLs regardless of its manifest; the deployment-handoff branch (deployments with `features.url_attachments`) is unaffected. Apps can opt out per-app via `features.external_url_fetch.enabled=false` even when the admin allows. | -| `EXTERNAL_URL_FETCH_HOST_ALLOWLIST` | — | No | Comma-separated allowlist of host patterns for external URL fetches. Unset (default) means no admin-level host restriction. Patterns: exact host (`example.com`) or `*.example.com` for any subdomain. Re-checked on every redirect hop. Per-app `features.external_url_fetch.host_allowlist` narrows further (intersection) but never expands. | -| `EXTERNAL_URL_FETCH_MAX_REDIRECTS` | `5` | No | Maximum HTTP redirects on external URL fetches. Each hop is SSRF-checked. Hard ceiling 10. | -| `EXTERNAL_URL_FETCH_CONNECT_TIMEOUT_SECONDS` | `5.0` | No | TCP connect timeout (seconds) for external URL fetches. Read/write/pool timeouts use the resolved tool timeout. | -| **Dynamic Tool Discovery** `[Preview]` | | | | -| `MIN_TOOLS_FOR_DEFERRAL` | `5` | No | Deployment-wide minimum toolset size for deferral to apply. Toolsets with fewer tools than this threshold are promoted to eager loading even when `deferred=true`. Apps override per-app via `orchestrator.tool_discovery.min_tools_for_deferral`. Requires `ENABLE_PREVIEW_FEATURES=true`. | -| **Skills** | | | | -| `DIAL_SKILLS_FILE_MAX_BYTES` | `262144` | No | Cap on a single file read from a DIAL skill resource, `SKILL.md` included. Must exceed the largest manifest you expect: an over-cap manifest drops the skill. See [docs/skills.md](docs/skills.md). | -| `DIAL_SKILLS_MAX_FILES` | `200` | No | Maximum bundled files advertised to the agent per DIAL skill resource; beyond it the listing is truncated | -| `DIAL_SKILLS_LISTING_MAX_PAGES` | `10` | No | Maximum file-listing pages followed per DIAL skill resource, bounding a server-supplied cursor | -| `SKILL_INVOCATION_MAX_SKILLS` | `10` | No | Maximum distinct skills a user may have invoked from the messages of one conversation (`custom_content.skills`), counted newest first. Each one adds a `` block to the system prompt and one DIAL Core fetch per turn; beyond the cap the oldest picks stop being registered. Preview-gated. See [docs/skills.md](docs/skills.md). | -| **Feature Gating** | | | | -| `ENABLE_PREVIEW_FEATURES` | `false` | No | Enable preview features across the deployment (schema visibility + runtime activation) | -| **Templates** | | | | -| `PREDEFINED_EXTRA_PATHS` | — | No | JSON list of directories layered on top of built-in predefined content (later entries override earlier ones) | -| `CONFIG_PROMPT_MAPPING` | *(built-in mapping)* | No | JSON mapping of predefined system prompts to DIAL Core deployments | -| **Observability** | | | | -| `OTEL_SERVICE_NAME` | `quickapps` | No | Service name stamped on all exported telemetry (traces, metrics, logs) | -| `OTEL_TRACES_EXPORTER` | — | No | Set to `otlp` to enable tracing and export spans over OTLP/gRPC. Instruments the FastAPI server and outgoing HTTP clients (`httpx`, `requests`, `aiohttp`, `urllib`) and stamps trace context onto log records — see [docs/logging.md](docs/logging.md). | -| `OTEL_METRICS_EXPORTER` | — | No | Comma-separated metric exporters: `otlp` (push over OTLP/gRPC) and/or `prometheus` (serve a scrape endpoint). Enables FastAPI and system/process metrics. | -| `OTEL_LOGS_EXPORTER` | — | No | Set to `otlp` to export log records (INFO and above) over OTLP/gRPC alongside console output — see [docs/logging.md](docs/logging.md). | -| `OTEL_EXPORTER_OTLP_ENDPOINT` | `http://localhost:4317` | No | OTLP/gRPC collector endpoint shared by trace, metric, and log export. One of the [standard OpenTelemetry SDK variables](https://opentelemetry.io/docs/specs/otel/configuration/sdk-environment-variables/), which the underlying exporters honor as usual (per-signal endpoints, headers, timeouts, resource attributes, …). | -| `OTEL_EXPORTER_PROMETHEUS_PORT` | `9464` | No | Port of the Prometheus scrape endpoint (effective only with `prometheus` in `OTEL_METRICS_EXPORTER`) | -| **Scripts & Tests** | | | | -| `REMOTE_DIAL_URL` | — | No | URL of the remote DIAL Core, used only by `generate_dial_config` script and e2e/integration tests | -| `REMOTE_DIAL_API_KEY` | — | No | API key of the remote DIAL Core, used only by `generate_dial_config` script and e2e/integration tests | +| **Proxy** | | | | +| `PROXY_LANGUAGE_HEADER` | `accept-language` | No | Name of the incoming HTTP request header that carries the locale for UI display (stage name localization). Override when a reverse proxy rewrites the standard `Accept-Language` header before forwarding the request. | +| **Logging** | | | | +| `DIAL_SDK_LOG_FORMAT` | `text` | No | Console log output format: `text` (human-readable) or `json` (escape-safe, one record per line). See [docs/logging.md](docs/logging.md). | +| `DIAL_SDK_TEXT_LOG_FORMAT` | [see docs/logging.md](docs/logging.md) | No | Custom `%`-style format string for `text` output. Unset (default) keeps the built-in format with the conditional OTEL trace block. | +| `DIAL_SDK_JSON_LOG_FORMAT` | [see docs/logging.md](docs/logging.md) | No | Custom template for `json` output — a JSON document whose string leaves are `%`-style format strings, values escaped via `json.dumps`. | +| `LOG_LEVEL` | `INFO` | No | Root logger level (all loggers except quickapp) | +| `QUICKAPP_LOG_LEVEL` | `INFO` | No | Log level for quickapp loggers | +| `LOG_PAYLOADS` | `false` | No | Emit payload content (message bodies, tool-call arguments, tool/LLM response bodies) at DEBUG. When `false`, no payload content is logged at **any** level and the payload-capable third-party loggers (`openai`/`httpx`/`httpcore`) are capped at INFO. **Local development only** — see [Payload Logging](#payload-logging). | +| `LOG_PAYLOADS_MAX_LENGTH` | `2000` | No | Per-field character cap applied to each payload value when `LOG_PAYLOADS=true`; longer values are truncated. Inert when `LOG_PAYLOADS=false`. | +| **Agent** | | | | +| `DEFAULT_AGENT_MAX_ITERATIONS` | `15` | No | Maximum number of orchestrator iterations (`-1` for infinite) | +| `DEFAULT_ORCHESTRATOR_DEPLOYMENT_ID` | — | No | Default DIAL deployment id used as the orchestrator model when a QuickApp manifest omits `orchestrator.deployment`. Also surfaces as the JSON-schema `default` for that field so DIAL Core can pre-fill new manifests. Apps can override per-app. | +| `SHOW_USAGE_STATISTICS` | `false` | No | Include usage statistics in chat completion stream | +| `SHOW_EXECUTION_TIME_STAGE` | `false` | No | Show execution time stage in the UI | +| **Python Interpreter** | | | | +| `PY_INTERPRETER_LOCAL_RUN` | `false` | No | Run PyInterpreter locally instead of via DIAL Core API | +| `PY_INTERPRETER_URL` | *(falls back to DIAL_URL)* | No | URL of the PyInterpreter service | +| `PY_INTERPRETER_API_KEY` | — | No | API key for local-run PyInterpreter | +| `PY_INTERPRETER_DEFAULT_SESSION_ID` | — | No | Default session ID for the PyInterpreter | +| `PY_INTERPRETER_CLIENT_MAX_RETRIES` | `3` | No | Max retries for PyInterpreter client requests | +| **Tool Timeouts** | | | | +| `DEFAULT_TOOL_TIMEOUT_SECONDS` | `300.0` | No | Deployment-wide default timeout (seconds, `0 < x ≤ 3600`) applied to every tool call (deployment, REST API, MCP, Python interpreter). Apps can override per-app via `tool_defaults.timeout_seconds`. | +| `DEFAULT_FILE_LOADING_SIZE_LIMIT` | `10485760` | No | Deployment-wide default maximum size (in bytes) for files the agent downloads. Apps can override per-app via `features.file_loading.size_limit`. | +| **Stage Display** | | | | +| `DEFAULT_STAGE_DISPLAY_LEVEL` | — | No | Deployment-wide override for stage visibility threshold (`none`, `error`, `info`, `debug`; case-insensitive). When set, wins over every app's `features.stage_display.level`. Unset (default) defers to the per-app config, which defaults to `info`. | +| **DIAL Files — Tool-Response Offload** | | | | +| `TOOL_CALL_RESULT_OFFLOAD__ENABLED_BY_DEFAULT` | `true` | No | Default value of the per-app `enabled` flag (`features.dial_files.tool_call_result_offload.enabled`). Apps override per-app; `enabled: false` disables offload for that app. | +| `TOOL_CALL_RESULT_OFFLOAD__SIZE_THRESHOLD` | `40000` | No | Default byte threshold above which a tool-call response is offloaded to a DIAL file. Apps override per-app via `features.dial_files.tool_call_result_offload.size_threshold`. | +| `TOOL_CALL_RESULT_OFFLOAD__EXCLUDED_TOOLS` | `[]` | No | Default JSON list of **additional** tool names exempt from offloading. The read-back tools (`internal_file_read_lines`, `internal_file_search`) are always excluded regardless of this value, so a large read-back slice is never re-offloaded. Apps add more per-app via `features.dial_files.tool_call_result_offload.excluded_tools`. | +| **External URL Egress** | | | | +| `EXTERNAL_URL_FETCH_ENABLED` | `false` | No | Admin cap on fetching external (non-DIAL) URLs. When `false` (default), no app may fetch external URLs regardless of its manifest; the deployment-handoff branch (deployments with `features.url_attachments`) is unaffected. Apps can opt out per-app via `features.external_url_fetch.enabled=false` even when the admin allows. | +| `EXTERNAL_URL_FETCH_HOST_ALLOWLIST` | — | No | Comma-separated allowlist of host patterns for external URL fetches. Unset (default) means no admin-level host restriction. Patterns: exact host (`example.com`) or `*.example.com` for any subdomain. Re-checked on every redirect hop. Per-app `features.external_url_fetch.host_allowlist` narrows further (intersection) but never expands. | +| `EXTERNAL_URL_FETCH_MAX_REDIRECTS` | `5` | No | Maximum HTTP redirects on external URL fetches. Each hop is SSRF-checked. Hard ceiling 10. | +| `EXTERNAL_URL_FETCH_CONNECT_TIMEOUT_SECONDS` | `5.0` | No | TCP connect timeout (seconds) for external URL fetches. Read/write/pool timeouts use the resolved tool timeout. | +| **Dynamic Tool Discovery** `[Preview]` | | | | +| `MIN_TOOLS_FOR_DEFERRAL` | `10` | No | Deployment-wide minimum toolset size for deferral to apply. Toolsets with fewer tools than this threshold are promoted to eager loading even when `deferred=true`. Apps override per-app via `orchestrator.tool_discovery.min_tools_for_deferral`. Requires `ENABLE_PREVIEW_FEATURES=true`. | +| **Skills** | | | | +| `DIAL_SKILLS_FILE_MAX_BYTES` | `262144` | No | Cap on a single file read from a DIAL skill resource, `SKILL.md` included. Must exceed the largest manifest you expect: an over-cap manifest drops the skill. See [docs/skills.md](docs/skills.md). | +| `DIAL_SKILLS_MAX_FILES` | `200` | No | Maximum bundled files advertised to the agent per DIAL skill resource; beyond it the listing is truncated | +| `DIAL_SKILLS_LISTING_MAX_PAGES` | `10` | No | Maximum file-listing pages followed per DIAL skill resource, bounding a server-supplied cursor | +| `SKILL_INVOCATION_MAX_SKILLS` | `10` | No | Maximum distinct skills a user may have invoked from the messages of one conversation (`custom_content.skills`), counted newest first. Each one adds a `` block to the system prompt and one DIAL Core fetch per turn; beyond the cap the oldest picks stop being registered. Preview-gated. See [docs/skills.md](docs/skills.md). | +| **Feature Gating** | | | | +| `ENABLE_PREVIEW_FEATURES` | `false` | No | Enable preview features across the deployment (schema visibility + runtime activation) | +| **Templates** | | | | +| `PREDEFINED_EXTRA_PATHS` | — | No | JSON list of directories layered on top of built-in predefined content (later entries override earlier ones) | +| `CONFIG_PROMPT_MAPPING` | *(built-in mapping)* | No | JSON mapping of predefined system prompts to DIAL Core deployments | +| **Observability** | | | | +| `OTEL_SERVICE_NAME` | `quickapps` | No | Service name stamped on all exported telemetry (traces, metrics, logs) | +| `OTEL_TRACES_EXPORTER` | — | No | Set to `otlp` to enable tracing and export spans over OTLP/gRPC. Instruments the FastAPI server and outgoing HTTP clients (`httpx`, `requests`, `aiohttp`, `urllib`) and stamps trace context onto log records — see [docs/logging.md](docs/logging.md). | +| `OTEL_METRICS_EXPORTER` | — | No | Comma-separated metric exporters: `otlp` (push over OTLP/gRPC) and/or `prometheus` (serve a scrape endpoint). Enables FastAPI and system/process metrics. | +| `OTEL_LOGS_EXPORTER` | — | No | Set to `otlp` to export log records (INFO and above) over OTLP/gRPC alongside console output — see [docs/logging.md](docs/logging.md). | +| `OTEL_EXPORTER_OTLP_ENDPOINT` | `http://localhost:4317` | No | OTLP/gRPC collector endpoint shared by trace, metric, and log export. One of the [standard OpenTelemetry SDK variables](https://opentelemetry.io/docs/specs/otel/configuration/sdk-environment-variables/), which the underlying exporters honor as usual (per-signal endpoints, headers, timeouts, resource attributes, …). | +| `OTEL_EXPORTER_PROMETHEUS_PORT` | `9464` | No | Port of the Prometheus scrape endpoint (effective only with `prometheus` in `OTEL_METRICS_EXPORTER`) | +| **Scripts & Tests** | | | | +| `REMOTE_DIAL_URL` | — | No | URL of the remote DIAL Core, used only by `generate_dial_config` script and e2e/integration tests | +| `REMOTE_DIAL_API_KEY` | — | No | API key of the remote DIAL Core, used only by `generate_dial_config` script and e2e/integration tests | #### Deprecated Environment Variables diff --git a/docs/generated-app-schema.json b/docs/generated-app-schema.json index 2aa2453a..5d7c12e4 100644 --- a/docs/generated-app-schema.json +++ b/docs/generated-app-schema.json @@ -3512,8 +3512,8 @@ "title": "Service Model" }, "min_tools_for_deferral": { - "default": 5, - "description": "Minimum number of tools in a toolset for deferral to apply. Toolsets with fewer tools than this threshold are promoted to eager loading even when deferred=true, avoiding discovery overhead for small toolsets. Default: 5 (or the value of MIN_TOOLS_FOR_DEFERRAL env var)", + "default": 10, + "description": "Minimum number of tools in a toolset for deferral to apply. Toolsets with fewer tools than this threshold are promoted to eager loading even when deferred=true, avoiding discovery overhead for small toolsets. Default: 10 (or the value of MIN_TOOLS_FOR_DEFERRAL env var)", "minimum": 1, "title": "Min Tools For Deferral", "type": "integer" diff --git a/src/quickapp/core/agent/_chat_completion_config_builder.py b/src/quickapp/core/agent/_chat_completion_config_builder.py index 19d9e35b..32208738 100644 --- a/src/quickapp/core/agent/_chat_completion_config_builder.py +++ b/src/quickapp/core/agent/_chat_completion_config_builder.py @@ -45,13 +45,7 @@ def build(self, messages: list[Message]) -> dict[str, Any]: exclude_none=True ) prepared_messages = self._prepare_messages(messages) - eager_names: set[str] = {t.get("function", {}).get("name", "") for t in self.__tools} - lazy_tools = [ - t - for t in self.__lazy_loaded_tools_holder.get_all() - if t.get("function", {}).get("name", "") not in eager_names - ] - all_tools = self.__tools + lazy_tools + all_tools = self._merge_tools() payload: dict[str, Any] = { "messages": prepared_messages, "stream": True, @@ -59,21 +53,7 @@ def build(self, messages: list[Message]) -> dict[str, Any]: "tools": all_tools, } - if self.__response_format: - logger.debug("Setting response format (type=%s)", type(self.__response_format).__name__) - log_payload(logger, "Response format: %s", self.__response_format) - if hasattr(self.__response_format, "model_dump"): - payload["response_format"] = self.__response_format.model_dump( - exclude_none=True, mode="json" - ) - elif isinstance(self.__response_format, dict): - payload["response_format"] = self.__response_format - else: - logger.error( - "Unsupported response format type: %s. The response format will not be applied.", - type(self.__response_format), - ) - + self._apply_response_format(payload) self._apply_tool_choice(payload) if self.__presentation_settings.show_usage_statistics: @@ -83,6 +63,41 @@ def build(self, messages: list[Message]) -> dict[str, Any]: payload["extra_headers"] = self.__forwarded_headers chat_completion_config.update(payload) + self._log_result(chat_completion_config, prepared_messages, all_tools) + return chat_completion_config + + def _merge_tools(self) -> list[OpenAiToolConfigDict]: + eager_names: set[str] = {t.get("function", {}).get("name", "") for t in self.__tools} + lazy_tools = [ + t + for t in self.__lazy_loaded_tools_holder.get_all() + if t.get("function", {}).get("name", "") not in eager_names + ] + return self.__tools + lazy_tools + + def _apply_response_format(self, payload: dict[str, Any]) -> None: + if not self.__response_format: + return + logger.debug("Setting response format (type=%s)", type(self.__response_format).__name__) + log_payload(logger, "Response format: %s", self.__response_format) + if hasattr(self.__response_format, "model_dump"): + payload["response_format"] = self.__response_format.model_dump( + exclude_none=True, mode="json" + ) + elif isinstance(self.__response_format, dict): + payload["response_format"] = self.__response_format + else: + logger.error( + "Unsupported response format type: %s. The response format will not be applied.", + type(self.__response_format), + ) + + def _log_result( + self, + chat_completion_config: dict[str, Any], + prepared_messages: list[dict[str, Any]], + all_tools: list[OpenAiToolConfigDict], + ) -> None: if logger.isEnabledFor(logging.DEBUG): logger.debug( "Chat completion config: messages=%d, roles=%s, tools=%d (eager=%d, lazy=%d), response_format=%s, " @@ -91,7 +106,7 @@ def build(self, messages: list[Message]) -> dict[str, Any]: summarize_roles(prepared_messages), len(all_tools), len(self.__tools), - len(lazy_tools), + len(all_tools) - len(self.__tools), "response_format" in chat_completion_config, chat_completion_config.get("model"), # Header NAMES only — forwarded X-* header values are never logged, even @@ -106,7 +121,6 @@ def build(self, messages: list[Message]) -> dict[str, Any]: log_payload( logger, "Chat completion config: %s", json.dumps(loggable, ensure_ascii=False) ) - return chat_completion_config def _apply_tool_choice(self, payload: dict[str, Any]) -> None: tool_choice = self.__tool_choice_holder.consume()