From c3d085e6aee76653cd6c7fd847df6da078f40610 Mon Sep 17 00:00:00 2001 From: Nan Yu Date: Fri, 30 Oct 2026 19:57:13 +0000 Subject: [PATCH 1/2] test(conformance): agent function RPC Adds conformance/agent/agent_functions.yaml for agent functions and the agent's side of function RPC: declaring agent functions on a catalog, answering callAgentFunction, validating model output that calls agent functions, and calling renderer functions on a fake clock. The Python agent SDK runs it; the TypeScript and Dart agent SDKs and the web_core runner skip its actions with a reason. Also adds a message_processor_v1_0.yaml case: under strict validation, a call to a function the surface catalog declares schema-only is accepted. Signed-off-by: Nan Yu --- conformance/README.md | 26 + conformance/agent/agent_functions.yaml | 904 ++++++++++++++++++ ...onformance_harness_implementation_guide.md | 10 +- conformance/conformance_schema.json | 341 ++++++- conformance/core/message_processor_v1_0.yaml | 56 ++ .../agent_functions_conformance_test.dart | 51 + .../test_agent_function_conformance.py | 349 +++++++ .../tests/conformance/agent-functions.test.ts | 47 + .../a2ui_agent/tests/conformance/loader.ts | 12 +- .../tests/conformance/conformance_test.mjs | 4 + 10 files changed, 1796 insertions(+), 4 deletions(-) create mode 100644 conformance/agent/agent_functions.yaml create mode 100644 dart/a2ui_agent/test/conformance/agent_functions_conformance_test.dart create mode 100644 python/a2ui_agent/tests/conformance/test_agent_function_conformance.py create mode 100644 typescript/agents/a2ui_agent/tests/conformance/agent-functions.test.ts diff --git a/conformance/README.md b/conformance/README.md index 573dedac04..4938ae6fc7 100644 --- a/conformance/README.md +++ b/conformance/README.md @@ -33,6 +33,7 @@ The suites that do not depend on an inference format sit directly under `agent/` - `agent/request_processor.yaml`: Building an inference format from its factory, and taking registered catalogs and renderer capabilities through to a request processor. - `agent/skill.yaml`: Generating skill documents and skill sets from a format and a catalog. - `agent/multi_catalog_formats.yaml`: Resolving components and function calls across active catalogs (omitting `createSurface.catalogId` when multiple catalogs are active, resolving un-annotated components and function calls by name when unique across active catalogs, and requiring an explicit `catalogId` override when ambiguous), across `direct_json` (and later `express`): compiling, decompiling (including omitting `catalogId` when name lookup is unambiguous), parsing a response, rendering a prompt snippet, and building a format. Component order inside a message carries no meaning, so a harness may compare the `components` of a compiled message by `id`. +- `agent/agent_functions.yaml`: Agent functions and the agent's side of function RPC (v1.0). It covers declaring agent functions on the agent's copy of a catalog, answering `callAgentFunction` (the response values and error codes, schema defaults, the handler's context, and which catalog a call runs in), validating model output that calls agent functions, and calling renderer functions with `callRendererFunction` (settling calls with responses, ignoring unknown, late and duplicate responses, timeouts, disposal and the limit on pending calls). See [Writing cases for the agent function suite](#writing-cases-for-the-agent-function-suite). - `agent/streamed_answer.yaml`: Reading a streamed answer into a renderer's message processor, for an app that is both the agent and the renderer, and keeping the answer's text, surfaces and rejected blocks in the order the model wrote them. It runs on `direct_json`, the format that streams. The suite belongs to the optional `agent_renderer_app` feature, and only an SDK that implements it runs the suite. A case names its format with `args.format`, an `InferenceFormatName`: `direct_json` (or `express`). @@ -140,6 +141,31 @@ The case then runs its `steps` in order. An `evaluate` step evaluates a `Functio The cases use no timers, so they do not cover the timeout of an agent call. An SDK that cannot run a case lists it as a known divergence with the reason: the Dart core cannot declare a function without an implementation, so it skips the cases that use `agentFunctions`. +### Writing cases for the agent function suite + +`agent/agent_functions.yaml` declares agent functions in `AgentFunctionCatalog` entries (`conformance_schema.json`): a catalog document in `catalog`, named as in the other agent suites, and `agent_functions`, the functions the agent adds to its copy of that catalog, by name. A declaration carries the JSON Schema of the function's `args`, its `returnType`, and optionally `description` and `allowedCallers`. In place of a handler it states a `behavior`, so every SDK runs the same function: + +- `returns`: returns that value, which may be `null`. +- `returns_args: true`: returns the arguments the handler received, after validation and with the schema's defaults applied. +- `returns_context: true`: returns the context of the call as `{catalogId, surfaceId, functionCallId}`. +- `raises`: fails with `message`. With `code`, it fails the way the SDK lets a handler choose the response's RPC error code; without one, it fails with an ordinary error. + +The suite has four actions: + +- `declare_agent_functions` builds one catalog with its agent functions. `expect` lists, without regard to order, the catalog's functions (agent functions included), its agent functions, and the functions its `toJson()` document lists. `expect_error` names a catalog the SDK must refuse to build, such as one whose agent function takes the name of a function the catalog already declares. +- `handle_agent_function` builds the `catalogs`, registers their agent functions, and answers `message`, a `callAgentFunction` message as received. `expect` is the `agentFunctionResponse` body: `functionCallId` and either `value` or `error`. `error.code` is compared exactly and `error.message` is a substring of the message sent. `expect_error` means handling the message fails instead of answering it, which happens only when the message has no `functionCallId` to copy. +- `validate_payload` validates `messages` against the `catalogs`, the way an agent checks model output before sending it. A case states `expect_valid: true` or `expect_error`. +- `call_renderer_function` makes calls to renderer functions and settles them, in `steps`. Each step does one thing: `call` makes a call named by its `label` (with `catalogId`, `name`, and optionally `args`, `surfaceId` and `timeout_ms`), `respond` hands over a `rendererFunctionResponse` for a labelled `call` or a literal `functionCallId`, `advance_ms` moves the clock, and `dispose` disposes the calls with an optional `reason`. `args` may set `default_timeout_ms` (otherwise the SDK's default of 30 seconds applies) and `max_pending`. + +A `call_renderer_function` harness runs the calls on a fake clock that only `advance_ms` moves, so the cases never wait on real time and a timer fires exactly when the clock reaches it. After each step it lets pending asynchronous work run, then checks: + +- `expect_message`, on a `call` step: the `callRendererFunction` message without its `functionCallId`. Separately, the harness checks that every call in a case got a different, non-empty `functionCallId`. +- `expect_error`, on a `call` step: the call is refused with that RPC error `code`. +- `expect_calls`: the state of each labelled call, which is `pending`, `{value}`, or `{error: {code, message?}}` with `message` a substring. +- `expect_pending_count`: how many calls are waiting for a response. + +The TypeScript and Dart agent SDKs do not implement agent functions yet, so their harnesses skip every case of this suite with that reason. + ### Writing cases for the agent SDK suites The suites under `agent/`, including the per-format folders, share one vocabulary, described in the `$defs` of `conformance_schema.json` and summarised here. `agent/builder/` is the exception, and uses the AST-plus-golden notation described in its own README. diff --git a/conformance/agent/agent_functions.yaml b/conformance/agent/agent_functions.yaml new file mode 100644 index 0000000000..f7954b1f2d --- /dev/null +++ b/conformance/agent/agent_functions.yaml @@ -0,0 +1,904 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Agent functions and the agent's side of function RPC (v1.0): functions an +# agent declares on its copy of a catalog and runs when a renderer sends +# `callAgentFunction`, and the agent's own calls to renderer functions with +# `callRendererFunction`. +# +# A case declares each agent function with the JSON Schema of its `args` and a +# scripted behaviour in place of a handler, so every SDK runs the same +# function: it returns a fixed value, returns the arguments it received, +# returns the context of the call, or fails. Renderer calls run on a fake +# clock that only an `advance_ms` step moves, so no case waits on real time. +# See "Writing cases for the agent function suite" in conformance/README.md. + +# --- Declaring agent functions --- + +- name: test_declare_agent_function_lists_it_with_the_catalog_functions + description: >- + An agent function is one of the catalog's functions, so validation, prompts and compilers see it, and the catalog document lists it like any declared function. + action: declare_agent_functions + args: + catalog: + catalog: "test_data/catalogs/simplified_catalog_v1_0.json" + agent_functions: + fetchFxRate: + description: Current exchange rate from base to target. + returnType: number + args: + type: object + properties: + base: {type: string} + target: {type: string, default: USD} + required: [base] + additionalProperties: false + behavior: {returns: 1.08} + expect: + functions: [required, formatString, openUrl, fetchFxRate] + agent_functions: [fetchFxRate] + document_functions: [required, formatString, openUrl, fetchFxRate] + +- name: test_declare_agent_function_with_a_taken_name_is_a_catalog_error + description: >- + An agent function may not take the name of a function the catalog already declares, since a renderer would then run its own function and never send the call to the agent. + action: declare_agent_functions + args: + catalog: + catalog: "test_data/catalogs/simplified_catalog_v1_0.json" + agent_functions: + formatString: + returnType: string + args: + type: object + properties: + value: {type: string} + required: [value] + behavior: {returns: "formatted"} + expect_error: + category: CatalogError + message: "formatString" + +# --- Answering callAgentFunction --- + +- name: test_handle_agent_function_returns_the_value_and_copies_the_call_id + description: >- + The response carries the handler's value and copies the call's functionCallId verbatim, however it is spelled. + action: handle_agent_function + args: + catalogs: + - catalog: "test_data/catalogs/simplified_catalog_v1_0.json" + agent_functions: + fetchFxRate: + returnType: number + args: + type: object + properties: + base: {type: string} + target: {type: string, default: USD} + required: [base] + additionalProperties: false + behavior: {returns: 1.08} + message: + version: "1.0" + callAgentFunction: + surfaceId: s1 + functionCallId: "fx call/ü 42" + callFunction: + "@call": fetchFxRate + catalogId: conformance/simplified + args: {base: EUR} + expect: + functionCallId: "fx call/ü 42" + value: 1.08 + +- name: test_handle_agent_function_sends_a_null_value + description: >- + A handler that returns null still answers with a value, so the renderer can tell the answer from a missing one. + action: handle_agent_function + args: + catalogs: + - catalog: "test_data/catalogs/simplified_catalog_v1_0.json" + agent_functions: + lookupNickname: + returnType: string + args: + type: object + properties: + userId: {type: string} + required: [userId] + behavior: {returns: null} + message: + version: "1.0" + callAgentFunction: + surfaceId: s1 + functionCallId: nick-1 + callFunction: + "@call": lookupNickname + catalogId: conformance/simplified + args: {userId: u1} + expect: + functionCallId: nick-1 + value: null + +- name: test_handle_agent_function_applies_schema_defaults + description: >- + The arguments are validated against the function's schema before the handler runs, and the handler receives them with the schema's defaults applied. + action: handle_agent_function + args: + catalogs: + - catalog: "test_data/catalogs/simplified_catalog_v1_0.json" + agent_functions: + fetchFxRate: + returnType: object + args: + type: object + properties: + base: {type: string} + target: {type: string, default: USD} + required: [base] + additionalProperties: false + behavior: {returns_args: true} + message: + version: "1.0" + callAgentFunction: + surfaceId: s1 + functionCallId: fx-1 + callFunction: + "@call": fetchFxRate + catalogId: conformance/simplified + args: {base: EUR} + expect: + functionCallId: fx-1 + value: {base: EUR, target: USD} + +- name: test_handle_agent_function_passes_the_call_context + description: >- + The handler learns the catalog the call resolved to, the surface it was made on, and its functionCallId. + action: handle_agent_function + args: + catalogs: + - catalog: "test_data/catalogs/simplified_catalog_v1_0.json" + agent_functions: + whereAmI: + returnType: object + args: {type: object, properties: {}} + behavior: {returns_context: true} + message: + version: "1.0" + callAgentFunction: + surfaceId: checkout + functionCallId: ctx-1 + callFunction: + "@call": whereAmI + catalogId: conformance/simplified + args: {} + expect: + functionCallId: ctx-1 + value: + catalogId: conformance/simplified + surfaceId: checkout + functionCallId: ctx-1 + +- name: test_handle_agent_function_with_an_argument_of_the_wrong_type + description: Arguments that fail the function's schema get INVALID_FUNCTION_CALL, and the handler does not run. + action: handle_agent_function + args: + catalogs: + - catalog: "test_data/catalogs/simplified_catalog_v1_0.json" + agent_functions: + fetchFxRate: + returnType: number + args: + type: object + properties: + base: {type: string} + target: {type: string, default: USD} + required: [base] + additionalProperties: false + behavior: {returns: 1.08} + message: + version: "1.0" + callAgentFunction: + surfaceId: s1 + functionCallId: fx-2 + callFunction: + "@call": fetchFxRate + catalogId: conformance/simplified + args: {base: 3} + expect: + functionCallId: fx-2 + error: + code: INVALID_FUNCTION_CALL + message: fetchFxRate + +- name: test_handle_agent_function_with_a_missing_required_argument + description: A call that leaves out a required argument gets INVALID_FUNCTION_CALL. + action: handle_agent_function + args: + catalogs: + - catalog: "test_data/catalogs/simplified_catalog_v1_0.json" + agent_functions: + fetchFxRate: + returnType: number + args: + type: object + properties: + base: {type: string} + target: {type: string, default: USD} + required: [base] + additionalProperties: false + behavior: {returns: 1.08} + message: + version: "1.0" + callAgentFunction: + surfaceId: s1 + functionCallId: fx-3 + callFunction: + "@call": fetchFxRate + catalogId: conformance/simplified + args: {target: GBP} + expect: + functionCallId: fx-3 + error: + code: INVALID_FUNCTION_CALL + +- name: test_handle_agent_function_with_an_undeclared_argument + description: A call with an argument that a closed schema does not declare gets INVALID_FUNCTION_CALL. + action: handle_agent_function + args: + catalogs: + - catalog: "test_data/catalogs/simplified_catalog_v1_0.json" + agent_functions: + fetchFxRate: + returnType: number + args: + type: object + properties: + base: {type: string} + target: {type: string, default: USD} + required: [base] + additionalProperties: false + behavior: {returns: 1.08} + message: + version: "1.0" + callAgentFunction: + surfaceId: s1 + functionCallId: fx-4 + callFunction: + "@call": fetchFxRate + catalogId: conformance/simplified + args: {base: EUR, precision: 4} + expect: + functionCallId: fx-4 + error: + code: INVALID_FUNCTION_CALL + +- name: test_handle_agent_function_with_an_unknown_name + description: A call to a name no catalog registers as an agent function gets UNKNOWN_FUNCTION. + action: handle_agent_function + args: + catalogs: + - catalog: "test_data/catalogs/simplified_catalog_v1_0.json" + agent_functions: + fetchFxRate: + returnType: number + args: {type: object, properties: {base: {type: string}}} + behavior: {returns: 1.08} + message: + version: "1.0" + callAgentFunction: + surfaceId: s1 + functionCallId: u-1 + callFunction: + "@call": fetchStockPrice + catalogId: conformance/simplified + args: {} + expect: + functionCallId: u-1 + error: + code: UNKNOWN_FUNCTION + +- name: test_handle_agent_function_in_an_unknown_catalog + description: A call that names a catalog the agent does not have gets UNKNOWN_FUNCTION, even when another catalog registers the name. + action: handle_agent_function + args: + catalogs: + - catalog: "test_data/catalogs/simplified_catalog_v1_0.json" + agent_functions: + fetchFxRate: + returnType: number + args: {type: object, properties: {base: {type: string}}} + behavior: {returns: 1.08} + message: + version: "1.0" + callAgentFunction: + surfaceId: s1 + functionCallId: u-2 + callFunction: + "@call": fetchFxRate + catalogId: https://example.com/other/catalog.json + args: {base: EUR} + expect: + functionCallId: u-2 + error: + code: UNKNOWN_FUNCTION + +- name: test_handle_agent_function_for_a_renderer_function + description: >- + A function the catalog declares for the renderer to run is not an agent function, so a call to it gets UNKNOWN_FUNCTION. + action: handle_agent_function + args: + catalogs: + - catalog: "test_data/catalogs/simplified_catalog_v1_0.json" + agent_functions: + fetchFxRate: + returnType: number + args: {type: object, properties: {base: {type: string}}} + behavior: {returns: 1.08} + message: + version: "1.0" + callAgentFunction: + surfaceId: s1 + functionCallId: u-3 + callFunction: + "@call": formatString + catalogId: conformance/simplified + args: {value: Hello} + expect: + functionCallId: u-3 + error: + code: UNKNOWN_FUNCTION + +- name: test_handle_agent_function_whose_handler_fails + description: >- + A handler that fails gets EXECUTION_ERROR, with the failure's message. + action: handle_agent_function + args: + catalogs: + - catalog: "test_data/catalogs/simplified_catalog_v1_0.json" + agent_functions: + fetchFxRate: + returnType: number + args: {type: object, properties: {base: {type: string}}} + behavior: + raises: {message: "rate service unavailable"} + message: + version: "1.0" + callAgentFunction: + surfaceId: s1 + functionCallId: e-1 + callFunction: + "@call": fetchFxRate + catalogId: conformance/simplified + args: {base: EUR} + expect: + functionCallId: e-1 + error: + code: EXECUTION_ERROR + message: "rate service unavailable" + +- name: test_handle_agent_function_whose_handler_fails_with_its_own_code + description: >- + A handler that fails with an RPC error code of its choosing sends that code instead of EXECUTION_ERROR. + action: handle_agent_function + args: + catalogs: + - catalog: "test_data/catalogs/simplified_catalog_v1_0.json" + agent_functions: + fetchFxRate: + returnType: number + args: {type: object, properties: {base: {type: string}}} + behavior: + raises: {code: PROVIDER_NOT_FOUND, message: "no rate provider for EUR"} + message: + version: "1.0" + callAgentFunction: + surfaceId: s1 + functionCallId: e-2 + callFunction: + "@call": fetchFxRate + catalogId: conformance/simplified + args: {base: EUR} + expect: + functionCallId: e-2 + error: + code: PROVIDER_NOT_FOUND + message: "no rate provider for EUR" + +- name: test_handle_agent_function_that_only_the_agent_may_call + description: >- + A renderer may not call a function whose allowedCallers is agentOnly, so the call gets INVALID_FUNCTION_CALL and the handler does not run. + action: handle_agent_function + args: + catalogs: + - catalog: "test_data/catalogs/simplified_catalog_v1_0.json" + agent_functions: + resetLedger: + returnType: void + allowedCallers: agentOnly + args: {type: object, properties: {}} + behavior: {returns: null} + message: + version: "1.0" + callAgentFunction: + surfaceId: s1 + functionCallId: a-1 + callFunction: + "@call": resetLedger + catalogId: conformance/simplified + args: {} + expect: + functionCallId: a-1 + error: + code: INVALID_FUNCTION_CALL + message: agentOnly + +- name: test_handle_agent_function_picks_the_catalog_the_call_names + description: >- + Two catalogs may each register an agent function of the same name, and a call runs the one in the catalog it names. + action: handle_agent_function + args: + catalogs: + - catalog: "test_data/catalogs/simplified_catalog_v1_0.json" + agent_functions: + lookup: + returnType: string + args: {type: object, properties: {}} + behavior: {returns: "from simplified"} + - catalog: "test_data/catalogs/agent_functions_catalog_v1_0.json" + agent_functions: + lookup: + returnType: string + args: {type: object, properties: {}} + behavior: {returns: "from agent_functions"} + message: + version: "1.0" + callAgentFunction: + surfaceId: s1 + functionCallId: m-1 + callFunction: + "@call": lookup + catalogId: conformance/agent_functions + args: {} + expect: + functionCallId: m-1 + value: "from agent_functions" + +- name: test_handle_agent_function_without_catalog_id_runs_the_only_match + description: >- + Renderers name the catalog on every call. A call that names none runs the agent function of that name if exactly one catalog registers it. + action: handle_agent_function + args: + catalogs: + - catalog: "test_data/catalogs/simplified_catalog_v1_0.json" + agent_functions: + lookup: + returnType: string + args: {type: object, properties: {}} + behavior: {returns: "from simplified"} + - catalog: "test_data/catalogs/agent_functions_catalog_v1_0.json" + agent_functions: {} + message: + version: "1.0" + callAgentFunction: + surfaceId: s1 + functionCallId: m-2 + callFunction: + "@call": lookup + args: {} + expect: + functionCallId: m-2 + value: "from simplified" + +- name: test_handle_agent_function_without_catalog_id_and_no_match + description: A call that names no catalog, to a name no catalog registers, gets UNKNOWN_FUNCTION. + action: handle_agent_function + args: + catalogs: + - catalog: "test_data/catalogs/simplified_catalog_v1_0.json" + agent_functions: + lookup: + returnType: string + args: {type: object, properties: {}} + behavior: {returns: "from simplified"} + message: + version: "1.0" + callAgentFunction: + surfaceId: s1 + functionCallId: m-3 + callFunction: + "@call": search + args: {} + expect: + functionCallId: m-3 + error: + code: UNKNOWN_FUNCTION + +- name: test_handle_agent_function_without_catalog_id_and_several_matches + description: >- + A call that names no catalog, to a name several catalogs register, is ambiguous and gets INVALID_FUNCTION_CALL rather than running either function. + action: handle_agent_function + args: + catalogs: + - catalog: "test_data/catalogs/simplified_catalog_v1_0.json" + agent_functions: + lookup: + returnType: string + args: {type: object, properties: {}} + behavior: {returns: "from simplified"} + - catalog: "test_data/catalogs/agent_functions_catalog_v1_0.json" + agent_functions: + lookup: + returnType: string + args: {type: object, properties: {}} + behavior: {returns: "from agent_functions"} + message: + version: "1.0" + callAgentFunction: + surfaceId: s1 + functionCallId: m-4 + callFunction: + "@call": lookup + args: {} + expect: + functionCallId: m-4 + error: + code: INVALID_FUNCTION_CALL + +- name: test_handle_agent_function_with_a_malformed_message + description: >- + A message that carries a functionCallId but fails the callAgentFunction schema is answered with INVALID_FUNCTION_CALL. + action: handle_agent_function + args: + catalogs: + - catalog: "test_data/catalogs/simplified_catalog_v1_0.json" + agent_functions: + fetchFxRate: + returnType: number + args: {type: object, properties: {base: {type: string}}} + behavior: {returns: 1.08} + message: + version: "1.0" + callAgentFunction: + surfaceId: s1 + functionCallId: bad-1 + expect: + functionCallId: bad-1 + error: + code: INVALID_FUNCTION_CALL + +- name: test_handle_agent_function_with_a_message_without_call_id + description: >- + A message without a functionCallId cannot be answered, since a response must copy it, so handling it fails instead. + action: handle_agent_function + args: + catalogs: + - catalog: "test_data/catalogs/simplified_catalog_v1_0.json" + agent_functions: + fetchFxRate: + returnType: number + args: {type: object, properties: {base: {type: string}}} + behavior: {returns: 1.08} + message: + version: "1.0" + callAgentFunction: + surfaceId: s1 + callFunction: + "@call": fetchFxRate + catalogId: conformance/simplified + args: {base: EUR} + expect_error: + category: ValidationError + +# --- Validating model output that calls agent functions --- + +- name: test_validate_payload_with_a_call_to_a_declared_agent_function + description: >- + Model output that calls an agent function the catalog declares validates like a call to any other declared function. + action: validate_payload + args: + catalogs: + - catalog: "test_data/catalogs/simplified_catalog_v1_0.json" + agent_functions: + fetchGreeting: + returnType: string + args: + type: object + properties: + name: {type: string} + required: [name] + additionalProperties: false + behavior: {returns: "Hello"} + messages: + - version: "1.0" + createSurface: + surfaceId: s1 + catalogId: conformance/simplified + components: + - id: root + component: Text + text: + "@call": fetchGreeting + args: {name: Ada} + expect_valid: true + +- name: test_validate_payload_with_a_call_to_an_undeclared_function + description: >- + Declaring agent functions does not open the catalog: a call to a function that is neither declared nor an agent function is still rejected. + action: validate_payload + args: + catalogs: + - catalog: "test_data/catalogs/simplified_catalog_v1_0.json" + agent_functions: + fetchGreeting: + returnType: string + args: + type: object + properties: + name: {type: string} + required: [name] + additionalProperties: false + behavior: {returns: "Hello"} + messages: + - version: "1.0" + createSurface: + surfaceId: s1 + catalogId: conformance/simplified + components: + - id: root + component: Text + text: + "@call": fetchFarewell + args: {name: Ada} + expect_error: + category: ValidationError + message: fetchFarewell + +- name: test_validate_payload_with_invalid_arguments_to_an_agent_function + description: Model output that calls an agent function with arguments its schema rejects is invalid. + action: validate_payload + args: + catalogs: + - catalog: "test_data/catalogs/simplified_catalog_v1_0.json" + agent_functions: + fetchGreeting: + returnType: string + args: + type: object + properties: + name: {type: string} + required: [name] + additionalProperties: false + behavior: {returns: "Hello"} + messages: + - version: "1.0" + createSurface: + surfaceId: s1 + catalogId: conformance/simplified + components: + - id: root + component: Text + text: + "@call": fetchGreeting + args: {nickname: Ada} + expect_error: + category: ValidationError + +# --- Calling renderer functions --- + +- name: test_call_renderer_function_builds_the_message_without_waiting + description: >- + A call returns its callRendererFunction message at once, with a generated functionCallId and the catalogId the message requires, and stays pending until the renderer answers. + action: call_renderer_function + steps: + - call: + label: a + catalogId: conformance/agent_functions + name: notify + args: {message: Saved} + surfaceId: s1 + expect_message: + version: "1.0" + callRendererFunction: + surfaceId: s1 + callFunction: + "@call": notify + catalogId: conformance/agent_functions + args: {message: Saved} + expect_calls: + a: pending + expect_pending_count: 1 + +- name: test_call_renderer_function_without_surface_or_args + description: >- + A call made without a surface leaves surfaceId out of the message, and one made without arguments sends empty args. + action: call_renderer_function + steps: + - call: + label: a + catalogId: conformance/agent_functions + name: now + expect_message: + version: "1.0" + callRendererFunction: + callFunction: + "@call": now + catalogId: conformance/agent_functions + args: {} + expect_calls: + a: pending + +- name: test_call_renderer_function_settled_by_its_response + description: >- + A rendererFunctionResponse settles the call whose functionCallId it copies with its value, and leaves the other calls pending. + action: call_renderer_function + steps: + - call: {label: a, catalogId: conformance/agent_functions, name: getLocale} + - call: {label: b, catalogId: conformance/agent_functions, name: getTimezone} + expect_pending_count: 2 + - respond: {call: b, value: Europe/Paris} + expect_calls: + a: pending + b: {value: Europe/Paris} + expect_pending_count: 1 + - respond: {call: a, value: {language: fr, region: FR}} + expect_calls: + a: {value: {language: fr, region: FR}} + b: {value: Europe/Paris} + expect_pending_count: 0 + +- name: test_call_renderer_function_settled_with_a_null_value + description: A response whose value is null settles the call with null. + action: call_renderer_function + steps: + - call: {label: a, catalogId: conformance/agent_functions, name: getSelection} + - respond: {call: a, value: null} + expect_calls: + a: {value: null} + +- name: test_call_renderer_function_failed_by_an_error_response + description: A response that carries an error fails the call with the renderer's code. + action: call_renderer_function + steps: + - call: {label: a, catalogId: conformance/agent_functions, name: getLocation} + - respond: + call: a + error: {code: PERMISSION_DENIED, message: "The user declined."} + expect_calls: + a: {error: {code: PERMISSION_DENIED, message: "The user declined."}} + expect_pending_count: 0 + +- name: test_call_renderer_function_ignores_an_unknown_response + description: >- + A response whose functionCallId matches no call comes from the network, so it is ignored without failing, and the pending calls stay pending. + action: call_renderer_function + steps: + - call: {label: a, catalogId: conformance/agent_functions, name: getLocale} + - respond: {functionCallId: never-sent, value: en} + expect_calls: + a: pending + expect_pending_count: 1 + +- name: test_call_renderer_function_ignores_a_duplicate_response + description: A second response to a settled call is ignored, and the call keeps the first value. + action: call_renderer_function + steps: + - call: {label: a, catalogId: conformance/agent_functions, name: getLocale} + - respond: {call: a, value: en} + - respond: {call: a, value: fr} + expect_calls: + a: {value: en} + +- name: test_call_renderer_function_times_out_after_thirty_seconds_by_default + description: >- + A call waits 30 seconds for its response by default, the same as the renderer waits for the agent, then fails with TIMEOUT and stops being pending. + action: call_renderer_function + steps: + - call: {label: a, catalogId: conformance/agent_functions, name: getLocale} + - advance_ms: 29999 + expect_calls: + a: pending + - advance_ms: 1 + expect_calls: + a: {error: {code: TIMEOUT}} + expect_pending_count: 0 + +- name: test_call_renderer_function_ignores_a_late_response + description: A response that arrives after its call timed out is ignored, and the call stays failed. + action: call_renderer_function + steps: + - call: {label: a, catalogId: conformance/agent_functions, name: getLocale, timeout_ms: 100} + - advance_ms: 100 + expect_calls: + a: {error: {code: TIMEOUT}} + - respond: {call: a, value: en} + expect_calls: + a: {error: {code: TIMEOUT}} + +- name: test_call_renderer_function_with_its_own_timeout + description: A timeout given to one call replaces the default for that call only. + action: call_renderer_function + steps: + - call: {label: short, catalogId: conformance/agent_functions, name: getLocale, timeout_ms: 500} + - call: {label: default, catalogId: conformance/agent_functions, name: getTimezone} + - advance_ms: 500 + expect_calls: + short: {error: {code: TIMEOUT}} + default: pending + +- name: test_call_renderer_function_with_a_default_timeout + description: A default timeout given when the calls are set up applies to every call that sets none of its own. + action: call_renderer_function + args: + default_timeout_ms: 2000 + steps: + - call: {label: a, catalogId: conformance/agent_functions, name: getLocale} + - advance_ms: 1999 + expect_calls: + a: pending + - advance_ms: 1 + expect_calls: + a: {error: {code: TIMEOUT}} + +- name: test_call_renderer_function_settled_before_its_timeout + description: A call that is answered in time does not time out later. + action: call_renderer_function + steps: + - call: {label: a, catalogId: conformance/agent_functions, name: getLocale, timeout_ms: 1000} + - advance_ms: 400 + - respond: {call: a, value: en} + - advance_ms: 1000 + expect_calls: + a: {value: en} + +- name: test_call_renderer_function_dispose_fails_pending_calls + description: >- + Disposing the calls fails every pending call with CANCELLED and the reason given, leaves settled calls alone, refuses later calls with DISPOSED, and ignores later responses. + action: call_renderer_function + steps: + - call: {label: done, catalogId: conformance/agent_functions, name: getLocale} + - call: {label: a, catalogId: conformance/agent_functions, name: getTimezone} + - call: {label: b, catalogId: conformance/agent_functions, name: getLocation} + - respond: {call: done, value: en} + - dispose: {reason: "the session ended"} + expect_calls: + done: {value: en} + a: {error: {code: CANCELLED, message: "the session ended"}} + b: {error: {code: CANCELLED, message: "the session ended"}} + expect_pending_count: 0 + - respond: {call: a, value: Europe/Paris} + expect_calls: + a: {error: {code: CANCELLED}} + - call: {label: c, catalogId: conformance/agent_functions, name: getLocale} + expect_error: {code: DISPOSED} + +- name: test_call_renderer_function_limits_pending_calls + description: >- + With a limit on pending calls, a call beyond it fails with TOO_MANY_PENDING instead of piling up, and settling a call makes room for another. + action: call_renderer_function + args: + max_pending: 2 + steps: + - call: {label: a, catalogId: conformance/agent_functions, name: getLocale} + - call: {label: b, catalogId: conformance/agent_functions, name: getTimezone} + - call: {label: c, catalogId: conformance/agent_functions, name: getLocation} + expect_error: {code: TOO_MANY_PENDING} + expect_pending_count: 2 + - respond: {call: a, value: en} + - call: {label: d, catalogId: conformance/agent_functions, name: getLocation} + expect_calls: + b: pending + d: pending + expect_pending_count: 2 diff --git a/conformance/conformance_harness_implementation_guide.md b/conformance/conformance_harness_implementation_guide.md index c4c63f36b9..0ffe0c43d5 100644 --- a/conformance/conformance_harness_implementation_guide.md +++ b/conformance/conformance_harness_implementation_guide.md @@ -122,6 +122,7 @@ Implement the suites in stages so each layer is tested before higher-level compo | A4. Multi-catalog and session pipeline | `agent/multi_catalog_formats.yaml`, `agent/request_processor.yaml`, `agent/streamed_answer.yaml` | Cross-catalog lookup, `InferenceFormatFactory`, `A2uiGenerator`, `A2uiRequestProcessor`, and `StreamedAnswer` | | A5. Specification round-trips | `catalogs/basic/v1/examples/*.json` (and v0.9 examples if supported) | End-to-end `decompile` followed by `compile` across every published basic catalog example | | A6. Skills, macros, and scaling | `agent/skill.yaml`, `agent/builder/builder.yaml`, `agent/macros/macros.yaml`, `agent/*/scaling.yaml` | Skill generation, typesafe fluent builders, macros, and linear-time parsing | +| A7. Agent functions | `agent/agent_functions.yaml` | Agent functions on a catalog, answering `callAgentFunction`, and calling renderer functions on a fake clock | | F1. Framework adapter | `core/accessibility.yaml` | Accessibility attributes applied by a rendered surface | Notice that `agent/express/` has no `response_streaming.yaml` file. Express parses whole buffered blocks and does not implement `openStream`. @@ -251,6 +252,10 @@ All four actions in `response_parser.yaml` run against `format.createParser()`: 2. `create_processor` (`agent/request_processor.yaml`): Build `A2uiGenerator` with `args.catalogs`, `args.examples`, `args.accepts_inline_catalogs`, and optional `args.format` (omit the factory argument when `args.format` is absent to test the default Express factory). Snapshot the generator's catalogs before calling `generator.createProcessor(args.renderer_capabilities, formatOverride: args.format_override)`, then check `expect.active_catalog_ids`, positional `expect.catalogs`, prompt snippet checks, and `expect.generator_catalogs_unchanged`. If a sibling `then_parse` block is present, run `processor.parseResponse(then_parse.input, wrapped: then_parse.wrapped ?? true)` and check `then_parse.expect` or `then_parse.expect_error`. 3. `stream_answer` (`agent/streamed_answer.yaml`): Build an `A2uiRequestProcessor` (`DirectJsonFormatFactory` with `args.catalog`) and a Core `MessageProcessor` (`args.renderer_catalogs ?? [args.catalog]`), then open `StreamedAnswer(processor, renderer)`. Unlike `parse_chunk` deltas, each step's `expect_parts` checks the full `answer.parts` list accumulated so far (`{"text": ...}`, `{"surface": surface.id}`, or `{"error": {"category": ...}}`). Assert reference identity (`same` / `===` / `is`) between each `AnswerSurfacePart.surface` and `renderer.model.getSurface(surface.id)`, and check `step.expect_in_block` plus `expect_close` after `answer.close()`. +### Agent functions and renderer calls + +`agent/agent_functions.yaml` (`declare_agent_functions`, `handle_agent_function`, `validate_payload` and `call_renderer_function`) is described in [Writing cases for the agent function suite](README.md#writing-cases-for-the-agent-function-suite). Turn each declared `behavior` into a handler in the harness, and run `call_renderer_function` cases on a fake clock: in Python, an event loop whose `time()` returns a counter that `advance_ms` steps add to; in JavaScript, fake timers. An SDK that does not implement agent functions yet registers every case as skipped with that reason. + ### Skills, builders, macros, round-trips, and timing suites 1. Skills (`agent/skill.yaml`): Dispatch five actions against `SkillGenerator` and `SkillSet`: `from_format` (`generateSkill`), `core_syntax` (`generateCoreSkill`), `from_catalog` (`generateCatalogSkill`), `skill_set` (`generateSkillSet`), and `export_to_directory` (`SkillSet.exportToDirectory`). Resolve all file paths in this suite from `REPO_ROOT`. Run `export_to_directory` in a temporary directory populated with `args.pre_existing_files` to check that exporting deletes only managed skill subdirectories and preserves sibling files (even on `expect_error`). @@ -298,8 +303,9 @@ Inspect these existing implementations for concrete code examples: 3. [python/a2ui_agent/tests/conformance/test_catalog_conformance.py](../python/a2ui_agent/tests/conformance/test_catalog_conformance.py) 4. [python/a2ui_agent/tests/conformance/test_skill_conformance.py](../python/a2ui_agent/tests/conformance/test_skill_conformance.py) 5. [python/a2ui_agent/tests/conformance/test_scaling_conformance.py](../python/a2ui_agent/tests/conformance/test_scaling_conformance.py) - 6. [python/a2ui_agent/tests/conformance/builder_suite.py](../python/a2ui_agent/tests/conformance/builder_suite.py) and [python/a2ui_agent/tests/conformance/macro_suite.py](../python/a2ui_agent/tests/conformance/macro_suite.py) - 7. [python/a2ui_agent/tests/test_specification_roundtrip.py](../python/a2ui_agent/tests/test_specification_roundtrip.py) + 6. [python/a2ui_agent/tests/conformance/test_agent_function_conformance.py](../python/a2ui_agent/tests/conformance/test_agent_function_conformance.py) + 7. [python/a2ui_agent/tests/conformance/builder_suite.py](../python/a2ui_agent/tests/conformance/builder_suite.py) and [python/a2ui_agent/tests/conformance/macro_suite.py](../python/a2ui_agent/tests/conformance/macro_suite.py) + 8. [python/a2ui_agent/tests/test_specification_roundtrip.py](../python/a2ui_agent/tests/test_specification_roundtrip.py) 3. Inference SDK, TypeScript (`vitest` suites with strict `KNOWN_FAILURES` vs `UNSUPPORTED` classification): 1. [typescript/agents/a2ui_agent/tests/conformance/loader.ts](../typescript/agents/a2ui_agent/tests/conformance/loader.ts), [typescript/agents/a2ui_agent/tests/conformance/fixtures.ts](../typescript/agents/a2ui_agent/tests/conformance/fixtures.ts), and [typescript/agents/a2ui_agent/tests/conformance/suite-helpers.ts](../typescript/agents/a2ui_agent/tests/conformance/suite-helpers.ts) 2. [typescript/agents/a2ui_agent/tests/conformance/direct-json-conformance.test.ts](../typescript/agents/a2ui_agent/tests/conformance/direct-json-conformance.test.ts) and [typescript/agents/a2ui_agent/tests/conformance/direct-json-response-streaming.test.ts](../typescript/agents/a2ui_agent/tests/conformance/direct-json-response-streaming.test.ts) diff --git a/conformance/conformance_schema.json b/conformance/conformance_schema.json index c24f6acf2b..cb5d05e809 100644 --- a/conformance/conformance_schema.json +++ b/conformance/conformance_schema.json @@ -31,7 +31,11 @@ {"$ref": "#/$defs/EvaluateFunctionTest"}, {"$ref": "#/$defs/DispatchActionTest"}, {"$ref": "#/$defs/RouteAgentFunctionTest"}, - {"$ref": "#/$defs/DeriveTest"} + {"$ref": "#/$defs/DeriveTest"}, + {"$ref": "#/$defs/DeclareAgentFunctionsTest"}, + {"$ref": "#/$defs/HandleAgentFunctionTest"}, + {"$ref": "#/$defs/ValidatePayloadTest"}, + {"$ref": "#/$defs/CallRendererFunctionTest"} ] }, "$defs": { @@ -2092,6 +2096,341 @@ }, "required": ["action", "args"] }, + "AgentFunctionBehavior": { + "description": "What an agent function does when it runs, in place of a handler, so every SDK runs the same function. Exactly one key.", + "type": "object", + "properties": { + "returns": { + "description": "Returns this value, which may be null." + }, + "returns_args": { + "const": true, + "description": "Returns the arguments it received, after validation and with the schema's defaults applied." + }, + "returns_context": { + "const": true, + "description": "Returns the context of the call: {catalogId, surfaceId, functionCallId}." + }, + "raises": { + "type": "object", + "description": "Fails with `message`. With `code`, it fails the way the SDK lets a handler choose the RPC error code of the response; without one, it fails with an ordinary error.", + "properties": { + "code": {"type": "string"}, + "message": {"type": "string"} + }, + "required": ["message"], + "additionalProperties": false + } + }, + "minProperties": 1, + "maxProperties": 1, + "additionalProperties": false + }, + "AgentFunctionDeclaration": { + "type": "object", + "description": "An agent function: a catalog function that runs on the agent.", + "properties": { + "description": {"type": "string"}, + "returnType": {"type": "string"}, + "allowedCallers": { + "enum": ["rendererOnly", "agentOnly", "rendererOrAgent"], + "default": "rendererOnly" + }, + "args": { + "type": "object", + "description": "JSON Schema of the function's args." + }, + "behavior": {"$ref": "#/$defs/AgentFunctionBehavior"} + }, + "required": ["args", "behavior"], + "additionalProperties": false + }, + "AgentFunctionCatalog": { + "type": "object", + "description": "A catalog document and the agent functions the agent declares on its copy of it.", + "properties": { + "catalog": {"$ref": "#/$defs/AgentCatalogRef"}, + "agent_functions": { + "type": "object", + "description": "The agent functions, by name.", + "additionalProperties": {"$ref": "#/$defs/AgentFunctionDeclaration"} + } + }, + "required": ["catalog"], + "additionalProperties": false + }, + "DeclareAgentFunctionsTest": { + "type": "object", + "description": "Declaring agent functions on a catalog, and what the catalog then lists.", + "properties": { + "name": {"type": "string"}, + "description": {"type": "string"}, + "action": {"const": "declare_agent_functions"}, + "args": { + "type": "object", + "properties": { + "catalog": {"$ref": "#/$defs/AgentFunctionCatalog"} + }, + "required": ["catalog"], + "additionalProperties": false + }, + "expect": { + "type": "object", + "description": "Exhaustive name lists, compared without regard to order.", + "properties": { + "functions": { + "type": "array", + "items": {"type": "string"}, + "description": "The catalog's functions, agent functions included." + }, + "agent_functions": { + "type": "array", + "items": {"type": "string"}, + "description": "The catalog's agent functions." + }, + "document_functions": { + "type": "array", + "items": {"type": "string"}, + "description": "The functions the catalog's document lists, which is what a renderer would load." + } + }, + "required": ["functions", "agent_functions", "document_functions"], + "additionalProperties": false + }, + "expect_error": {"$ref": "#/$defs/ExpectError"} + }, + "required": ["name", "action", "args"], + "oneOf": [{"required": ["expect"]}, {"required": ["expect_error"]}], + "additionalProperties": false + }, + "HandleAgentFunctionTest": { + "type": "object", + "description": "Answering one callAgentFunction message with the agent functions of `catalogs`.", + "properties": { + "name": {"type": "string"}, + "description": {"type": "string"}, + "action": {"const": "handle_agent_function"}, + "args": { + "type": "object", + "properties": { + "catalogs": { + "type": "array", + "items": {"$ref": "#/$defs/AgentFunctionCatalog"}, + "minItems": 1 + }, + "message": { + "type": "object", + "description": "The callAgentFunction message, as received. It may be malformed." + } + }, + "required": ["catalogs", "message"], + "additionalProperties": false + }, + "expect": { + "type": "object", + "description": "The agentFunctionResponse body of the response. `error.message` is a substring of the message sent.", + "properties": { + "functionCallId": {"type": "string"}, + "value": {}, + "error": { + "type": "object", + "properties": { + "code": {"type": "string"}, + "message": {"type": "string"} + }, + "required": ["code"], + "additionalProperties": false + } + }, + "required": ["functionCallId"], + "oneOf": [{"required": ["value"]}, {"required": ["error"]}], + "additionalProperties": false + }, + "expect_error": { + "$ref": "#/$defs/ExpectError", + "description": "Handling the message fails instead of answering it." + } + }, + "required": ["name", "action", "args"], + "oneOf": [{"required": ["expect"]}, {"required": ["expect_error"]}], + "additionalProperties": false + }, + "ValidatePayloadTest": { + "type": "object", + "description": "Validating model output against catalogs that declare agent functions, as the agent does before sending it.", + "properties": { + "name": {"type": "string"}, + "description": {"type": "string"}, + "action": {"const": "validate_payload"}, + "args": { + "type": "object", + "properties": { + "catalogs": { + "type": "array", + "items": {"$ref": "#/$defs/AgentFunctionCatalog"}, + "minItems": 1 + }, + "messages": { + "type": "array", + "items": {"type": "object"}, + "description": "The A2UI messages to validate." + } + }, + "required": ["catalogs", "messages"], + "additionalProperties": false + }, + "expect_valid": {"const": true}, + "expect_error": {"$ref": "#/$defs/ExpectError"} + }, + "required": ["name", "action", "args"], + "oneOf": [{"required": ["expect_valid"]}, {"required": ["expect_error"]}], + "additionalProperties": false + }, + "CallRendererFunctionTest": { + "type": "object", + "description": "Calling renderer functions from the agent with callRendererFunction, and settling the calls with the renderer's responses, on a fake clock that only `advance_ms` steps move.", + "properties": { + "name": {"type": "string"}, + "description": {"type": "string"}, + "action": {"const": "call_renderer_function"}, + "args": { + "type": "object", + "properties": { + "default_timeout_ms": { + "type": ["integer", "null"], + "exclusiveMinimum": 0, + "description": "How long a call waits for its response unless it sets its own timeout. Without it, the SDK's default, 30000. Null waits without a limit." + }, + "max_pending": { + "type": "integer", + "minimum": 1, + "description": "The most calls that may wait for a response at once." + } + }, + "additionalProperties": false + }, + "steps": { + "type": "array", + "items": {"$ref": "#/$defs/RendererCallStep"}, + "minItems": 1 + } + }, + "required": ["name", "action", "steps"], + "additionalProperties": false + }, + "RendererCallState": { + "description": "The state of a call: still pending, settled with a value, or failed with an RPC error code. `error.message` is a substring of the failure's message.", + "oneOf": [ + {"const": "pending"}, + { + "type": "object", + "properties": {"value": {}}, + "required": ["value"], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "error": { + "type": "object", + "properties": { + "code": {"type": "string"}, + "message": {"type": "string"} + }, + "required": ["code"], + "additionalProperties": false + } + }, + "required": ["error"], + "additionalProperties": false + } + ] + }, + "RendererCallStep": { + "type": "object", + "description": "One step of a call_renderer_function case: exactly one of `call`, `respond`, `advance_ms` or `dispose`, then its expectations, checked once pending asynchronous work has run.", + "properties": { + "call": { + "type": "object", + "description": "Makes a call. `label` names it for later steps.", + "properties": { + "label": {"type": "string"}, + "catalogId": {"type": "string"}, + "name": {"type": "string"}, + "args": {"type": "object"}, + "surfaceId": {"type": "string"}, + "timeout_ms": { + "type": "integer", + "exclusiveMinimum": 0, + "description": "How long this call waits for its response, in place of the default." + } + }, + "required": ["label", "catalogId", "name"], + "additionalProperties": false + }, + "respond": { + "type": "object", + "description": "Hands the calls a rendererFunctionResponse, for the call `call` labels or for a literal `functionCallId`.", + "properties": { + "call": {"type": "string"}, + "functionCallId": {"type": "string"}, + "value": {}, + "error": { + "type": "object", + "properties": { + "code": {"type": "string"}, + "message": {"type": "string"} + }, + "required": ["code", "message"], + "additionalProperties": false + } + }, + "allOf": [ + {"oneOf": [{"required": ["call"]}, {"required": ["functionCallId"]}]}, + {"oneOf": [{"required": ["value"]}, {"required": ["error"]}]} + ], + "additionalProperties": false + }, + "advance_ms": { + "type": "integer", + "minimum": 0, + "description": "Moves the fake clock forward." + }, + "dispose": { + "type": "object", + "description": "Disposes the calls, with an optional reason.", + "properties": { + "reason": {"type": "string"} + }, + "additionalProperties": false + }, + "expect_message": { + "type": "object", + "description": "On a `call` step, the callRendererFunction message, less its functionCallId." + }, + "expect_error": { + "$ref": "#/$defs/ExpectError", + "description": "On a `call` step, the call is refused with this RPC error code instead of being made." + }, + "expect_calls": { + "type": "object", + "description": "The state of the calls it names, by label.", + "additionalProperties": {"$ref": "#/$defs/RendererCallState"} + }, + "expect_pending_count": { + "type": "integer", + "minimum": 0, + "description": "How many calls are waiting for a response." + } + }, + "oneOf": [ + {"required": ["call"]}, + {"required": ["respond"]}, + {"required": ["advance_ms"]}, + {"required": ["dispose"]} + ], + "additionalProperties": false + }, "ExpectError": { "description": "Expected error specification (error message string, regex, or structured error object).", "oneOf": [ diff --git a/conformance/core/message_processor_v1_0.yaml b/conformance/core/message_processor_v1_0.yaml index e2ec91c343..5b6162ef2e 100644 --- a/conformance/core/message_processor_v1_0.yaml +++ b/conformance/core/message_processor_v1_0.yaml @@ -1216,6 +1216,62 @@ s: exists: true +- name: test_v10_strict_function_call_to_schema_only_function_of_surface_catalog + description: >- + Under strictMode: true, a function call that names no catalogId is checked against the surface default catalog, and a function that catalog declares without an implementation (schema-only, as for a function that runs on the agent) passes. Strict validation rejects only functions no catalog declares; a renderer that validates strictly can declare the agent's functions this way and still route calls to them via callAgentFunction. + action: process_messages + protocolVersion: "1.0" + strictMode: true + catalogs: + - catalogId: "agent-fns-v10" + protocolVersion: "1.0" + components: + Badge: + type: object + properties: + component: + const: Badge + label: + $ref: "common_types.json#/$defs/DynamicString" + required: ["component", "label"] + functions: + fetchGreeting: + type: object + returnType: string + properties: + "@call": + const: fetchGreeting + args: + type: object + properties: + name: + type: string + required: ["name"] + additionalProperties: false + required: ["@call", "args"] + messages: + - version: "1.0" + createSurface: + surfaceId: "s" + catalogId: "agent-fns-v10" + - version: "1.0" + updateComponents: + surfaceId: "s" + components: + - id: "root" + component: "Badge" + label: + "@call": "fetchGreeting" + args: + name: "Ada" + expect: + surfaces: + s: + exists: true + components: + - id: "root" + component: "Badge" + - name: test_v10_function_call_naming_unknown_catalog_errors description: >- A function call that names a catalogId the client does not have is rejected. diff --git a/dart/a2ui_agent/test/conformance/agent_functions_conformance_test.dart b/dart/a2ui_agent/test/conformance/agent_functions_conformance_test.dart new file mode 100644 index 0000000000..cf410195bd --- /dev/null +++ b/dart/a2ui_agent/test/conformance/agent_functions_conformance_test.dart @@ -0,0 +1,51 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import 'package:test/test.dart'; + +import 'suites.dart'; + +/// Registers the shared `conformance/agent/agent_functions.yaml` suite. +/// +/// The suite covers agent functions and the agent's calls to renderer +/// functions, which this SDK does not implement yet, so every case is +/// skipped with the reason [_unimplementedActions] gives. A case whose action +/// has no reason fails instead of passing without running. +void main() { + const path = 'agent/agent_functions.yaml'; + group('conformance $path', () { + final List> cases = loadSuite(path); + test('suite is not empty', () => expect(cases, isNotEmpty)); + for (final testCase in cases) { + final action = testCase['action']! as String; + test( + '$action · ${testCase['name']}', + () => fail('Unhandled conformance action: $action'), + skip: _unimplementedActions[action], + ); + } + }); +} + +/// The suite's actions this SDK has no implementation for, with the reason. +const Map _unimplementedActions = { + 'declare_agent_functions': _notImplemented, + 'handle_agent_function': _notImplemented, + 'validate_payload': _notImplemented, + 'call_renderer_function': _notImplemented, +}; + +const String _notImplemented = + 'this SDK does not implement agent functions or calls to renderer ' + 'functions yet'; diff --git a/python/a2ui_agent/tests/conformance/test_agent_function_conformance.py b/python/a2ui_agent/tests/conformance/test_agent_function_conformance.py new file mode 100644 index 0000000000..63d1f09688 --- /dev/null +++ b/python/a2ui_agent/tests/conformance/test_agent_function_conformance.py @@ -0,0 +1,349 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Runs the shared `conformance/agent/agent_functions.yaml` suite. + +A case declares each agent function with a scripted behaviour, which +`_handler` turns into a handler. The renderer call cases run on +`_FakeClockLoop`, an event loop whose clock moves only when a case's +`advance_ms` step moves it, so their timeouts don't depend on real time. +""" + +from __future__ import annotations + +import asyncio +from collections.abc import Callable, Mapping +import os +import re +import selectors +from typing import Any + +import pytest + +from a2ui.core import ( + A2uiCatalogError, + A2uiRpcError, + A2uiValidationError, + CatalogApi, +) +from a2ui.core.catalog import AgentFunction, AgentFunctionContext +from a2ui.processor import InMemoryCatalogProvider +from a2ui.rpc import AgentFunctionRegistry, RendererFunctionCalls +from a2ui.utils import validate_payload + +from .conformance_helpers import load_conformance_json, load_conformance_yaml + +SUITE = "agent/agent_functions.yaml" + +CATEGORY_TO_EXCEPTION: dict[str, type[Exception]] = { + "CatalogError": A2uiCatalogError, + "ValidationError": A2uiValidationError, +} + +AgentFunctionHandler = Callable[[dict[str, Any], AgentFunctionContext], Any] + + +def _load_cases() -> list[Any]: + cases = load_conformance_yaml(SUITE) + return [pytest.param(case, id=case["name"]) for case in cases] + + +CASES = _load_cases() + + +def test_suite_is_not_empty() -> None: + assert CASES + + +def _document(ref: str | Mapping[str, Any]) -> tuple[dict[str, Any], str]: + """The catalog document a case names, and the id to use if it has none.""" + if isinstance(ref, str): + return load_conformance_json(ref), os.path.basename(ref).removesuffix(".json") + return dict(ref), "inline" + + +def _handler(behavior: Mapping[str, Any]) -> AgentFunctionHandler: + """The handler that carries out a case's scripted behaviour.""" + if "returns" in behavior: + value = behavior["returns"] + + async def returns(args: dict[str, Any], context: AgentFunctionContext) -> Any: + return value + + return returns + if behavior.get("returns_args"): + + async def returns_args( + args: dict[str, Any], context: AgentFunctionContext + ) -> Any: + return args + + return returns_args + if behavior.get("returns_context"): + + async def returns_context( + args: dict[str, Any], context: AgentFunctionContext + ) -> Any: + return { + "catalogId": context.catalog_id, + "surfaceId": context.surface_id, + "functionCallId": context.function_call_id, + } + + return returns_context + if "raises" in behavior: + failure = behavior["raises"] + + async def raises(args: dict[str, Any], context: AgentFunctionContext) -> Any: + if "code" in failure: + raise A2uiRpcError(failure["message"], code=failure["code"]) + raise RuntimeError(failure["message"]) + + return raises + raise ValueError(f"Unknown agent function behavior: {behavior}") + + +def _agent_function(name: str, declaration: Mapping[str, Any]) -> AgentFunction: + return AgentFunction( + name, + handler=_handler(declaration["behavior"]), + return_type=declaration.get("returnType", "any"), + schema=declaration["args"], + allowed_callers=declaration.get("allowedCallers", "rendererOnly"), + description=declaration.get("description"), + ) + + +def _catalog(entry: Mapping[str, Any]) -> CatalogApi: + """The catalog a case's entry describes, with its agent functions.""" + document, fallback_id = _document(entry["catalog"]) + catalog = InMemoryCatalogProvider( + document, + protocol_version=None if "protocolVersion" in document else "1.0", + catalog_id=None if "catalogId" in document else fallback_id, + ).load() + return catalog.copy_with( + agent_functions=[ + _agent_function(name, declaration) + for name, declaration in entry.get("agent_functions", {}).items() + ] + ) + + +def _assert_error_matches(error: BaseException, expected: Mapping[str, Any]) -> None: + if "category" in expected: + assert isinstance(error, CATEGORY_TO_EXCEPTION[expected["category"]]), error + if "code" in expected: + assert getattr(error, "code", None) == expected["code"], error + if "message" in expected: + assert re.search(re.escape(expected["message"]), str(error)), error + + +def _run_declare(case: Mapping[str, Any]) -> None: + entry = case["args"]["catalog"] + if "expect_error" in case: + with pytest.raises(Exception) as excinfo: + _catalog(entry) + _assert_error_matches(excinfo.value, case["expect_error"]) + return + catalog = _catalog(entry) + expected = case["expect"] + assert sorted(catalog.functions) == sorted(expected["functions"]) + assert sorted(catalog.agent_functions) == sorted(expected["agent_functions"]) + document_functions = catalog.to_json().get("functions", {}) + assert sorted(document_functions) == sorted(expected["document_functions"]) + + +def _run_handle(case: Mapping[str, Any]) -> None: + args = case["args"] + registry = AgentFunctionRegistry([_catalog(entry) for entry in args["catalogs"]]) + + if "expect_error" in case: + with pytest.raises(Exception) as excinfo: + asyncio.run(registry.handle(args["message"])) + _assert_error_matches(excinfo.value, case["expect_error"]) + return + + message = asyncio.run(registry.handle(args["message"])) + dumped = message.model_dump(by_alias=True, exclude_unset=True) + assert dumped["version"] == "1.0" + response = dumped["agentFunctionResponse"] + expected = case["expect"] + assert response["functionCallId"] == expected["functionCallId"] + if "error" in expected: + assert "value" not in response, response + error = response["error"] + assert error["code"] == expected["error"]["code"], error + if "message" in expected["error"]: + assert expected["error"]["message"] in error["message"], error + else: + assert "error" not in response, response + assert "value" in response, response + assert response["value"] == expected["value"] + + +def _run_validate(case: Mapping[str, Any]) -> None: + args = case["args"] + catalogs = [_catalog(entry) for entry in args["catalogs"]] + if "expect_error" in case: + with pytest.raises(Exception) as excinfo: + validate_payload(catalogs, args["messages"]) + _assert_error_matches(excinfo.value, case["expect_error"]) + return + assert case["expect_valid"] is True + validate_payload(catalogs, args["messages"]) + + +class _FakeClockLoop(asyncio.SelectorEventLoop): + """An event loop whose clock moves only when `advance` moves it.""" + + def __init__(self) -> None: + super().__init__(selectors.DefaultSelector()) + self._now_ms = 0 + + def time(self) -> float: + return self._now_ms / 1000 + + def advance(self, ms: int) -> None: + self._now_ms += ms + + +async def _let_pending_work_run() -> None: + for _ in range(5): + await asyncio.sleep(0) + + +def _call_state(future: asyncio.Future[Any]) -> Any: + """A call's state in the suite's vocabulary.""" + if not future.done(): + return "pending" + error = future.exception() + if error is None: + return {"value": future.result()} + assert isinstance(error, A2uiRpcError), error + return {"error": {"code": error.code, "message": str(error)}} + + +def _assert_call_state(label: str, actual: Any, expected: Any) -> None: + if expected == "pending" or actual == "pending": + assert actual == expected, f"call {label}: {actual}" + return + if "value" in expected: + assert actual == expected, f"call {label}: {actual}" + return + assert "error" in actual, f"call {label}: {actual}" + assert actual["error"]["code"] == expected["error"]["code"], f"call {label}" + if "message" in expected["error"]: + assert expected["error"]["message"] in actual["error"]["message"] + + +async def _run_calls_on_loop(case: Mapping[str, Any], loop: _FakeClockLoop) -> None: + args = case.get("args", {}) + options: dict[str, Any] = {} + if "default_timeout_ms" in args: + timeout_ms = args["default_timeout_ms"] + options["default_timeout"] = None if timeout_ms is None else timeout_ms / 1000 + if "max_pending" in args: + options["max_pending"] = args["max_pending"] + calls = RendererFunctionCalls(**options) + futures: dict[str, asyncio.Future[Any]] = {} + call_ids: dict[str, str] = {} + + for index, step in enumerate(case["steps"]): + where = f"step {index}" + if "call" in step: + spec = step["call"] + kwargs: dict[str, Any] = { + "catalog_id": spec["catalogId"], + "name": spec["name"], + } + if "args" in spec: + kwargs["args"] = spec["args"] + if "surfaceId" in spec: + kwargs["surface_id"] = spec["surfaceId"] + if "timeout_ms" in spec: + kwargs["timeout"] = spec["timeout_ms"] / 1000 + if "expect_error" in step: + with pytest.raises(A2uiRpcError) as excinfo: + calls.call(**kwargs) + _assert_error_matches(excinfo.value, step["expect_error"]) + else: + message, future = calls.call(**kwargs) + dumped = message.model_dump(by_alias=True, exclude_unset=True) + body = dumped["callRendererFunction"] + call_id = body.pop("functionCallId") + assert isinstance(call_id, str) and call_id, where + assert call_id not in call_ids.values(), f"{where}: reused {call_id}" + call_ids[spec["label"]] = call_id + futures[spec["label"]] = future + if "expect_message" in step: + assert dumped == step["expect_message"], where + elif "respond" in step: + spec = step["respond"] + call_id = ( + call_ids[spec["call"]] if "call" in spec else spec["functionCallId"] + ) + response: dict[str, Any] = {"functionCallId": call_id} + if "error" in spec: + response["error"] = spec["error"] + else: + response["value"] = spec["value"] + calls.resolve({"version": "1.0", "rendererFunctionResponse": response}) + elif "advance_ms" in step: + loop.advance(step["advance_ms"]) + elif "dispose" in step: + reason = step["dispose"].get("reason") + if reason is None: + calls.dispose() + else: + calls.dispose(reason) + else: + raise ValueError(f"{where}: unknown step {step}") + + await _let_pending_work_run() + for label, expected in step.get("expect_calls", {}).items(): + _assert_call_state(label, _call_state(futures[label]), expected) + if "expect_pending_count" in step: + assert calls.pending_count == step["expect_pending_count"], where + + calls.dispose() + await _let_pending_work_run() + for future in futures.values(): + if not future.cancelled(): + future.exception() + + +def _run_calls(case: Mapping[str, Any]) -> None: + loop = _FakeClockLoop() + try: + loop.run_until_complete(_run_calls_on_loop(case, loop)) + finally: + loop.close() + + +RUNNERS: dict[str, Callable[[Mapping[str, Any]], None]] = { + "declare_agent_functions": _run_declare, + "handle_agent_function": _run_handle, + "validate_payload": _run_validate, + "call_renderer_function": _run_calls, +} + + +@pytest.mark.parametrize("case", CASES) +def test_agent_function_conformance(case: Mapping[str, Any]) -> None: + action = case["action"] + runner = RUNNERS.get(action) + if runner is None: + raise ValueError(f"Unhandled conformance action: {action}") + runner(case) diff --git a/typescript/agents/a2ui_agent/tests/conformance/agent-functions.test.ts b/typescript/agents/a2ui_agent/tests/conformance/agent-functions.test.ts new file mode 100644 index 0000000000..8a2bd7bab8 --- /dev/null +++ b/typescript/agents/a2ui_agent/tests/conformance/agent-functions.test.ts @@ -0,0 +1,47 @@ +/* + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import {describe, expect, test} from 'vitest'; + +import {classify, loadCases} from './loader.js'; +import {conformancePath} from './suite-helpers.js'; + +/** + * `agent_functions.yaml` covers agent functions and the agent's calls to renderer functions, + * which this SDK does not implement yet. Every case is registered as skipped with the reason + * `classify` gives, so the gap shows in the test report, and a case with an action that has + * no reason fails instead of passing without running. + */ +describe('Conformance: agent_functions.yaml', () => { + const cases = loadCases([conformancePath('agent/agent_functions.yaml')]); + + test('suite is not empty', () => { + expect(cases.length).toBeGreaterThan(0); + }); + + for (const testCase of cases) { + const title = `${testCase.action} · ${testCase.name}`; + const verdict = classify(testCase); + const run = () => { + throw new Error(`Unhandled conformance action: ${testCase.action}`); + }; + if (verdict.runnable) { + test(title, run); + } else { + test.skip(`${title} (${verdict.reason})`, run); + } + } +}); diff --git a/typescript/agents/a2ui_agent/tests/conformance/loader.ts b/typescript/agents/a2ui_agent/tests/conformance/loader.ts index 9b959ac9f6..fdc6f07813 100644 --- a/typescript/agents/a2ui_agent/tests/conformance/loader.ts +++ b/typescript/agents/a2ui_agent/tests/conformance/loader.ts @@ -116,7 +116,17 @@ const SUPPORTED_FORMATS = new Set(['direct_json', 'express']); * Conformance actions this SDK has no implementation for. Their cases are skipped with the * reason given, rather than falling through the harness and passing without running. */ -const UNIMPLEMENTED_ACTIONS = new Map([]); +const UNIMPLEMENTED_ACTIONS = new Map( + [ + 'declare_agent_functions', + 'handle_agent_function', + 'validate_payload', + 'call_renderer_function', + ].map((action): [string, string] => [ + action, + 'this SDK does not implement agent functions or calls to renderer functions yet', + ]), +); export function classify(testCase: TestCase) { const superseded = SUPERSEDED.get(testCase.name) ?? SUPERSEDED_ACTIONS.get(testCase.action); diff --git a/typescript/core/web_core/tests/conformance/conformance_test.mjs b/typescript/core/web_core/tests/conformance/conformance_test.mjs index c346bec8ba..44fdd4deb9 100644 --- a/typescript/core/web_core/tests/conformance/conformance_test.mjs +++ b/typescript/core/web_core/tests/conformance/conformance_test.mjs @@ -238,6 +238,10 @@ const UNIMPLEMENTED_ACTIONS = new Map([ ['transform_catalog', 'catalog transformers are agent-side only'], ['provide_catalog', 'catalog providers are agent-side only'], ['resolve_catalogs', 'catalog resolution is agent-side only'], + ['declare_agent_functions', 'agent functions are agent-side only'], + ['handle_agent_function', 'agent functions are agent-side only'], + ['validate_payload', 'validating model output is agent-side only'], + ['call_renderer_function', "the agent's calls to renderer functions are agent-side only"], ['common_types_schema', 'web_core does not generate the common types schema from its own models'], [ 'agent_to_renderer_schema', From 38bb93deb49dbe8c67494326dd91fc67e55dedcf Mon Sep 17 00:00:00 2001 From: Nan Yu Date: Fri, 30 Oct 2026 20:08:54 +0000 Subject: [PATCH 2/2] test(conformance): harden the agent function runner's fake-clock loop Set the fake clock before the base event loop constructor runs, install the loop as the current event loop while a case runs, and only read failures of futures that are done. Signed-off-by: Nan Yu --- .../conformance/test_agent_function_conformance.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/python/a2ui_agent/tests/conformance/test_agent_function_conformance.py b/python/a2ui_agent/tests/conformance/test_agent_function_conformance.py index 63d1f09688..fc47ad63d8 100644 --- a/python/a2ui_agent/tests/conformance/test_agent_function_conformance.py +++ b/python/a2ui_agent/tests/conformance/test_agent_function_conformance.py @@ -209,8 +209,9 @@ class _FakeClockLoop(asyncio.SelectorEventLoop): """An event loop whose clock moves only when `advance` moves it.""" def __init__(self) -> None: - super().__init__(selectors.DefaultSelector()) + # Set before the base constructor, which may already read the clock. self._now_ms = 0 + super().__init__(selectors.DefaultSelector()) def time(self) -> float: return self._now_ms / 1000 @@ -319,17 +320,22 @@ async def _run_calls_on_loop(case: Mapping[str, Any], loop: _FakeClockLoop) -> N calls.dispose() await _let_pending_work_run() + # Reads each failure, so asyncio doesn't log "exception was never + # retrieved". `dispose` settles every pending call, but a future that is + # somehow still pending is skipped rather than raising InvalidStateError. for future in futures.values(): - if not future.cancelled(): + if future.done() and not future.cancelled(): future.exception() def _run_calls(case: Mapping[str, Any]) -> None: loop = _FakeClockLoop() + asyncio.set_event_loop(loop) try: loop.run_until_complete(_run_calls_on_loop(case, loop)) finally: loop.close() + asyncio.set_event_loop(None) RUNNERS: dict[str, Callable[[Mapping[str, Any]], None]] = {