diff --git a/.github/workflows/agentcore-evaluation-provider-parity.yml b/.github/workflows/agentcore-evaluation-provider-parity.yml new file mode 100644 index 0000000..f28d825 --- /dev/null +++ b/.github/workflows/agentcore-evaluation-provider-parity.yml @@ -0,0 +1,137 @@ +name: AgentCore evaluation provider parity + +on: + workflow_dispatch: + inputs: + mode: + type: choice + options: [validate, direct-spans] + default: validate + confirmation: + type: string + required: false + +permissions: + contents: read + +concurrency: + group: cloudai-agentcore-evaluation-provider-parity + cancel-in-progress: false + +jobs: + validate: + if: ${{ inputs.mode == 'validate' }} + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Set up Node.js + uses: actions/setup-node@v4 + with: + node-version: "22" + + - name: Set up pnpm + uses: pnpm/action-setup@v4 + with: + version: 11.7.0 + + - name: Install API dependencies + working-directory: providers/aws/app/api + run: pnpm install --frozen-lockfile + + - name: Test API contracts + working-directory: providers/aws/app/api + run: pnpm test + + - name: Validate provider parity contracts + working-directory: providers/aws/app/api + run: pnpm agentcore-eval:provider-parity -- --mode validate + + direct-spans: + if: ${{ inputs.mode == 'direct-spans' }} + runs-on: ubuntu-latest + environment: aws-sandbox + permissions: + contents: read + id-token: write + env: + CONFIRMATION: ${{ inputs.confirmation }} + AGENTCORE_EVALUATION_READY: ${{ vars.AGENTCORE_EVALUATION_READY }} + AGENTCORE_EVALUATION_MAX_CALLS: ${{ vars.AGENTCORE_EVALUATION_MAX_CALLS }} + AWS_AGENTCORE_EVALUATION_ROLE_TO_ASSUME: ${{ vars.AWS_AGENTCORE_EVALUATION_ROLE_TO_ASSUME || secrets.AWS_AGENTCORE_EVALUATION_ROLE_TO_ASSUME }} + AWS_REGION: ${{ vars.AWS_REGION || 'ap-southeast-2' }} + PROVIDER_PARITY_MODE: direct-spans + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Preflight protected direct evaluation + shell: bash + run: | + set -euo pipefail + [[ "$CONFIRMATION" == "I_UNDERSTAND_AGENTCORE_EVALUATION_PROVIDER_PARITY" ]] || { + echo "exact confirmation is required" >&2 + exit 1 + } + [[ "$AGENTCORE_EVALUATION_READY" == "true" ]] || { + echo "evaluation readiness is not enabled" >&2 + exit 1 + } + [[ "$AGENTCORE_EVALUATION_MAX_CALLS" == "6" ]] || { + echo "the direct evaluation call cap must be six" >&2 + exit 1 + } + [[ "$AWS_REGION" == "ap-southeast-2" ]] || { + echo "the protected evaluation region must be ap-southeast-2" >&2 + exit 1 + } + [[ -n "$AWS_AGENTCORE_EVALUATION_ROLE_TO_ASSUME" ]] || { + echo "the protected evaluation role is required" >&2 + exit 1 + } + [[ "$GITHUB_REF" == "refs/heads/main" ]] || { + echo "the protected evaluation must run from main" >&2 + exit 1 + } + [[ "$GITHUB_SHA" =~ ^[0-9a-f]{40}$ ]] || { + echo "the protected evaluation requires a full lowercase commit SHA" >&2 + exit 1 + } + + - name: Set up Node.js + uses: actions/setup-node@v4 + with: + node-version: "22" + + - name: Set up pnpm + uses: pnpm/action-setup@v4 + with: + version: 11.7.0 + + - name: Install API dependencies + working-directory: providers/aws/app/api + run: pnpm install --frozen-lockfile + + - name: Test API contracts + working-directory: providers/aws/app/api + run: pnpm test + + - name: Configure protected AWS role + uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: ${{ env.AWS_AGENTCORE_EVALUATION_ROLE_TO_ASSUME }} + aws-region: ${{ env.AWS_REGION }} + mask-aws-account-id: true + + - name: Run direct provider parity evaluation + working-directory: providers/aws/app/api + run: pnpm agentcore-eval:provider-parity -- --mode direct-spans --output "$RUNNER_TEMP/provider-direct-evaluation-report.json" + + - name: Upload sanitized provider parity report + uses: actions/upload-artifact@v4 + with: + name: provider-direct-evaluation-report + path: ${{ runner.temp }}/provider-direct-evaluation-report.json + if-no-files-found: error + retention-days: 7 diff --git a/.github/workflows/update-aws-bootstrap.yml b/.github/workflows/update-aws-bootstrap.yml index acbfd23..a6a5f7c 100644 --- a/.github/workflows/update-aws-bootstrap.yml +++ b/.github/workflows/update-aws-bootstrap.yml @@ -163,6 +163,26 @@ jobs: --query 'Stacks[0].{StackStatus:StackStatus,LastUpdatedTime:LastUpdatedTime}' \ --output json + - name: Publish AgentCore evaluation role handoff + if: ${{ github.event_name == 'workflow_dispatch' && inputs.mode == 'apply' && success() }} + run: | + agentcore_evaluation_role_arn="$(aws cloudformation describe-stacks \ + --stack-name "$AWS_BOOTSTRAP_STACK_NAME" \ + --query "Stacks[0].Outputs[?OutputKey=='AgentCoreEvaluationRoleArn'].OutputValue | [0]" \ + --output text)" + if [ -z "$agentcore_evaluation_role_arn" ] || [ "$agentcore_evaluation_role_arn" = "None" ]; then + echo "::error::Bootstrap stack did not return AgentCoreEvaluationRoleArn." + exit 1 + fi + echo "::add-mask::$agentcore_evaluation_role_arn" + { + printf '%s\n' '## AgentCore evaluation Environment handoff' + printf '%s\n' '' + printf '%s\n' 'Add this exact masked value as the protected aws-sandbox Environment setting `AWS_AGENTCORE_EVALUATION_ROLE_TO_ASSUME`:' + printf '%s\n' '' + printf '`%s`\n' "$agentcore_evaluation_role_arn" + } >> "$GITHUB_STEP_SUMMARY" + - name: Publish Budget Guardrails role handoff if: ${{ github.event_name == 'workflow_dispatch' && inputs.mode == 'apply' && success() }} run: | diff --git a/docs/architecture/agentcore-governed-rag-poc.md b/docs/architecture/agentcore-governed-rag-poc.md index 4737ba2..40c167f 100644 --- a/docs/architecture/agentcore-governed-rag-poc.md +++ b/docs/architecture/agentcore-governed-rag-poc.md @@ -174,25 +174,39 @@ the sandbox is currently kept deployed for demonstrations. The evaluation boundary separates an agent framework from the quality gate. Synthetic OpenTelemetry GenAI and OpenInference traces are normalized into one -contract before deterministic scenario and trajectory checks are applied: +contract before deterministic scenario and trajectory checks are applied. The +direct evaluation path is beside, not inside, the deployed Runtime and +CloudWatch path: ```text -Agent framework - -> OpenTelemetry GenAI or OpenInference spans - -> framework-neutral normalizer - -> deterministic local dimensions - -> versioned thresholds - -> metadata-only CI evidence - -> optional protected AgentCore parity evaluation +local fixtures -> direct sessionSpans -> AgentCore Evaluate -> provider-direct evidence + +Gateway -> Runtime -> ADOT -> CloudWatch -> AgentCore Evaluate -> provider-runtime evidence + Stage B: not implemented by this change ``` Accepted scopes are limited to `opentelemetry.instrumentation.*` and `openinference.instrumentation.*`. The gate checks invoke-agent, inference, and execute-tool evidence, including `local.telemetry_compatibility` and -`local.tool_trajectory_accuracy`. It is locally contract-tested and its -ordinary CI job does not call AWS. OTLP export, CloudWatch trace ingestion, -and managed AgentCore scoring remain a future protected provider-parity lane. -The operating procedure and non-claim boundary are in the +`local.tool_trajectory_accuracy`. `local-contract` evidence is locally +contract-tested and its ordinary CI job does not call AWS. Stage A source +implements the `provider-parity-v1` fixed, six-call direct-spans request matrix +for future `provider-direct` evidence; provider validation is pending. It +precedes Runtime ingestion so the reviewed spans, scenario, evaluator matrix, +and metadata boundary can be checked without claiming that the Runtime emits +or CloudWatch receives them. Stage B Runtime-to-CloudWatch evidence is the +future `provider-runtime` lane and is not implemented. + +Within Stage A, Correctness uses a trace-scoped expected response, +ToolSelectionAccuracy uses only a targeted tool span, and GoalSuccessRate uses +a session-scoped assertion. Managed trajectory parity is outside this fixed +profile and would require a separately reviewed `Builtin.Trajectory*` policy. + +Managed scores supplement deterministic controls. They never authorize IAM, +admission or approval, tool execution, deployment, remediation, rollback, or +deletion. No provider, runtime, or production evaluation has been validated. +The operating procedure +and non-claim boundary are in the [agent evaluation telemetry runbook](../solutions/agent-evaluation-telemetry-runbook.md). ### 5. What this diagram does and does not claim diff --git a/docs/practices/current-status.md b/docs/practices/current-status.md index 39bd53c..6d69a63 100644 --- a/docs/practices/current-status.md +++ b/docs/practices/current-status.md @@ -57,11 +57,12 @@ The repository can now demonstrate: IAM Gateway/Runtime target, and confirmation-gated CI ingestion/invocation controls deployed; direct Bedrock preflight, Gateway end-to-end evidence, and bounded CloudWatch observability are complete; teardown remains separately gated -- **Framework-neutral agent evaluation telemetry:** locally contract-tested - OpenTelemetry GenAI and OpenInference normalization, fixed synthetic prompts, - expected tool trajectories, strict deterministic dimensions, and a - metadata-only required CI artifact; the local gate does not call AWS and the - protected provider-parity lane remains pending +- **Framework-neutral agent evaluation telemetry:** Stage A source implemented; + provider validation pending. The protected lane is manual, synthetic-only, + evaluate-only, and bounded to six calls. Stage B Runtime-to-CloudWatch + evaluation is not implemented. `local-contract` remains the only validated + evidence; future `provider-direct` and `provider-runtime` evidence are not + provider, runtime, or production validation. - **P5a AI-Assisted DevSecOps Boundary:** advisory AI use, human review, CI/security checks, and release evidence - **P5b AI-Assisted Review Evidence:** review summaries, threat-model checklists, CI failure summaries, and release-note drafts - **P6f AI Platform Security and Operations Controls:** identity, data protection, AI AppSec, delivery, operations, and FinOps @@ -122,7 +123,7 @@ It currently has six mock-first lanes: | P8h AgentCore knowledge-lookup readiness | Complete static gateway-first reference architecture; no AgentCore resource or call | `docs/solutions/p8h-agentcore-knowledge-lookup-readiness.md` | | P8i AgentCore synthetic contract pack | Complete local synthetic contract evidence; provider-neutral pack remains separate from the live AWS validation | `shared/schemas/agentcore-readiness/`, `shared/examples/agentcore-readiness/`, `providers/aws/app/api/tests/agentcoreReadinessContracts.test.ts`, and `docs/solutions/p8i-agentcore-synthetic-contract-pack.md` | | AgentCore governed RAG POC | Synthetic data foundation, arm64 Runtime, IAM Gateway/Runtime target, direct Bedrock preflight, Gateway end-to-end evidence, and bounded CloudWatch observability complete through protected CI; teardown remains separately gated | `providers/aws/app/agentcore-rag-runtime/`, `providers/aws/agentcore/`, `.github/workflows/terraform-agentcore-rag-sandbox.yml`, `providers/aws/infra/bootstrap/github-oidc-terraform-backend.yaml`, `docs/solutions/p8i-agentcore-rag-data-foundation.md`, `docs/solutions/p8i-agentcore-rag-key-process-record.md`, and `docs/solutions/agentcore-governed-rag-poc-runbook.md` | -| Framework-neutral agent evaluation telemetry | Locally contract-tested with synthetic OpenTelemetry GenAI and OpenInference fixtures; required CI gate does not call AWS; protected provider-parity lane pending | `shared/schemas/agent-evaluation-telemetry/`, `shared/examples/agent-evaluation-telemetry/`, `providers/aws/app/api/src/evals/agentEvaluationTelemetryNormalizer.ts`, `providers/aws/app/api/src/evals/agentEvaluationTelemetryGate.ts`, and `docs/solutions/agent-evaluation-telemetry-runbook.md` | +| Framework-neutral agent evaluation telemetry | Stage A source implemented; provider validation pending. The protected lane is manual, synthetic-only, evaluate-only, and bounded to six calls. Stage B Runtime-to-CloudWatch evaluation is not implemented. | `shared/schemas/agent-evaluation-telemetry/`, `shared/examples/agent-evaluation-telemetry/`, `providers/aws/app/api/src/evals/agentEvaluationTelemetryNormalizer.ts`, `providers/aws/app/api/src/evals/agentEvaluationTelemetryGate.ts`, `providers/aws/app/api/src/evals/agentCoreEvaluationProviderGate.ts`, `.github/workflows/agentcore-evaluation-provider-parity.yml`, and `docs/solutions/agent-evaluation-telemetry-runbook.md` | | P5a AI-assisted DevSecOps boundary | Complete | `docs/practices/ai-assisted-devsecops-pattern.md` and `.github/workflows/ai-assisted-devsecops.yml` | | P5b AI-assisted review evidence | Complete | `docs/evidence/ai-assisted-review-evidence.md`, `shared/schemas/ai-assisted-devsecops/`, and `shared/examples/ai-assisted-devsecops/` | | P6d control-plane evidence map | Complete | `docs/evidence/control-plane-evidence-map.md`, `shared/schemas/control-plane-evidence/`, and `shared/examples/control-plane-evidence/` | diff --git a/docs/solutions/agent-evaluation-telemetry-runbook.md b/docs/solutions/agent-evaluation-telemetry-runbook.md index 0abf712..bc04272 100644 --- a/docs/solutions/agent-evaluation-telemetry-runbook.md +++ b/docs/solutions/agent-evaluation-telemetry-runbook.md @@ -8,9 +8,8 @@ conventions or OpenInference attributes, but both paths are normalized into the same versioned evaluation contract before a pull request can pass. The ordinary CI gate is deterministic, synthetic-only, and provider-neutral. -It does not call AWS. A protected provider-parity lane may later compare the -same scenarios with Amazon Bedrock AgentCore Evaluations, but that is a -separate, manually approved evidence level. +It does not call AWS. The protected provider-parity lane is Stage A source +only; provider validation is pending. This runbook is its only operator guide. ## Telemetry Compatibility Contract @@ -32,8 +31,7 @@ Agent framework -> framework-neutral normalizer -> deterministic local dimensions -> versioned thresholds - -> metadata-only CI evidence - -> optional protected AgentCore parity evaluation + -> metadata-only `local-contract` evidence ``` ## Fixed Inputs @@ -136,39 +134,110 @@ Forbidden artifact fields: 6. Keep the metadata-only report contract and existing thresholds intact. 7. Run the complete API tests and the standalone gate. -## Protected Provider-Parity Lane +## Evidence lanes and current boundary -A future protected provider-parity lane may run the same synthetic cases with -Amazon Bedrock AgentCore on-demand evaluation. It must remain optional and -must not weaken the required local gate. Before it runs, require GitHub OIDC, -an approved environment, a cost limit, an exact confirmation phrase, and a -reviewed cleanup boundary. +```text +Lane 1: local CI + synthetic fixtures -> deterministic gate -> local-contract evidence + +Lane 2: Stage A, protected direct spans + fixed synthetic spans -> AgentCore Evaluate -> provider-direct evidence -That lane must also: +Lane 3: Stage B, future Runtime-to-CloudWatch + Runtime -> ADOT -> CloudWatch -> AgentCore Evaluate -> provider-runtime evidence + Stage B: not implemented by this change +``` -- emit supported OpenTelemetry GenAI or OpenInference instrumentation scopes; -- preserve `session.id` across runtime and evaluation telemetry; -- emit invoke-agent, inference, and execute-tool spans; -- enable only the minimum synthetic message content required for response - evaluation; -- wait for CloudWatch trace ingestion before requesting on-demand evaluation; -- use fixed prompts, expected tool trajectories, and versioned score - thresholds; -- publish a separately labelled provider-parity artifact with the same - metadata restrictions; -- avoid representing a provider score as an authorization decision. +`local-contract` is the existing cloud-free, deterministic evidence. Stage A +is source implemented only: its `provider-parity-v1` policy creates six direct +requests using the fixed cited-answer scenario, two conventions, and three +evaluators, including `Builtin.ToolSelectionAccuracy`. A successful protected +execution would produce separately labelled `provider-direct` evidence. It +has not been provider validated. `provider-runtime` is reserved for a future +Stage B Runtime-to-CloudWatch path and has not been implemented or validated. + +The Stage A ground-truth contract is evaluator-specific: Correctness receives +one trace-scoped `expectedResponse`, ToolSelectionAccuracy receives only the +targeted tool span and no reference input, and GoalSuccessRate receives one +session-scoped `assertions` reference. For generic OpenTelemetry input, the +invoke-agent prompt and final response are emitted as clean +`gen_ai.task.input` and `gen_ai.task.output` strings; inference and tool spans +retain their documented fields. The fixed managed profile does not measure +trajectory parity. That remains a deterministic local dimension; adding a +managed `Builtin.Trajectory*` evaluator requires a new reviewed policy and call +budget. + +Managed scores supplement deterministic controls. They never authorize IAM, +admission or approval, tool execution, deployment, remediation, rollback, or +deletion. No evidence lane proves provider, runtime, or production validation. + +## Cloud-free validation + +Run the source and contract validation from the repository root: -The framework-neutral gate is locally contract-tested against synthetic -OpenTelemetry GenAI and OpenInference fixtures. It does not prove OTLP export, -CloudWatch ingestion, AgentCore managed evaluation, or production agent -quality. Those require a separately approved provider-parity run. +```bash +corepack pnpm@11.7.0 --dir providers/aws/app/api agentcore-eval:provider-parity -- \ + --mode validate +``` -## Evidence Classification +This command uses deterministic local fakes, makes no AWS call, assumes no +role, creates no provider artifact, and produces no `provider-direct` claim. + +## Protected Stage A direct-span preflight + +Stage A is a manual GitHub Actions workflow-dispatch lane. It is +synthetic-only, evaluate-only, and bounded to six calls. Before a separately +approved run, an operator must verify all of the following in the protected +environment and workflow UI: + +1. Protected-environment approval is present and the selected revision is the + current `main` revision with a full commit identifier. +2. The fixed scenario is `synthetic-cited-answer`; no prompt, response, + fixture, evaluator, threshold, or policy is substituted. +3. The `provider-parity-v1` evaluator matrix is exactly `Builtin.Correctness`, + `Builtin.ToolSelectionAccuracy`, and `Builtin.GoalSuccessRate` for each of + the two approved telemetry conventions. +4. The exact call budget is `AGENTCORE_EVALUATION_MAX_CALLS=6`; no retry, + expansion, or additional scenario is permitted. +5. The workflow-dispatch confirmation is exactly + `I_UNDERSTAND_AGENTCORE_EVALUATION_PROVIDER_PARITY`. + +Protected environment settings are named here by purpose only. Do not place +their values, role identifiers, endpoints, credentials, or account details in +this runbook, an issue, or an artifact: + +| Setting | Safe purpose | +| --- | --- | +| `AGENTCORE_EVALUATION_READY` | Enables the protected evaluate-only lane after environment approval. | +| `AGENTCORE_EVALUATION_MAX_CALLS` | Locks the reviewed direct-evaluation budget to six calls. | +| `AWS_AGENTCORE_EVALUATION_ROLE_TO_ASSUME` | Supplies the dedicated evaluate-only role to the protected OIDC workflow. | +| `AWS_REGION` | Selects the reviewed protected evaluation region. | + +This PR authorizes neither a role apply nor the first AWS run. Do not invoke +the direct lane from a laptop or add a local AWS command to this guide. + +## Stage A artifacts and bounded failure handling + +The only publishable Stage A artifact is a sanitized `provider-direct` report. +Allowed fields are the policy/contract version, evidence level, source commit, +workflow run identifier, synthetic scenario and convention identifiers, +evaluator IDs, bounded scores and thresholds, pass/fail status, reason codes, +and a duration bucket. Forbidden fields are raw prompts, responses, message +content, tool arguments, tool results, retrieved content, provider response +payloads, credentials, secrets, account identifiers, role identifiers, ARNs, +and endpoints. + +On a preflight, request, validation, or artifact failure, stop after the +bounded runner outcome. Inspect only its bounded reason code and the +workflow's sanitized status. Do not paste raw provider output into issues, +notes, logs, or commits; do not retry beyond the six-call budget; and do not +turn a score into an authorization or remediation action. Escalate any future +role change, AWS run, or Stage B work through a separately reviewed change. + +## Evidence classification | Evidence | Current state | Claim allowed | | --- | --- | --- | -| Local normalization and deterministic gate | Complete | Locally contract-tested across two telemetry conventions. | -| Pull-request quality gate | Implemented | CI enforces fixed synthetic scenarios without AWS access. | -| OTLP export and CloudWatch ingestion | Pending | No provider telemetry-delivery claim. | -| AgentCore managed evaluation | Pending | No managed evaluator or provider-score claim. | -| Production agent quality | Out of scope | No production-quality or safety-effectiveness claim. | +| `local-contract` deterministic gate | Complete | Locally contract-tested across two telemetry conventions without AWS access. | +| Stage A `provider-direct` | Source implemented; provider validation pending | No provider-validation claim until a separately approved protected run. | +| Stage B `provider-runtime` | Not implemented | No Runtime-to-CloudWatch, provider, runtime, or production-evaluation claim. | diff --git a/docs/solutions/p8i-agentcore-rag-key-process-record.md b/docs/solutions/p8i-agentcore-rag-key-process-record.md index 8d118bd..249fd38 100644 --- a/docs/solutions/p8i-agentcore-rag-key-process-record.md +++ b/docs/solutions/p8i-agentcore-rag-key-process-record.md @@ -172,10 +172,29 @@ completeness, tool-trajectory accuracy, behavioural outcome, and goal success. The pull-request job runs without AWS credentials, does not call AWS, and retains only a metadata-safe report. This work is locally contract-tested; it does not prove OTLP delivery, CloudWatch trace ingestion, AgentCore managed -evaluation, or production agent quality. Those remain a separately approved -protected provider-parity lane described in the +evaluation, or production agent quality. + +Stage A source now defines a `provider-parity-v1` direct-spans request matrix: +the fixed cited-answer scenario, two telemetry conventions, and three managed +evaluators, including `Builtin.ToolSelectionAccuracy`, for six bounded calls. +It exists beside the Runtime path so reviewed direct spans can be assessed +before Runtime ingestion. Any resulting evidence would be `provider-direct`, +not `local-contract`; provider validation is pending. Stage B +Runtime-to-CloudWatch evaluation would produce `provider-runtime` evidence, +but it is not implemented by this change. Managed scores supplement deterministic +controls. They never authorize IAM, admission or approval, tool execution, +deployment, remediation, rollback, or deletion. The protected +workflow requires an exact confirmation and evaluate-only identity, but this PR authorizes neither role +apply nor the first AWS run. The operator procedure is the [evaluation telemetry runbook](./agent-evaluation-telemetry-runbook.md). +The reviewed provider profile uses only evaluator-supported inputs: +Correctness gets one trace-scoped expected response, ToolSelectionAccuracy gets +the targeted tool span without a ground-truth reference, and GoalSuccessRate +gets one session-scoped assertion. Trajectory parity remains in the local +deterministic gate; a managed trajectory evaluator would require a new policy +and call budget. + ## Current next work 1. Keep the sandbox deployed for interview demonstrations while monitoring cost and quotas. diff --git a/docs/superpowers/plans/2026-08-29-agentcore-evaluation-provider-parity.md b/docs/superpowers/plans/2026-08-29-agentcore-evaluation-provider-parity.md new file mode 100644 index 0000000..83f4446 --- /dev/null +++ b/docs/superpowers/plans/2026-08-29-agentcore-evaluation-provider-parity.md @@ -0,0 +1,1131 @@ +# AgentCore Evaluation Provider-Parity Stage A Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Add a protected, framework-neutral AgentCore on-demand evaluation lane that submits one fixed synthetic cited-answer session in OpenTelemetry GenAI and OpenInference form, evaluates each with three fixed built-in evaluators, and publishes only fail-closed metadata evidence. + +**Architecture:** Ordinary pull-request CI remains local and deterministic. A new manually dispatched Stage A workflow has a cloud-free `validate` job and an environment-protected `direct-spans` job; the latter obtains a dedicated OIDC role only after confirmation, readiness, source-revision, region, and six-call-budget checks succeed. Pure TypeScript converts the existing provider-neutral fixtures to AgentCore `sessionSpans`, an injected client isolates AWS access, and a separate gate sanitizes results and compares both conventions. + +**Tech Stack:** TypeScript 5.9 / Node.js 22, Node test runner, JSON Schema 2020-12, `@aws-sdk/client-bedrock-agentcore` 3.1121.0, GitHub Actions OIDC, AWS CloudFormation, Ruby Minitest, cfn-lint. + +**Spec:** `docs/superpowers/specs/2026-08-29-agentcore-evaluation-provider-parity-design.md` + +## Global Constraints + +- Stage A accepts only `synthetic-cited-answer`; all six scenarios under both conventions remain covered by the existing local `strict-v1` gate. +- Supported conventions are exactly `otel-genai` and `openinference` with scopes under `opentelemetry.instrumentation.*` and `openinference.instrumentation.*` respectively. +- Evaluators are exactly `Builtin.Correctness`, `Builtin.ToolSelectionAccuracy`, and `Builtin.GoalSuccessRate`. +- The fixed managed profile does not measure trajectory parity: `Builtin.ToolSelectionAccuracy` uses only the targeted tool span. Adding a `Builtin.Trajectory*` evaluator requires a new reviewed policy and call budget. +- Threshold profile `provider-parity-v1` requires every evaluator score to be at least `0.70` and each cross-convention absolute delta to be at most `0.20`. +- A direct run makes exactly six serial `Evaluate` calls and has a hard maximum of six; evaluator IDs and thresholds are code-owned, not workflow inputs. +- The exact direct-run phrase is `I_UNDERSTAND_AGENTCORE_EVALUATION_PROVIDER_PARITY`. +- Direct execution requires `AGENTCORE_EVALUATION_READY=true`, `AGENTCORE_EVALUATION_MAX_CALLS=6`, `AWS_REGION=ap-southeast-2`, `refs/heads/main`, and a 40-character source SHA. +- Ordinary `pull_request` and `push` CI must never request an OIDC token, obtain AWS credentials, or call AgentCore Evaluations. +- Provider prompts, responses, assertions, trajectories, tool arguments/results, raw spans, request bodies, provider responses, explanations, error messages, account IDs, ARNs, endpoints, and environment values must never enter the evidence artifact or logs. +- Managed scores are quality evidence only; they never authorize IAM, admission, approval, execution, remediation, rollback, or deletion. +- Source implementation does not authorize an AWS call, CloudFormation plan/apply, IAM mutation, Runtime image release, CloudWatch change, or deletion. +- Stage B Runtime-to-CloudWatch evaluation is outside this implementation plan and requires its own reviewed spec after Stage A provider evidence succeeds. + +## File Structure + +### New files + +- `shared/examples/agent-evaluation-telemetry/provider-parity-thresholds.v1.json` — fixed evaluator allowlist, score thresholds, parity tolerance, and call cap. +- `shared/schemas/agent-evaluation-telemetry/provider-parity-report.schema.json` — metadata-only provider-direct artifact contract. +- `providers/aws/app/api/src/evals/agentCoreEvaluationProviderTypes.ts` — Stage A request, result, report, policy, and injected-client types. +- `providers/aws/app/api/src/evals/agentCoreEvaluationRequestBuilder.ts` — validates the fixed fixture and constructs typed AgentCore requests with deterministic valid IDs. +- `providers/aws/app/api/src/evals/agentCoreEvaluationProviderGate.ts` — sanitizes results, enforces thresholds and parity, and constructs provider-direct evidence. +- `providers/aws/app/api/src/clients/agentCoreEvaluationClient.ts` — narrow injected client plus real AWS SDK adapter. +- `providers/aws/app/api/src/scripts/runAgentCoreEvaluationProviderParity.ts` — cloud-free validation and protected direct execution entry point. +- `providers/aws/app/api/tests/agentCoreEvaluationProviderContracts.test.ts` — policy and evidence schema boundary tests. +- `providers/aws/app/api/tests/agentCoreEvaluationRequestBuilder.test.ts` — request equivalence, ID, target, reference, and rejection tests. +- `providers/aws/app/api/tests/agentCoreEvaluationProviderGate.test.ts` — fake-client, partial-result, threshold, parity, and sanitization tests. +- `providers/aws/app/api/tests/runAgentCoreEvaluationProviderParity.test.ts` — preflight-before-client and exact-six-call runner tests. +- `providers/aws/app/api/tests/agentCoreEvaluationProviderWorkflow.test.ts` — static protected-workflow and no-AWS-PR boundary tests. +- `.github/workflows/agentcore-evaluation-provider-parity.yml` — manual validate/direct-spans workflow. + +### Modified files + +- `providers/aws/app/api/package.json` and `providers/aws/app/api/pnpm-lock.yaml` — pin AgentCore data-plane SDK and expose the Stage A CLI. +- `providers/aws/infra/bootstrap/github-oidc-terraform-backend.yaml` — add the dedicated evaluate-only OIDC role and output. +- `providers/aws/infra/bootstrap/test_github_oidc_terraform_backend.rb` — prove trust and least privilege statically. +- `.github/workflows/update-aws-bootstrap.yml` — publish a masked environment handoff for the dedicated role after a separately approved bootstrap apply. +- `providers/aws/infra/bootstrap/README.md` — document role separation and CI/CD-only handoff. +- `docs/solutions/agent-evaluation-telemetry-runbook.md` — operator workflow, evidence levels, and Stage A/Stage B boundary. +- `docs/architecture/agentcore-governed-rag-poc.md` — add provider-direct evaluation lane without claiming Runtime telemetry ingestion. +- `docs/solutions/p8i-agentcore-rag-key-process-record.md` — record decisions, source state, and next approval gates. +- `docs/practices/current-status.md` — mark Stage A source implementation separately from provider validation. +- `providers/aws/app/api/README.md` — local validate command and protected execution boundary. +- `providers/aws/app/api/tests/agentEvaluationTelemetryDocumentation.test.ts` — prevent status overclaim and require new documentation terms. + +--- + +### Task 1: Version the provider policy and metadata-only artifact contract + +**Files:** +- Create: `shared/examples/agent-evaluation-telemetry/provider-parity-thresholds.v1.json` +- Create: `shared/schemas/agent-evaluation-telemetry/provider-parity-report.schema.json` +- Create: `providers/aws/app/api/src/evals/agentCoreEvaluationProviderTypes.ts` +- Create: `providers/aws/app/api/tests/agentCoreEvaluationProviderContracts.test.ts` + +**Interfaces:** +- Consumes: `EvaluationConvention`, `EvaluationScenario`, and `TelemetryFixture` from `agentEvaluationTelemetryTypes.ts`. +- Produces: `ProviderEvaluatorId`, `ProviderParityPolicy`, `ProviderEvaluationRequest`, `ProviderEvaluationResponse`, `ProviderParityReport`, and `AgentCoreEvaluateClient` for Tasks 2–4. + +- [ ] **Step 1: Write the failing contract test** + +Create `providers/aws/app/api/tests/agentCoreEvaluationProviderContracts.test.ts` with tests that load both new JSON files and require the exact immutable policy: + +```ts +import assert from "node:assert/strict"; +import { readFile } from "node:fs/promises"; +import { resolve } from "node:path"; +import test from "node:test"; + +const ROOT = resolve(process.cwd(), "../../../.."); +const EXAMPLES = resolve(ROOT, "shared/examples/agent-evaluation-telemetry"); +const SCHEMAS = resolve(ROOT, "shared/schemas/agent-evaluation-telemetry"); + +test("provider-parity-v1 fixes three evaluators, thresholds, tolerance, and six calls", async () => { + const policy = JSON.parse(await readFile( + resolve(EXAMPLES, "provider-parity-thresholds.v1.json"), "utf8")); + assert.deepEqual(policy, { + contractVersion: "1.0", + profileId: "provider-parity-v1", + scenarioId: "synthetic-cited-answer", + evaluatorThresholds: { + "Builtin.Correctness": 0.70, + "Builtin.ToolSelectionAccuracy": 0.70, + "Builtin.GoalSuccessRate": 0.70 + }, + maximumParityDelta: 0.20, + maximumProviderCalls: 6 + }); +}); + +test("provider-direct schema is closed and contains no raw-content fields", async () => { + const schema = JSON.parse(await readFile( + resolve(SCHEMAS, "provider-parity-report.schema.json"), "utf8")); + assert.equal(schema.additionalProperties, false); + assert.equal(schema.properties.evidenceLevel.const, "provider-direct"); + const serialized = JSON.stringify(schema).toLowerCase(); + for (const forbidden of [ + "prompt", "response", "assertion", "trajectory", "toolarguments", + "toolresult", "sessionspans", "explanation", "errormessage", + "accountid", "resourcearn", "endpoint" + ]) assert.equal(serialized.includes(`\"${forbidden}\"`), false, forbidden); +}); +``` + +- [ ] **Step 2: Run the focused test and verify it fails** + +Run: + +```bash +corepack pnpm@11.7.0 --dir providers/aws/app/api test +``` + +Expected: FAIL because `provider-parity-thresholds.v1.json` and `provider-parity-report.schema.json` do not exist. + +- [ ] **Step 3: Add the fixed policy file** + +Create `provider-parity-thresholds.v1.json` with exactly the object asserted above. Do not add environment-variable interpolation or alternate evaluator IDs. + +- [ ] **Step 4: Add the closed provider-direct report schema** + +Define required top-level properties: + +```json +{ + "contractVersion": "1.0", + "thresholdVersion": "1.0", + "evidenceLevel": "provider-direct", + "generatedAt": "2026-08-29T00:00:00.000Z", + "sourceCommit": "40-character Git SHA", + "githubRunId": "numeric GitHub run ID", + "regionLabel": "ap-southeast-2", + "scenarioId": "synthetic-cited-answer", + "status": "passed", + "providerCallCount": 6, + "durationBucket": "under-5m", + "aggregateTokenUsage": { "inputTokens": 0, "outputTokens": 0, "totalTokens": 0 }, + "results": [], + "parity": [] +} +``` + +Each result permits only `convention`, `evaluatorId`, `level`, `score`, `label`, `threshold`, `passed`, `reasonCode`, and a token-usage object. Each parity item permits only `evaluatorId`, `otelGenaiScore`, `openInferenceScore`, `absoluteDelta`, `maximumDelta`, and `passed`. Use `additionalProperties: false` on every object, exact enums for conventions/evaluators/levels, `0..1` numeric bounds, a `^[a-z0-9_]+$` reason-code pattern, and `minimum: 0` integer token counts. + +- [ ] **Step 5: Add exact TypeScript provider types** + +Create `agentCoreEvaluationProviderTypes.ts` with these public contracts: + +```ts +import type { EvaluationConvention } from "./agentEvaluationTelemetryTypes.js"; + +export type ProviderEvaluatorId = + | "Builtin.Correctness" + | "Builtin.ToolSelectionAccuracy" + | "Builtin.GoalSuccessRate"; +export type ProviderEvaluationLevel = "trace" | "tool-call" | "session"; + +export type ProviderParityErrorCode = + | "provider_fixture_not_synthetic" + | "provider_scenario_not_allowed" + | "provider_scope_not_allowed" + | "provider_required_span_missing" + | "provider_session_count_invalid" + | "provider_attribute_not_allowed" + | "provider_policy_invalid" + | "provider_call_count_invalid" + | "provider_result_missing" + | "provider_result_duplicate" + | "provider_evaluator_unexpected" + | "provider_result_failed" + | "provider_reference_input_ignored" + | "provider_score_invalid" + | "provider_context_mismatch" + | "provider_token_usage_invalid" + | "provider_label_invalid" + | "provider_score_below_threshold" + | "provider_parity_delta_exceeded" + | "provider_result_coverage_invalid" + | "provider_confirmation_required" + | "provider_readiness_required" + | "provider_call_budget_invalid" + | "provider_region_invalid" + | "provider_source_ref_invalid" + | "provider_source_commit_invalid" + | "provider_output_path_required" + | "provider_input_file_invalid" + | "provider_request_failed" + | "provider_artifact_write_failed"; + +export class ProviderParityError extends Error { + constructor(public readonly code: ProviderParityErrorCode) { + super(code); + this.name = "ProviderParityError"; + } +} + +export type ProviderParityPolicy = { + contractVersion: "1.0"; + profileId: "provider-parity-v1"; + scenarioId: "synthetic-cited-answer"; + evaluatorThresholds: Record; + maximumParityDelta: number; + maximumProviderCalls: 6; +}; + +export type ProviderDocument = null | boolean | number | string | ProviderDocument[] | { + [key: string]: ProviderDocument; +}; + +export type ProviderEvaluationRequest = { + evaluatorId: ProviderEvaluatorId; + evaluationInput: { sessionSpans: Array<{ [key: string]: ProviderDocument }> }; + evaluationTarget?: { traceIds: string[] } | { spanIds: string[] }; + evaluationReferenceInputs?: Array<{ + context: { spanContext: { sessionId: string; traceId?: string; spanId?: string } }; + expectedResponse?: { text: string }; + assertions?: Array<{ text: string }>; + }>; +}; + +export type ProviderEvaluationResponse = { + evaluationResults?: Array<{ + evaluatorId?: string; + value?: number; + label?: string; + errorCode?: string; + errorMessage?: string; + explanation?: string; + ignoredReferenceInputFields?: string[]; + context?: { spanContext?: { sessionId?: string; traceId?: string; spanId?: string } }; + tokenUsage?: { inputTokens?: number; outputTokens?: number; totalTokens?: number }; + }>; +}; + +export interface AgentCoreEvaluateClient { + evaluate(request: ProviderEvaluationRequest): Promise; +} + +export type ProviderParityResult = { + convention: EvaluationConvention; + evaluatorId: ProviderEvaluatorId; + level: ProviderEvaluationLevel; + score: number; + label: string; + threshold: number; + passed: boolean; + reasonCode: string; + tokenUsage: { inputTokens: number; outputTokens: number; totalTokens: number }; +}; +``` + +Also define `ProviderParityReport` to match the closed JSON schema; it must contain no field capable of storing raw content or provider diagnostic prose. + +```ts +export type ProviderParityReport = { + contractVersion: "1.0"; + thresholdVersion: "1.0"; + evidenceLevel: "provider-direct"; + generatedAt: string; + sourceCommit: string; + githubRunId: string; + regionLabel: "ap-southeast-2"; + scenarioId: "synthetic-cited-answer"; + status: "passed" | "failed"; + providerCallCount: 6; + durationBucket: "under-1m" | "under-5m" | "under-15m" | "15m-or-more"; + aggregateTokenUsage: { inputTokens: number; outputTokens: number; totalTokens: number }; + results: ProviderParityResult[]; + parity: Array<{ + evaluatorId: ProviderEvaluatorId; + otelGenaiScore: number; + openInferenceScore: number; + absoluteDelta: number; + maximumDelta: number; + passed: boolean; + }>; +}; +``` + +- [ ] **Step 6: Run the tests and verify they pass** + +Run the package test command. Expected: the new contract tests and all existing API tests PASS. + +- [ ] **Step 7: Commit the contract slice** + +```bash +git add shared/examples/agent-evaluation-telemetry/provider-parity-thresholds.v1.json \ + shared/schemas/agent-evaluation-telemetry/provider-parity-report.schema.json \ + providers/aws/app/api/src/evals/agentCoreEvaluationProviderTypes.ts \ + providers/aws/app/api/tests/agentCoreEvaluationProviderContracts.test.ts +git commit -m "feat: define AgentCore provider parity contracts" +``` + +--- + +### Task 2: Build deterministic AgentCore direct-span requests + +**Files:** +- Create: `providers/aws/app/api/src/evals/agentCoreEvaluationRequestBuilder.ts` +- Create: `providers/aws/app/api/tests/agentCoreEvaluationRequestBuilder.test.ts` + +**Interfaces:** +- Consumes: `TelemetryFixture`, `EvaluationScenario`, `ProviderParityPolicy`, and `ProviderEvaluationRequest`. +- Produces: `buildProviderEvaluationRequests(fixture, scenario, policy): ProviderEvaluationRequest[]` and `deriveProviderIds(scenarioId, convention, originalSpanIds)`. + +- [ ] **Step 1: Write failing happy-path and equivalence tests** + +The test loads only `synthetic-cited-answer` from each existing fixture file, calls the builder, and asserts: + +```ts +assert.equal(otelRequests.length, 3); +assert.equal(openInferenceRequests.length, 3); +assert.deepEqual(otelRequests.map((request) => request.evaluatorId), [ + "Builtin.Correctness", + "Builtin.ToolSelectionAccuracy", + "Builtin.GoalSuccessRate" +]); +assert.deepEqual( + summarizeSemantics(otelRequests), + summarizeSemantics(openInferenceRequests) +); +``` + +For each convention assert that generated trace IDs match `/^[0-9a-f]{32}$/`, span IDs match `/^[0-9a-f]{16}$/`, parent IDs refer to generated spans, every span retains `session.id`, and the recognized `scope.name` prefix is preserved. + +- [ ] **Step 2: Write failing target and reference tests** + +Assert exact evaluator behavior: + +```ts +assert.deepEqual(correctness.evaluationTarget, { traceIds: [generatedTraceId] }); +assert.deepEqual(toolSelection.evaluationTarget, { spanIds: [generatedToolSpanId] }); +assert.equal(goalSuccess.evaluationTarget, undefined); +assert.deepEqual(reference.expectedResponse, { text: scenario.expectedResponse }); +assert.equal(toolSelection.evaluationReferenceInputs, undefined); +assert.deepEqual(reference.assertions, [ + { text: "The final answer cites the approved synthetic source." } +]); +``` + +Verify the Correctness reference context includes session and trace, the +ToolSelectionAccuracy result context is derived independently from the targeted +tool span as session/trace/span, and the GoalSuccessRate reference context +includes only session. + +- [ ] **Step 3: Write failing rejection tests** + +Mutate one input at a time and require stable error codes: + +```ts +assertBuilderError(nonSyntheticScenario, "provider_fixture_not_synthetic"); +assertBuilderError(wrongScenario, "provider_scenario_not_allowed"); +assertBuilderError(unknownScopeFixture, "provider_scope_not_allowed"); +assertBuilderError(missingAgentSpan, "provider_required_span_missing"); +assertBuilderError(secondSessionFixture, "provider_session_count_invalid"); +assertBuilderError(unknownAttributeFixture, "provider_attribute_not_allowed"); +``` + +Also reject a policy with a fourth evaluator, a changed threshold, a call cap other than six, an empty expected response, or more than one tool trajectory entry. + +- [ ] **Step 4: Run the focused test and verify it fails** + +Run: + +```bash +corepack pnpm@11.7.0 --dir providers/aws/app/api test +``` + +Expected: FAIL because `buildProviderEvaluationRequests` does not exist. + +- [ ] **Step 5: Implement deterministic identifier derivation** + +Use Node `createHash("sha256")`. Derive one 32-character trace ID and unique 16-character span IDs from `scenarioId`, convention, original span ID, and fixed domain separators: + +```ts +function hexId(width: 16 | 32, ...parts: string[]): string { + return createHash("sha256").update(parts.join("\u0000")).digest("hex").slice(0, width); +} +``` + +Never use random IDs in Stage A; tests and parity comparison must be reproducible. + +- [ ] **Step 6: Implement the reviewed attribute allowlists and span mapping** + +Use convention-specific sets containing only the keys already present in the cited-answer fixtures. Reject extras before mapping. Convert each wrapper to an AgentCore-compatible OpenTelemetry JSON document with these fields: + +```ts +{ + traceId: generatedTraceId, + spanId: generatedSpanId, + parentSpanId: generatedParentSpanId, + name: spanRole, + kind: 1, + startTimeUnixNano: source.startTimeUnixNano, + endTimeUnixNano: (BigInt(source.startTimeUnixNano) + 1n).toString(), + attributes: { ...source.attributes, "session.id": preservedSessionId }, + scope: { name: source.scopeName, version: "1.0.0" }, + resource: { + attributes: { + "service.name": "cloudai-provider-parity-synthetic", + "cloudai.data.scope": "synthetic-only" + } + }, + status: { code: 1 } +} +``` + +Preserve the fixture's single reviewed `session.id` as `preservedSessionId`. +For the OpenTelemetry invoke-agent span, strictly extract the one reviewed user +text and one reviewed assistant text from `gen_ai.input.messages` and +`gen_ai.output.messages`, then emit them as provider-readable +`gen_ai.task.input` and `gen_ai.task.output` strings. Reject malformed, +ambiguous, non-text, empty, or unreviewed wrappers. Preserve the documented +inference and tool fields. OpenInference keeps its documented agent, inference, +and tool fields. Do not expose the returned documents from the CLI or artifact +layer. + +Map the local assertion code through one fixed source constant: + +```ts +const PROVIDER_ASSERTION_TEXT = { + "citation-present": "The final answer cites the approved synthetic source." +} as const; +``` + +Do not send the internal assertion token `citation-present` as natural-language ground truth. + +- [ ] **Step 7: Implement the fixed three-request matrix** + +Build requests in this exact order: Correctness, ToolSelectionAccuracy, +GoalSuccessRate. Correctness targets the generated trace and supplies only one +trace-scoped `expectedResponse`. ToolSelectionAccuracy targets the generated +tool span and omits `evaluationReferenceInputs`. GoalSuccessRate has no explicit +target and supplies only one session-scoped `assertions` reference. + +- [ ] **Step 8: Run tests and commit** + +Expected: builder tests and the full API suite PASS. + +```bash +git add providers/aws/app/api/src/evals/agentCoreEvaluationRequestBuilder.ts \ + providers/aws/app/api/tests/agentCoreEvaluationRequestBuilder.test.ts +git commit -m "feat: build deterministic AgentCore evaluation requests" +``` + +--- + +### Task 3: Add the fail-closed provider result and parity gate + +**Files:** +- Create: `providers/aws/app/api/src/evals/agentCoreEvaluationProviderGate.ts` +- Create: `providers/aws/app/api/tests/agentCoreEvaluationProviderGate.test.ts` + +**Interfaces:** +- Consumes: convention-tagged `ProviderEvaluationRequest` and `ProviderEvaluationResponse` pairs plus `ProviderParityPolicy`. +- Produces: `sanitizeProviderResult(...)`, `buildProviderParityReport(...)`, and `assertProviderParityGate(report)` using the stable error contract from Task 1. + +- [ ] **Step 1: Write the failing successful-result test** + +Build six fake responses with one result each: three scores for `otel-genai` and three for `openinference`. Use scores `0.90/0.85/0.80` and `0.88/0.82/0.78`; include deliberately sensitive `explanation`, `errorMessage`, and `evaluatorArn` fields in the fake object. Assert: + +```ts +assert.equal(report.evidenceLevel, "provider-direct"); +assert.equal(report.providerCallCount, 6); +assert.equal(report.results.length, 6); +assert.equal(report.parity.length, 3); +assert.equal(report.status, "passed"); +assert.doesNotThrow(() => assertProviderParityGate(report)); +const serialized = JSON.stringify(report); +assert.equal(serialized.includes("provider explanation"), false); +assert.equal(serialized.includes("arn:aws"), false); +assert.equal(serialized.includes("Which controls"), false); +``` + +- [ ] **Step 2: Write the failing malformed and partial-result table test** + +For each mutation below, assert the stable error code and confirm the provider diagnostic text is absent from the thrown message: + +| Mutation | Required code | +| --- | --- | +| `evaluationResults` missing or empty | `provider_result_missing` | +| two results for a single-target request | `provider_result_duplicate` | +| mismatched evaluator ID | `provider_evaluator_unexpected` | +| any `errorCode` | `provider_result_failed` | +| non-empty `ignoredReferenceInputFields` | `provider_reference_input_ignored` | +| missing/NaN/infinite/out-of-range value | `provider_score_invalid` | +| missing or wrong span context | `provider_context_mismatch` | +| negative/non-integer token count | `provider_token_usage_invalid` | +| label empty, longer than 80, or non-printable | `provider_label_invalid` | + +- [ ] **Step 3: Write the failing threshold and parity tests** + +Require `provider_score_below_threshold` for `0.69`, `provider_parity_delta_exceeded` when paired scores differ by more than `0.20`, `provider_result_coverage_invalid` for missing/duplicate convention-evaluator pairs, and `provider_call_count_invalid` unless the report contains exactly six results. + +- [ ] **Step 4: Run the focused test and verify it fails** + +Run the package test command. Expected: FAIL because the provider gate module does not exist. + +- [ ] **Step 5: Implement stable error use and result sanitization** + +Throw only the Task 1 error class, whose message is always its bounded code: + +```ts +export class ProviderParityError extends Error { + constructor(public readonly code: ProviderParityErrorCode) { + super(code); + this.name = "ProviderParityError"; + } +} +``` + +`sanitizeProviderResult` derives the expected result context from the actual +session spans and target rather than requiring a reference input. It validates +session/trace/span for the tool-level result even though the tool evaluator has +no reference input, enforces each evaluator's exact supported reference shape, +and returns only the `ProviderParityResult` fields. It must never retain the raw +response, `errorMessage`, `explanation`, evaluator ARN/name, or request. + +- [ ] **Step 6: Implement report aggregation and parity comparison** + +Index results by `${convention}:${evaluatorId}`, require exactly the six known keys, and calculate each delta as: + +```ts +const absoluteDelta = Math.abs(otel.score - openInference.score); +``` + +Aggregate provider-reported token counts with safe-integer checks. Set `status` to `passed` only when all six thresholds and all three parity comparisons pass. + +- [ ] **Step 7: Implement the public gate assertion** + +`assertProviderParityGate` revalidates the complete report rather than trusting the `status` field. It rejects unknown/duplicate evaluator keys, bad numerics, inconsistent `passed` booleans, incorrect call count, failed result rows, or parity rows that do not reproduce the score differences. + +- [ ] **Step 8: Run tests and commit** + +Expected: all provider gate and existing API tests PASS. + +```bash +git add providers/aws/app/api/src/evals/agentCoreEvaluationProviderGate.ts \ + providers/aws/app/api/tests/agentCoreEvaluationProviderGate.test.ts +git commit -m "feat: gate AgentCore provider parity evidence" +``` + +--- + +### Task 4: Add an injected AgentCore client and protected runner + +**Files:** +- Create: `providers/aws/app/api/src/clients/agentCoreEvaluationClient.ts` +- Create: `providers/aws/app/api/src/scripts/runAgentCoreEvaluationProviderParity.ts` +- Create: `providers/aws/app/api/tests/runAgentCoreEvaluationProviderParity.test.ts` +- Modify: `providers/aws/app/api/package.json` +- Modify: `providers/aws/app/api/pnpm-lock.yaml` + +**Interfaces:** +- Consumes: the request builder and provider gate from Tasks 2–3. +- Produces: `createAwsAgentCoreEvaluateClient(region)`, `validateProviderParityEnvironment(environment)`, `runProviderParityEvaluation(options, clientFactory)`, and CLI modes `validate` and `direct-spans`. + +Use these exact runner interfaces: + +```ts +export type ProviderParityMode = "validate" | "direct-spans"; +export type ProviderParityRunOptions = { + mode: ProviderParityMode; + scenarioPath: string; + fixturePaths: [string, string]; + policyPath: string; + outputPath?: string; + generatedAt: string; + githubRunId: string; + environment: NodeJS.ProcessEnv; +}; +export type ProviderParityRunResult = + | { mode: "validate"; status: "passed"; requestCount: 6 } + | { mode: "direct-spans"; status: "passed"; report: ProviderParityReport }; +export type AgentCoreEvaluateClientFactory = () => AgentCoreEvaluateClient; + +export async function runProviderParityEvaluation( + options: ProviderParityRunOptions, + clientFactory: AgentCoreEvaluateClientFactory +): Promise; +``` + +- [ ] **Step 1: Write failing preflight-before-client tests** + +Inject a factory that increments a counter and assert it is never called for each invalid direct environment: + +```ts +const valid = { + PROVIDER_PARITY_MODE: "direct-spans", + CONFIRMATION: "I_UNDERSTAND_AGENTCORE_EVALUATION_PROVIDER_PARITY", + AGENTCORE_EVALUATION_READY: "true", + AGENTCORE_EVALUATION_MAX_CALLS: "6", + AWS_REGION: "ap-southeast-2", + GITHUB_REF: "refs/heads/main", + GITHUB_SHA: "a".repeat(40) +}; +``` + +Test missing/wrong confirmation, readiness not exactly `true`, max calls not exactly `6`, wrong region, non-main ref, malformed SHA, missing output path, missing fixture/policy files, and an input policy whose maximum is not six. + +- [ ] **Step 2: Write the failing validate-mode fake-client test** + +Run `validate` with a fake client returning deterministic passing scores. Assert six serial calls, successful request/result contract validation, no output artifact, no environment confirmation requirement, and no construction of the real AWS adapter. The fake path must not label itself `provider-direct` because no provider was called. + +- [ ] **Step 3: Write the failing direct-mode exact-six-call test** + +Use a fake injected client with an `activeCalls` counter. Assert maximum concurrency equals one, calls occur in convention then evaluator order, the seventh call is impossible, and an exception on call four yields only `provider_request_failed` without the provider message. + +- [ ] **Step 4: Run tests and verify they fail** + +Run the package test command. Expected: FAIL because the runner and client modules do not exist. + +- [ ] **Step 5: Pin the reviewed AWS SDK** + +From `providers/aws/app/api`, run: + +```bash +corepack pnpm@11.7.0 add --save-exact @aws-sdk/client-bedrock-agentcore@3.1121.0 +``` + +Confirm `package.json` contains an exact `"3.1121.0"` value, not a caret or range, and `pnpm-lock.yaml` resolves the same package version. + +- [ ] **Step 6: Implement the narrow AWS adapter** + +Use only the AgentCore data-plane client: + +```ts +import { + BedrockAgentCoreClient, + EvaluateCommand, + type EvaluateCommandInput +} from "@aws-sdk/client-bedrock-agentcore"; + +export function createAwsAgentCoreEvaluateClient(region: string): AgentCoreEvaluateClient { + const client = new BedrockAgentCoreClient({ region, maxAttempts: 2 }); + return { + async evaluate(request) { + const input: EvaluateCommandInput = request; + return client.send(new EvaluateCommand(input)); + } + }; +} +``` + +Do not import the control-plane client, Runtime invocation client, CloudWatch client, or credential providers. + +- [ ] **Step 7: Implement environment validation and file loading** + +`validateProviderParityEnvironment` returns a typed configuration only after checking all Global Constraints. `validate` accepts `GITHUB_SHA=local` and does not check protected-cloud fields; `direct-spans` requires every exact value. Load the two existing fixture arrays, select the fixed scenario from each, load the one fixed scenario definition and provider policy, and reject all extra provider cases. + +- [ ] **Step 8: Implement serial execution and sanitized logging** + +Build three requests per convention, assert total length equals six before creating the client, then execute with a `for...of` loop. Log only bounded fields: + +```ts +console.info(`agentcore-provider-parity-start mode=${mode} call_budget=6`); +console.info( + `agentcore-provider-evaluation-complete convention=${convention} evaluator=${evaluatorId}` +); +console.info("agentcore-provider-parity-passed evidence_level=provider-direct calls=6"); +``` + +On failure, use the bounded code and set a nonzero exit code: + +```ts +console.error(`agentcore-provider-parity-failed code=${code}`); +process.exitCode = 1; +``` + +`code` must be a member of `ProviderParityErrorCode`. Never stringify a request, raw response, exception, or environment object. + +- [ ] **Step 9: Add the package script and CLI parser** + +Add: + +```json +"agentcore-eval:provider-parity": "pnpm run build && node dist/src/scripts/runAgentCoreEvaluationProviderParity.js" +``` + +Accept exactly `--mode validate` or `--mode direct-spans --output /private/tmp/provider-parity.json`. `validate` installs a deterministic fake client, runs the complete request/result gate, emits only a bounded success line, and writes no artifact. `direct-spans` requires the output path and invokes `createAwsAgentCoreEvaluateClient` only after preflight and request-count validation. + +- [ ] **Step 10: Run tests and commit** + +Run the full package suite and one cloud-free CLI validation: + +```bash +corepack pnpm@11.7.0 --dir providers/aws/app/api agentcore-eval:provider-parity -- \ + --mode validate +``` + +Expected: tests PASS; CLI prints the bounded local-validation success line; it writes no artifact and never claims `provider-direct` evidence. + +```bash +git add providers/aws/app/api/package.json providers/aws/app/api/pnpm-lock.yaml \ + providers/aws/app/api/src/clients/agentCoreEvaluationClient.ts \ + providers/aws/app/api/src/scripts/runAgentCoreEvaluationProviderParity.ts \ + providers/aws/app/api/tests/runAgentCoreEvaluationProviderParity.test.ts +git commit -m "feat: add protected AgentCore evaluation runner" +``` + +--- + +### Task 5: Add the manual protected GitHub Actions lane + +**Files:** +- Create: `.github/workflows/agentcore-evaluation-provider-parity.yml` +- Create: `providers/aws/app/api/tests/agentCoreEvaluationProviderWorkflow.test.ts` + +**Interfaces:** +- Consumes: `pnpm agentcore-eval:provider-parity` from Task 4. +- Produces: a cloud-free `validate` job and a protected `direct-spans` job; only the direct job can publish the seven-day metadata-only artifact. + +- [ ] **Step 1: Write the failing workflow boundary test** + +Read the workflow as text and split the two jobs. Require: + +```ts +assert.match(source, /workflow_dispatch:/); +assert.doesNotMatch(source, /\n\s+(pull_request|push|schedule|workflow_call):/); +assert.match(validateJob, /--mode validate/); +assert.doesNotMatch(validateJob, /environment:|id-token:\s*write|configure-aws-credentials/); +assert.match(directJob, /environment:\s*aws-sandbox/); +assert.match(directJob, /id-token:\s*write/); +assert.match(directJob, /I_UNDERSTAND_AGENTCORE_EVALUATION_PROVIDER_PARITY/); +assert.match(directJob, /AGENTCORE_EVALUATION_MAX_CALLS/); +assert.match(directJob, /retention-days:\s*7/); +assert.doesNotMatch(source, /strategy:|matrix:|continue-on-error:\s*true/); +``` + +Also read `.github/workflows/ci.yml` and prove its required API job still contains no AWS credential action or provider-parity `direct-spans` invocation. + +- [ ] **Step 2: Run tests and verify they fail** + +Expected: FAIL because the workflow does not exist. + +- [ ] **Step 3: Add the manual workflow shell** + +Use only `workflow_dispatch` with inputs: + +```yaml +mode: + type: choice + options: [validate, direct-spans] + default: validate +confirmation: + type: string + required: false +``` + +Set `permissions: contents: read` globally and `concurrency.group: cloudai-agentcore-evaluation-provider-parity` with `cancel-in-progress: false`. + +- [ ] **Step 4: Add the cloud-free validate job** + +The job runs only for `mode == 'validate'`, has no environment and no OIDC permission, checks out code, installs Node 22/pnpm 11.7.0 with frozen lockfile, runs the complete API tests, and runs the provider CLI in `validate` mode. It uploads no `provider-direct` artifact because no provider call occurred. + +- [ ] **Step 5: Add the protected direct-spans job** + +The job runs only for `mode == 'direct-spans'`, declares: + +```yaml +environment: aws-sandbox +permissions: + contents: read + id-token: write +env: + CONFIRMATION: ${{ inputs.confirmation }} + AGENTCORE_EVALUATION_READY: ${{ vars.AGENTCORE_EVALUATION_READY }} + AGENTCORE_EVALUATION_MAX_CALLS: ${{ vars.AGENTCORE_EVALUATION_MAX_CALLS }} + AWS_AGENTCORE_EVALUATION_ROLE_TO_ASSUME: ${{ vars.AWS_AGENTCORE_EVALUATION_ROLE_TO_ASSUME || secrets.AWS_AGENTCORE_EVALUATION_ROLE_TO_ASSUME }} + AWS_REGION: ${{ vars.AWS_REGION || 'ap-southeast-2' }} + PROVIDER_PARITY_MODE: direct-spans +``` + +Before `configure-aws-credentials`, a shell step checks the exact confirmation, readiness, six-call cap, region, role presence, `GITHUB_REF == refs/heads/main`, and 40-hex `GITHUB_SHA`. Then install locked dependencies, run tests, configure the dedicated role with `mask-aws-account-id: true`, execute one serial CLI process, and upload the sanitized artifact with `if-no-files-found: error` and seven-day retention. + +- [ ] **Step 6: Run tests and commit** + +Expected: workflow boundary test and full API suite PASS. + +```bash +git add .github/workflows/agentcore-evaluation-provider-parity.yml \ + providers/aws/app/api/tests/agentCoreEvaluationProviderWorkflow.test.ts +git commit -m "ci: protect AgentCore provider parity evaluation" +``` + +--- + +### Task 6: Add a dedicated evaluate-only GitHub OIDC role source + +**Files:** +- Modify: `providers/aws/infra/bootstrap/github-oidc-terraform-backend.yaml` +- Modify: `providers/aws/infra/bootstrap/test_github_oidc_terraform_backend.rb` +- Modify: `.github/workflows/update-aws-bootstrap.yml` +- Modify: `providers/aws/infra/bootstrap/README.md` + +**Interfaces:** +- Consumes: the existing account-level GitHub OIDC provider and `aws-sandbox` environment trust pattern. +- Produces: CloudFormation output `AgentCoreEvaluationRoleArn`, intended only for protected setting `AWS_AGENTCORE_EVALUATION_ROLE_TO_ASSUME`. + +- [ ] **Step 1: Write failing least-privilege role tests** + +Extend the Ruby Minitest file with a helper that extracts only `GitHubActionsAgentCoreEvaluationRole`. Require: + +```ruby +def test_includes_dedicated_agentcore_evaluation_role + assert_includes template, "GitHubActionsAgentCoreEvaluationRole:" + assert_includes agentcore_evaluation_role, + 'RoleName: !Sub "${GitHubRepo}-${GitHubEnvironment}-agentcore-evaluation"' + assert_includes agentcore_evaluation_role, + 'token.actions.githubusercontent.com:sub: !Sub "repo:${GitHubOrg}/${GitHubRepo}:environment:${GitHubEnvironment}"' + assert_includes agentcore_evaluation_role, "bedrock-agentcore:Evaluate" + assert_includes template, "AgentCoreEvaluationRoleArn:" +end + +def test_agentcore_evaluation_role_cannot_mutate_or_invoke_other_services + %w[ + bedrock-agentcore:CreateEvaluator bedrock-agentcore:UpdateEvaluator + bedrock-agentcore:DeleteEvaluator bedrock-agentcore:InvokeAgentRuntime + logs:StartQuery cloudwatch:GetMetricData iam:PassRole s3:GetObject + ].each { |action| refute_includes agentcore_evaluation_role, action } +end +``` + +Add the exact helper: + +```ruby +def agentcore_evaluation_role + template + .split("GitHubActionsAgentCoreEvaluationRole:", 2) + .fetch(1, "") + .split(/\n\s{2}[A-Z][A-Za-z0-9]+:/, 2) + .first + .to_s +end +``` + +Extend the bootstrap-role test to require its resource allowlist to contain only the exact new role name pattern, not `role/*`. + +- [ ] **Step 2: Run the focused Ruby test and verify it fails** + +Run: + +```bash +ruby providers/aws/infra/bootstrap/test_github_oidc_terraform_backend.rb +``` + +Expected: FAIL because the dedicated role and output are absent. + +- [ ] **Step 3: Add the dedicated CloudFormation role** + +Add a named role following the current dedicated-role pattern: + +```yaml +GitHubActionsAgentCoreEvaluationRole: + Type: AWS::IAM::Role + DependsOn: GitHubActionsBootstrapRole + Properties: + RoleName: !Sub "${GitHubRepo}-${GitHubEnvironment}-agentcore-evaluation" + AssumeRolePolicyDocument: + Version: "2012-10-17" + Statement: + - Effect: Allow + Principal: + Federated: !Ref ExistingGitHubOidcProviderArn + Action: sts:AssumeRoleWithWebIdentity + Condition: + StringEquals: + token.actions.githubusercontent.com:aud: sts.amazonaws.com + StringLike: + token.actions.githubusercontent.com:sub: !Sub "repo:${GitHubOrg}/${GitHubRepo}:environment:${GitHubEnvironment}" + Policies: + - PolicyName: AgentCoreEvaluationDataPlanePolicy + PolicyDocument: + Version: "2012-10-17" + Statement: + - Sid: EvaluateOnlySyntheticProviderParity + Effect: Allow + Action: + - bedrock-agentcore:Evaluate + Resource: "*" + Tags: + - { Key: Project, Value: cloudai-platform } + - { Key: Environment, Value: aws-sandbox } + - { Key: ManagedBy, Value: cloudformation } + - { Key: DataScope, Value: synthetic-only } +``` + +`Resource: "*"` is isolated because the current data-plane action does not offer a proven evaluator ARN scope in this project. The workflow's fixed evaluator IDs, six-call cap, environment approval, and main-only execution are the compensating controls. + +- [ ] **Step 4: Bound bootstrap role management and add the output** + +Add only: + +```yaml +- !Sub "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/${GitHubRepo}-${GitHubEnvironment}-agentcore-evaluation" +``` + +to `ManageOnlyTerraformBootstrapAndBudgetRoles`, then add: + +```yaml +AgentCoreEvaluationRoleArn: + Description: Store as AWS_AGENTCORE_EVALUATION_ROLE_TO_ASSUME in the aws-sandbox GitHub environment. + Value: !GetAtt GitHubActionsAgentCoreEvaluationRole.Arn +``` + +Do not attach this policy to `GitHubActionsTerraformRole`, Runtime roles, or the bootstrap role. + +- [ ] **Step 5: Add the post-apply masked environment handoff** + +Extend `update-aws-bootstrap.yml` with a success-only step that queries `AgentCoreEvaluationRoleArn`, fails if absent, masks it using `::add-mask::`, and writes only the setting name and masked handoff instruction to `$GITHUB_STEP_SUMMARY`. Do not print the ARN to normal logs. + +- [ ] **Step 6: Document the role boundary** + +Update the bootstrap README role list and flow. State explicitly that source merge creates no role; the sequence remains validate → change-set plan → review → separately confirmed apply → copy output into the protected environment. + +- [ ] **Step 7: Validate and commit** + +Run: + +```bash +ruby providers/aws/infra/bootstrap/test_github_oidc_terraform_backend.rb +pipx run cfn-lint providers/aws/infra/bootstrap/github-oidc-terraform-backend.yaml +``` + +Expected: Ruby tests and cfn-lint PASS without AWS credentials. + +```bash +git add providers/aws/infra/bootstrap/github-oidc-terraform-backend.yaml \ + providers/aws/infra/bootstrap/test_github_oidc_terraform_backend.rb \ + .github/workflows/update-aws-bootstrap.yml \ + providers/aws/infra/bootstrap/README.md +git commit -m "feat: add evaluate-only AgentCore OIDC role" +``` + +--- + +### Task 7: Update architecture, runbook, status, and evidence language + +**Files:** +- Modify: `docs/solutions/agent-evaluation-telemetry-runbook.md` +- Modify: `docs/architecture/agentcore-governed-rag-poc.md` +- Modify: `docs/solutions/p8i-agentcore-rag-key-process-record.md` +- Modify: `docs/practices/current-status.md` +- Modify: `providers/aws/app/api/README.md` +- Modify: `providers/aws/app/api/tests/agentEvaluationTelemetryDocumentation.test.ts` + +**Interfaces:** +- Consumes: implemented source paths, workflow, role output, and evidence schema from Tasks 1–6. +- Produces: operator instructions that distinguish `local-contract`, `provider-direct`, and unimplemented `provider-runtime` evidence. + +- [ ] **Step 1: Write failing documentation assertions** + +Extend `agentEvaluationTelemetryDocumentation.test.ts` to require these exact concepts across the documentation set: + +```ts +for (const required of [ + "provider-parity-v1", + "provider-direct", + "I_UNDERSTAND_AGENTCORE_EVALUATION_PROVIDER_PARITY", + "AWS_AGENTCORE_EVALUATION_ROLE_TO_ASSUME", + "AGENTCORE_EVALUATION_MAX_CALLS=6", + "Builtin.ToolSelectionAccuracy", + "Stage B", + "Runtime-to-CloudWatch" +]) assert.match(documentation, new RegExp(escapeRegExp(required), "i")); +``` + +Update the current-status assertion so the row contains `source implemented` and `provider validation pending`, while rejecting `provider validated`, `runtime validated`, and `production evaluation`. + +- [ ] **Step 2: Run tests and verify they fail** + +Expected: documentation test FAILS because the new workflow and evidence levels are not yet documented. + +- [ ] **Step 3: Update the operator runbook** + +Add: + +1. a three-lane diagram for local CI, Stage A direct spans, and Stage B Runtime-to-CloudWatch; +2. environment setting names and safe purposes, with no values or ARNs; +3. `validate` instructions that explicitly make no AWS call; +4. a direct-run preflight checklist listing environment approval, main revision, fixed scenario, fixed evaluator matrix, exact call budget, and exact confirmation; +5. sanitized artifact allowed/forbidden fields; +6. failure response: inspect bounded codes, do not paste raw provider output into issues or notes; +7. a statement that no role apply or first AWS run is authorized by this PR. + +- [ ] **Step 4: Update architecture and process record** + +In the AgentCore architecture document, draw Stage A beside—not inside—the Runtime/CloudWatch path: + +```text +local fixtures -> direct sessionSpans -> AgentCore Evaluate -> provider-direct evidence + +Gateway -> Runtime -> ADOT -> CloudWatch -> AgentCore Evaluate -> provider-runtime evidence + Stage B: not implemented by this change +``` + +Record why direct spans precede Runtime ingestion, why managed scores supplement deterministic controls, and why the score cannot authorize tool execution or deployment. + +- [ ] **Step 5: Update current status and API README** + +Use the status wording: + +```text +Stage A source implemented; provider validation pending. The protected lane is +manual, synthetic-only, evaluate-only, and bounded to six calls. Stage B +Runtime-to-CloudWatch evaluation is not implemented. +``` + +The API README documents the local `validate` command only. It links to the protected runbook rather than presenting a laptop-local AWS command. + +- [ ] **Step 6: Run docs tests and commit** + +Run the full API suite so the Markdown relative-link checker also executes. Expected: PASS. + +```bash +git add docs/solutions/agent-evaluation-telemetry-runbook.md \ + docs/architecture/agentcore-governed-rag-poc.md \ + docs/solutions/p8i-agentcore-rag-key-process-record.md \ + docs/practices/current-status.md providers/aws/app/api/README.md \ + providers/aws/app/api/tests/agentEvaluationTelemetryDocumentation.test.ts +git commit -m "docs: record AgentCore provider parity boundary" +``` + +--- + +### Task 8: Run full source verification and prepare the reviewable PR + +**Files:** +- Verify all files changed in Tasks 1–7. +- Do not create account-specific evidence or modify protected environment settings. + +**Interfaces:** +- Consumes: the complete Stage A source implementation. +- Produces: a clean branch, full local verification record, pushed feature branch, and PR that requests source review only. + +- [ ] **Step 1: Reinstall locked dependencies and run the complete API suite** + +```bash +corepack pnpm@11.7.0 --dir providers/aws/app/api install --frozen-lockfile +corepack pnpm@11.7.0 --dir providers/aws/app/api test +``` + +Expected: every test passes, including all six local scenarios under both conventions and all new provider-parity negative cases. + +- [ ] **Step 2: Run both cloud-free evaluation gates** + +```bash +temp_dir="$(mktemp -d)" +corepack pnpm@11.7.0 --dir providers/aws/app/api agent-eval:gate -- \ + --output "$temp_dir/local-contract.json" +PROVIDER_PARITY_MODE=validate \ + corepack pnpm@11.7.0 --dir providers/aws/app/api agentcore-eval:provider-parity -- \ + --mode validate +``` + +Expected: both pass without AWS credentials. Inspect with `jq 'keys'` and `rg` for forbidden fixture phrases; do not copy temporary artifacts into the repository. + +- [ ] **Step 3: Run infrastructure and static checks** + +```bash +ruby providers/aws/infra/bootstrap/test_github_oidc_terraform_backend.rb +pipx run cfn-lint providers/aws/infra/bootstrap/github-oidc-terraform-backend.yaml +scripts/validate-argocd-gitops.sh +git diff --check +``` + +Expected: all commands PASS. Confirm `git diff` contains no account IDs, resolved role ARNs, credentials, endpoint values, raw provider output, or `_private` notes. + +- [ ] **Step 4: Review source boundaries manually** + +Confirm: + +- `.github/workflows/ci.yml` has no AgentCore direct call or OIDC grant; +- the new workflow has no PR/push/schedule/workflow-call trigger; +- only its protected direct job has `id-token: write` and `environment: aws-sandbox`; +- the dedicated role grants only `bedrock-agentcore:Evaluate`; +- `validate` cannot construct the real AWS client; +- direct preflight completes before credential configuration/client construction; +- exactly six serial calls are built; +- only the metadata-only report is written; +- all documentation says provider validation pending. + +- [ ] **Step 5: Commit any verification-only corrections** + +If verification required corrections, list them with `git status --short`, stage each printed path explicitly with individual `git add path/to/file` commands, review `git diff --cached`, and commit with `git commit -m "test: harden AgentCore provider parity boundaries"`. + +If no correction was needed, do not create an empty commit. + +- [ ] **Step 6: Push and open the PR** + +```bash +git push -u origin feature/agentcore-evaluation-provider-parity +gh pr create \ + --base main \ + --head feature/agentcore-evaluation-provider-parity \ + --title "feat: add protected AgentCore evaluation provider parity" \ + --body $'## Summary\n\n- adds Stage A source only; no AWS evaluation occurred\n- keeps required PR CI cloud-free and adds a protected manual six-call lane\n- adds an evaluate-only OIDC role source; it has not been applied\n- emits only metadata-safe provider-direct evidence\n\n## Validation\n\n- full API tests\n- local deterministic evaluation gate\n- cloud-free provider-parity validate mode\n- Ruby IAM boundary tests and cfn-lint\n\n## Pending approvals\n\nProvider validation, bootstrap plan/apply, environment handoff, and Stage B Runtime-to-CloudWatch evaluation remain pending. Do not merge without explicit approval.' +``` + +- [ ] **Step 7: Stop at the source-review gate** + +Do not merge, dispatch the protected workflow, create a CloudFormation change set, apply the role, or set GitHub environment values. Report the PR URL and checks, then request the next explicit approval. + +## Post-Merge Approval Sequence + +These are operational gates, not part of source implementation: + +1. Run `update-aws-bootstrap` in `validate` mode. +2. Request approval before creating a non-executing CloudFormation change set. +3. Review the exact IAM-only change set. +4. Request the existing exact bootstrap-apply confirmation and user approval. +5. Add the masked `AgentCoreEvaluationRoleArn` output to the protected `aws-sandbox` environment as `AWS_AGENTCORE_EVALUATION_ROLE_TO_ASSUME`. +6. Set `AGENTCORE_EVALUATION_READY=true` and `AGENTCORE_EVALUATION_MAX_CALLS=6` only after the handoff is reviewed. +7. Run the workflow in `validate` mode. +8. Request fresh confirmation before the first `direct-spans` execution. +9. Inspect and retain only the metadata-only `provider-direct` artifact. +10. Write and approve a separate Stage B Runtime-to-CloudWatch spec only after Stage A provider validation succeeds. diff --git a/docs/superpowers/specs/2026-08-29-agentcore-evaluation-provider-parity-design.md b/docs/superpowers/specs/2026-08-29-agentcore-evaluation-provider-parity-design.md new file mode 100644 index 0000000..2448a07 --- /dev/null +++ b/docs/superpowers/specs/2026-08-29-agentcore-evaluation-provider-parity-design.md @@ -0,0 +1,427 @@ +# AgentCore Evaluation Provider-Parity Design + +## Status + +Approved direction: implement Stage A before Stage B. + +- Stage A proves that equivalent synthetic OpenTelemetry GenAI and + OpenInference traces can be scored by Amazon Bedrock AgentCore on-demand + evaluation through a protected, manually dispatched workflow. +- Stage B later proves Runtime telemetry export, CloudWatch ingestion, session + reconstruction, and managed evaluation as a separate end-to-end path. + +This document authorizes design and source implementation only. It does not +authorize an AWS evaluation call, IAM apply, Runtime image release, CloudWatch +configuration change, or resource deletion. + +## Problem + +The repository now has a required local quality gate that normalizes synthetic +OpenTelemetry GenAI and OpenInference fixtures into one deterministic contract. +That evidence proves local compatibility and fail-closed policy behaviour, but +it does not prove that Amazon Bedrock AgentCore Evaluations accepts the +telemetry, interprets both conventions consistently, or returns usable managed +scores. + +The next step must add provider evidence without weakening the ordinary pull +request path or conflating managed model scores with authorization decisions. + +## Goals + +1. Reuse one canonical synthetic scenario across OpenTelemetry GenAI and + OpenInference. +2. Convert the repository fixtures into the documented AgentCore + `sessionSpans` request shape. +3. Run only a fixed allowlist of built-in evaluators with evaluator-specific + expected-response or assertion references where those evaluators support them. +4. Fail closed on missing, malformed, partial, below-threshold, or materially + divergent results. +5. Retain a metadata-only provider-parity artifact. +6. Keep AWS execution manually dispatched, environment-protected, + confirmation-gated, budget-bounded, and separate from required PR CI. +7. Preserve a clean extension point for the later Runtime-to-CloudWatch path. + +## Non-Goals + +- Online evaluation or continuous production sampling. +- Production traffic or non-synthetic data. +- An `OnDemandEvaluationDatasetRunner` benchmark suite. +- Automatic AWS evaluation on pull requests or pushes. +- Creation of a new AgentCore CLI project or `agentcore/agentcore.json`. +- Custom LLM-as-a-judge or Lambda evaluator creation. +- Runtime, Gateway, Knowledge Base, model, prompt, or tool changes in Stage A. +- Treating an evaluation score as an IAM, admission, approval, or execution + decision. +- Claiming managed trajectory parity from `Builtin.ToolSelectionAccuracy`; + adding a `Builtin.Trajectory*` evaluator requires a new reviewed policy and + call budget. +- Claiming OTLP export, CloudWatch ingestion, or production agent quality from + Stage A. + +## Official Compatibility Basis + +The design follows these current AWS interfaces: + +- `Evaluate` is a synchronous AgentCore data-plane API accepting + `evaluationInput.sessionSpans` plus optional target and reference inputs. +- The AWS SDK for JavaScript exposes `BedrockAgentCoreClient` and + `EvaluateCommand` in `@aws-sdk/client-bedrock-agentcore`. +- Generic evaluation compatibility is selected by instrumentation scopes under + `opentelemetry.instrumentation.*` or `openinference.instrumentation.*`. +- Session reconstruction requires `session.id`; quality evaluation also needs + message content. +- AgentCore reads invoke-agent, inference, and execute-tool span roles while + ignoring unfamiliar contextual spans. +- A provider result may contain partial failures, error details, explanations, + token usage, and at most ten results. The workflow must sanitize that output + before artifact upload. + +References: + +- [AWS announcement: evaluate any agent framework](https://aws.amazon.com/blogs/machine-learning/evaluate-any-agent-framework-with-amazon-bedrock-agentcore-evaluations/) +- [AgentCore Evaluate API](https://docs.aws.amazon.com/bedrock-agentcore/latest/APIReference/API_Evaluate.html) +- [AWS SDK for JavaScript EvaluateCommand](https://docs.aws.amazon.com/AWSJavaScriptSDK/v3/latest/client/bedrock-agentcore/command/EvaluateCommand/) +- [Getting started with on-demand evaluation](https://docs.aws.amazon.com/bedrock-agentcore/latest/devguide/getting-started-on-demand.html) +- [Understanding evaluation input spans](https://docs.aws.amazon.com/bedrock-agentcore/latest/devguide/understanding-input-spans.html) +- [Generic framework span mapping](https://docs.aws.amazon.com/bedrock-agentcore/latest/devguide/supported-frameworks-generic.html) +- [Ground-truth inputs by evaluator](https://docs.aws.amazon.com/bedrock-agentcore/latest/devguide/ground-truth-evaluations.html) + +## Chosen Architecture + +### Why staged delivery + +Three options were considered: + +1. Directly submit constructed spans to `Evaluate`. +2. Instrument and invoke the deployed Runtime, wait for CloudWatch, then + evaluate the ingested session. +3. Start with the dataset runner or online evaluation. + +Option 1 is the first stage because it isolates the managed evaluator contract +from Runtime instrumentation and CloudWatch ingestion. Option 2 follows because +it proves the operational path that Stage A deliberately excludes. Option 3 is +deferred because it adds invocation orchestration, sampling, cost, and runtime +coupling before the two smaller boundaries are proven. + +```text +Required pull-request lane — no AWS + +Canonical scenarios + -> OpenTelemetry GenAI + OpenInference fixtures + -> local normalizer + -> deterministic strict-v1 gate + -> metadata-only local-contract artifact + +Protected Stage A — manually approved AWS call + +Canonical cited-answer scenario + -> provider request builder + -> valid AWS sessionSpans per convention + -> fixed built-in AgentCore evaluators + -> result validation + cross-convention comparison + -> metadata-only provider-direct artifact + +Protected Stage B — later, separately approved + +Gateway invocation + -> instrumented Runtime + -> ADOT force flush + -> CloudWatch unified observability + -> bounded ingestion retry + -> AgentCore evaluation + -> metadata-only provider-runtime artifact +``` + +## Stage A: Direct-Span Managed Parity + +### Scenario scope + +The initial managed run uses only `synthetic-cited-answer`, represented once by +OpenTelemetry GenAI and once by OpenInference. The local required gate continues +to cover all six scenarios under both conventions. + +The smaller provider set is deliberate: + +- it proves generic framework routing; +- it contains one invoke-agent, inference, and execute-tool path; +- it supports the fixed correctness response and goal-success assertion + references, plus a tool-span target for tool-selection scoring; +- it caps cost and makes provider differences easier to diagnose; +- it avoids treating an expensive managed evaluator as a duplicate of every + deterministic local assertion. + +Expansion to the full six-case pack requires a new reviewed threshold profile, +call budget, and explicit execution approval. + +### Fixed evaluator matrix + +Each convention is evaluated with exactly three built-in evaluators: + +| Evaluator | Level | Reference or target | +| --- | --- | --- | +| `Builtin.Correctness` | Trace | Fixed expected response and the generated trace ID. | +| `Builtin.ToolSelectionAccuracy` | Tool call | Generated tool span ID only; no unsupported ground-truth reference. | +| `Builtin.GoalSuccessRate` | Session | One fixed session-scoped assertion. | + +The result is six managed `Evaluate` calls: two conventions multiplied by +three evaluators. Evaluator IDs are code-owned and cannot be supplied through a +workflow input. + +### Threshold profile + +Managed model scores are probabilistic, so they use a separate versioned +profile rather than the deterministic local `strict-v1` profile. + +`provider-parity-v1` requires: + +- `Builtin.Correctness >= 0.70`; +- `Builtin.ToolSelectionAccuracy >= 0.70`; +- `Builtin.GoalSuccessRate >= 0.70`; +- absolute score difference between the two conventions for the same evaluator + `<= 0.20`; +- zero failed, missing, non-finite, duplicated, or unexpected results. + +The workflow must not permit threshold overrides. Changing a threshold requires +a reviewed repository change. A failed managed score blocks that manually +dispatched run but does not change access policy or trigger remediation. + +### Provider request builder + +The canonical fixtures remain provider-neutral and are not sent as-is. A pure +TypeScript builder will: + +1. accept one validated `TelemetryFixture` and its `EvaluationScenario`; +2. require `syntheticOnly: true` and the fixed scenario ID; +3. require exactly one session and the three required span roles; +4. map the generic wrapper to the AWS JSON document shape; +5. generate deterministic valid OpenTelemetry trace and span identifiers from + the scenario and convention using SHA-256-derived lowercase hexadecimal IDs; +6. preserve the supported instrumentation scope and `session.id`, and translate + the reviewed OpenTelemetry invoke-agent prompt/final-response wrappers to + clean `gen_ai.task.input` and `gen_ai.task.output` string attributes; +7. build evaluator-specific target and reference inputs; +8. reject unknown attributes that are not part of the reviewed compatibility + subset; +9. return a typed `EvaluateCommandInput` without writing the request to disk. + +Synthetic prompt, response, tool arguments, and tool result content are sent to +the managed evaluator because those fields are required to score correctness, +tool selection, and goal completion. They are public synthetic data and must +never be copied into the evidence artifact. + +### Provider client boundary + +The package will pin `@aws-sdk/client-bedrock-agentcore` to the reviewed version +used by the implementation. Production code creates a +`BedrockAgentCoreClient` only after all local confirmation, budget, fixture, and +allowlist checks pass. Tests inject a small client interface and never obtain +AWS credentials or call a provider. + +The runner executes calls serially. SDK retries remain bounded; application +code must not add unbounded retry loops. Each call is associated with one +convention and evaluator before the response is sanitized. + +### Fail-closed result handling + +The managed run fails with a bounded code when any of these occurs: + +- exact confirmation is absent; +- source commit is not the protected branch or approved revision; +- readiness or call-budget variables are absent or invalid; +- fixture or scenario is not the fixed synthetic case; +- an instrumentation scope is unrecognized; +- generated request count is not exactly six or exceeds the maximum; +- an evaluator result is absent, duplicated, unexpected, or contains + `errorCode`; +- a score is missing, non-numeric, non-finite, outside zero-to-one, or below its + threshold; +- cross-convention score delta exceeds the parity tolerance; +- the SDK returns access, throttling, quota, validation, or internal-service + failure; +- the sanitized artifact cannot be written. + +Provider exception messages, evaluation explanations, raw request content, and +raw provider responses must not be printed or uploaded. Logs use a stable local +reason code and the evaluator ID only. + +## Protected Workflow + +Create `.github/workflows/agentcore-evaluation-provider-parity.yml` with two +manual modes: + +- `validate`: runs contract tests, request-builder tests, workflow boundary + checks, and a dry-run that uses an injected fake client. It requires no OIDC + token, environment, or AWS access. +- `direct-spans`: uses the `aws-sandbox` GitHub Environment, obtains short-lived + credentials through OIDC, and runs the six fixed managed evaluations. + +`direct-spans` requires all of the following: + +- exact confirmation + `I_UNDERSTAND_AGENTCORE_EVALUATION_PROVIDER_PARITY`; +- `AGENTCORE_EVALUATION_READY=true` in the protected environment; +- `AWS_AGENTCORE_EVALUATION_ROLE_TO_ASSUME` stored as a protected environment + variable or secret; +- `AWS_REGION=ap-southeast-2` unless a separately reviewed regional change is + committed; +- `AGENTCORE_EVALUATION_MAX_CALLS=6` and an implementation hard cap of six; +- one concurrency group with `cancel-in-progress: false`; +- no matrix fan-out and no parallel provider calls; +- a seven-day sanitized artifact retention period. + +The workflow is `workflow_dispatch` only. It is never called from `pull_request`, +`push`, another workflow, or a scheduled trigger. + +## IAM Design + +Stage A uses a dedicated GitHub OIDC evaluation role rather than expanding the +Terraform execution role or Runtime role. Its trust policy is limited to this +repository and the protected `aws-sandbox` environment subject. + +The permission policy allows only the AgentCore on-demand `Evaluate` data-plane +action and the minimum AWS SDK support actions proven necessary during a +reviewed plan. If AWS does not support evaluator resource-level scoping for the +data-plane action, the policy may require `Resource: "*"`; that exception must +be isolated in this dedicated role and compensated by: + +- fixed evaluator IDs in source; +- exact call-count cap; +- environment approval; +- no evaluator creation, update, deletion, online configuration, Runtime + invocation, CloudWatch read, or infrastructure permissions; +- a separately reviewed CloudFormation change set. + +The role and policy are added to the existing bootstrap CloudFormation +template, but no bootstrap apply is authorized by source implementation. A +future apply requires its own plan, change-set review, exact confirmation, and +user approval. + +## Metadata-Only Evidence + +The uploaded `agentcore-evaluation-provider-parity` artifact may contain: + +- contract, threshold, and evidence versions; +- source commit and GitHub run ID; +- region label without account ID; +- scenario ID and telemetry convention; +- evaluator ID, score, label, threshold, and pass/fail status; +- absolute parity delta per evaluator; +- aggregate input, output, and total evaluation token counts; +- sanitized failure reason code; +- timestamps and total duration bucket. + +It must not contain: + +- prompt, response, assertion, expected response, or tool content; +- raw spans, request bodies, or provider responses; +- explanation or `errorMessage` text; +- credentials, tokens, account IDs, ARNs, endpoints, resource names, or + CloudWatch query output; +- repository-local notes or environment-variable values. + +The evidence level is `provider-direct`. It is never labelled `provider-runtime`, +`managed production evaluation`, or `production quality`. + +## Stage B: Runtime and CloudWatch Parity + +Stage B is a separate feature and approval boundary after Stage A has one +successful managed run. + +It will: + +1. add reviewed OpenTelemetry instrumentation to the custom TypeScript Runtime; +2. emit a top-level `invoke_agent` span and only semantically truthful child + spans for the actual Runtime operations; +3. preserve the Gateway invocation `runtimeSessionId` as `session.id`; +4. retain only the synthetic message content required for evaluation; +5. force-flush both trace and log providers before the Runtime handler returns; +6. use unified observability or explicitly query every required log source; +7. verify CloudWatch Transaction Search readiness before invocation; +8. invoke one synthetic Gateway request; +9. poll ingestion with bounded retries rather than assuming a fixed delay; +10. evaluate only after a complete session is present; +11. publish separate `provider-runtime` evidence; +12. retain the current Gateway-only, read-only, citation-or-abstention contract. + +The AWS skill documents about ten seconds as the typical end-to-end put-to-get +delay, while current AWS examples also warn that ingestion may take longer. +Stage B therefore uses a bounded readiness poll with a documented maximum; it +does not encode a single optimistic sleep as proof of ingestion. + +Stage B requires a new design review for Runtime dependencies, image release, +CloudWatch configuration, IAM, log-content handling, cost, rollback, and +teardown. Approval of this spec does not authorize those changes. + +## Testing Strategy + +All implementation follows test-first development. + +### Stage A unit and contract tests + +- both conventions create semantically equivalent evaluator inputs; +- deterministic generated IDs have valid widths and do not collide; +- only the cited-answer synthetic fixture is accepted; +- the OpenTelemetry provider span exposes clean `gen_ai.task.input` and + `gen_ai.task.output` strings while the OpenInference provider span exposes + semantically equal `input.value` and `output.value` strings; +- Correctness includes only `expectedResponse`, ToolSelectionAccuracy omits + reference inputs, and GoalSuccessRate includes only `assertions`; +- evaluator levels select the correct trace, span, or session target; +- unknown scope, missing session, missing span role, or unknown attribute fails; +- six and only six provider calls are built; +- fake-client success produces a sanitized report; +- partial failure, missing result, bad score, low score, duplicate result, and + excess parity delta fail closed; +- output contains none of the forbidden content fields; +- no client call occurs before confirmation and budget validation; +- ordinary CI contains no AWS credentials or managed evaluation call; +- protected workflow has manual trigger, environment, OIDC, exact confirmation, + hard call cap, serial execution, and short artifact retention. + +### Repository validation + +- complete API TypeScript build and test suite; +- standalone local deterministic evaluation gate; +- AgentCore Runtime tests remain green; +- GitOps and documentation-link checks remain green; +- CloudFormation lint and static policy tests after the role source is added; +- secret and scope scans remain green. + +No test claims live provider success until a separately approved +`direct-spans` workflow run completes. + +## Delivery Sequence + +1. Add provider request/result schemas and threshold profile. +2. Add pure request builder and its negative tests. +3. Add managed result sanitizer and parity gate with fake-client tests. +4. Add the protected manual workflow and boundary tests. +5. Add the dedicated OIDC evaluation-role source and static IAM tests. +6. Update runbook, architecture, current status, and key process record. +7. Run local verification and open a pull request. +8. After merge, prepare a bootstrap change-set plan; do not apply without fresh + approval. +9. After an approved role apply and environment handoff, run `validate`. +10. Request fresh confirmation before the first `direct-spans` AWS call. +11. Record only sanitized evidence and then decide whether Stage B should start. + +## Success Criteria + +Source implementation is complete when: + +- local tests prove request equivalence and fail-closed managed result handling; +- the protected workflow cannot run AWS evaluation without every gate; +- the dedicated role source cannot mutate AgentCore, Runtime, Gateway, + CloudWatch, or evaluator configuration; +- documentation accurately labels Stage A as source implemented and runtime + validation pending; +- all repository checks pass and a reviewable pull request is open. + +Stage A provider validation is complete only after a separately approved AWS +run returns six accepted results, every score meets its versioned threshold, +every parity delta is within tolerance, and the uploaded artifact passes the +metadata boundary checks. + +Stage B remains incomplete until a separate design and execution cycle proves +Runtime instrumentation, CloudWatch ingestion, and managed evaluation end to +end. diff --git a/providers/aws/app/api/README.md b/providers/aws/app/api/README.md index ed37f68..24ade19 100644 --- a/providers/aws/app/api/README.md +++ b/providers/aws/app/api/README.md @@ -215,11 +215,21 @@ corepack pnpm@11.7.0 --dir providers/aws/app/api agent-eval:gate -- \ --output /tmp/agent-evaluation-report.json ``` -The required CI path does not call AWS and emits only a metadata-safe local -contract report. OTLP export, CloudWatch ingestion, and AgentCore managed -evaluation belong to a separately approved protected provider-parity lane. -See the [agent evaluation telemetry runbook](../../../../docs/solutions/agent-evaluation-telemetry-runbook.md) -for the evidence and non-claim boundary. +The required CI path does not call AWS and emits only a metadata-safe +`local-contract` report. The only provider-parity command documented for local +use is validation of the reviewed source contract: + +```bash +corepack pnpm@11.7.0 --dir providers/aws/app/api agentcore-eval:provider-parity -- \ + --mode validate +``` + +It uses local deterministic fakes and makes no AWS call. Stage A +`provider-direct` evidence and the future Stage B `provider-runtime` +Runtime-to-CloudWatch path are not provider, runtime, or production validated. +Protected execution is manual and is documented only in the +[agent evaluation telemetry runbook](../../../../docs/solutions/agent-evaluation-telemetry-runbook.md); +this README intentionally provides no laptop-local AWS invocation. ## Chat Response diff --git a/providers/aws/app/api/package.json b/providers/aws/app/api/package.json index 27e1a3a..8c03954 100644 --- a/providers/aws/app/api/package.json +++ b/providers/aws/app/api/package.json @@ -11,6 +11,7 @@ "start": "node dist/src/server.js", "test": "tsc -p tsconfig.json && node --test \"dist/tests/**/*.test.js\"", "agent-eval:gate": "pnpm run build && node dist/src/scripts/runAgentEvaluationTelemetryGate.js", + "agentcore-eval:provider-parity": "pnpm run build && node dist/src/scripts/runAgentCoreEvaluationProviderParity.js", "bedrock:smoke": "pnpm run build && node dist/src/scripts/bedrockAdapterSmoke.js" }, "devDependencies": { @@ -18,6 +19,7 @@ "typescript": "^5.8.0" }, "dependencies": { + "@aws-sdk/client-bedrock-agentcore": "3.1120.0", "@aws-sdk/client-bedrock-runtime": "^3.1090.0", "prom-client": "15.1.3" } diff --git a/providers/aws/app/api/pnpm-lock.yaml b/providers/aws/app/api/pnpm-lock.yaml index 277eaae..7837662 100644 --- a/providers/aws/app/api/pnpm-lock.yaml +++ b/providers/aws/app/api/pnpm-lock.yaml @@ -8,6 +8,9 @@ importers: .: dependencies: + '@aws-sdk/client-bedrock-agentcore': + specifier: 3.1120.0 + version: 3.1120.0 '@aws-sdk/client-bedrock-runtime': specifier: ^3.1090.0 version: 3.1090.0 @@ -24,6 +27,10 @@ importers: packages: + '@aws-sdk/client-bedrock-agentcore@3.1120.0': + resolution: {integrity: sha512-DR6l5UXq/ekg6u6yamQXRhYYdSazY8yfKhDl7HL+ZLHvGpezaYObPE/wxuuZIJjojpb3XvPkeQ7U7uAvUInBmg==} + engines: {node: '>=20.0.0'} + '@aws-sdk/client-bedrock-runtime@3.1090.0': resolution: {integrity: sha512-uFo62YTKik92NxlnExaTU2rH2WQCseW8XvZnBiSab+YWU9pl1URhVsnxY8whUe4EVOuxdAAlgCuChTJo6gZIfQ==} engines: {node: '>=20.0.0'} @@ -32,14 +39,30 @@ packages: resolution: {integrity: sha512-7ur3kCKuvPLqlsZ2XlvnNBVQ7KkpSu6Y6dOTwSPHLrFpTEfZM8isLBJc4cgv96WB7GifeVM436mpycwxBd2vEA==} engines: {node: '>=20.0.0'} + '@aws-sdk/core@3.977.9': + resolution: {integrity: sha512-reqPFEQrZxDZpeGj4PFMepBeR5LGYHRqq/L0motTzgFkCRBA4rFdaVXDSLYyGHhxVz7sT2PDnPN9CluGSfgyJA==} + engines: {node: '>=20.0.0'} + '@aws-sdk/credential-provider-env@3.972.59': resolution: {integrity: sha512-Ny5e4Mfh3QPmiAc0AiUe+cbTXDlxkU3Rc+EpWOfyWeWEy6yp7Fa1KmfNeCc+1a8by9zQ9gtohmiQUkMPScF3ng==} engines: {node: '>=20.0.0'} + '@aws-sdk/credential-provider-env@3.972.70': + resolution: {integrity: sha512-H404B7dJl2mCrBqahDEYsanB0xhdDp6tXnXcTUnXmmpy2Q3J0Ho0bUajZ2jr/RdwzCyS59Gi8xXIFwPLGBl6Uw==} + engines: {node: '>=20.0.0'} + '@aws-sdk/credential-provider-http@3.972.61': resolution: {integrity: sha512-8jAjgStl5Ytq4+HF3X/9f+EmRinaRbGRRtQGktlPfBRVx73H+R1y48vIeXerQtYGFaUqkEp3fT6jP854rVO2yQ==} engines: {node: '>=20.0.0'} + '@aws-sdk/credential-provider-http@3.972.72': + resolution: {integrity: sha512-X98zYOrVOeuosCX+6ktf29FC2N2GHPLia7qv6mzPzTc+RPAuHWCDS++Z6JK7eGYqb/v6uaW7bAXaOvDBfol+0w==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/credential-provider-ini@3.973.15': + resolution: {integrity: sha512-Rykg6s5ceBuynMOGWgoowO4N+27JfnqXAnVaSunZl0hOO1XodSrxGNz6sCEbnmS0lAfQZDKyb3fbr46gSuv6Sg==} + engines: {node: '>=20.0.0'} + '@aws-sdk/credential-provider-ini@3.973.4': resolution: {integrity: sha512-e6ZvVsj90aRALf1kHP+J4iqC1496ZpVgqI/+u0LJ5HL7q7ATauGy4gdDvRCP13L1pN/fMiZLah162PGIYkbUVQ==} engines: {node: '>=20.0.0'} @@ -48,14 +71,30 @@ packages: resolution: {integrity: sha512-g2fsqm87r/nKthLZ0VkkDBElkGg0PvSa8d97HQ6EilMbJTZ6hxa8FxkSZyJfgPfFdZn0TTmkOffQmTSUcAHIng==} engines: {node: '>=20.0.0'} + '@aws-sdk/credential-provider-login@3.972.77': + resolution: {integrity: sha512-Jb59xfEISoN5mmbnA+HYqdtrSX3CgCtJoof+V5D8/TgUI56W63GEEd5Y58WijU3Ou6+WEgaLD1feVzaRXV5IDQ==} + engines: {node: '>=20.0.0'} + '@aws-sdk/credential-provider-node@3.972.70': resolution: {integrity: sha512-3xzvkGdykBunxqh8WudmUpSyLWvIhfI6aBQo1b5rb3mDO5mNLadK+0hiI0qBQBMVynJbfLO+Ajy9dztMwy9O8w==} engines: {node: '>=20.0.0'} + '@aws-sdk/credential-provider-node@3.972.81': + resolution: {integrity: sha512-Rml+WitoFvXmv6JZ18U/xGdGDGGvB/mOin0ya0lTnTrdC0Z1lrVxTYh7iNklZBcvcRMrs4DoEf6xy1KWyrLQQw==} + engines: {node: '>=20.0.0'} + '@aws-sdk/credential-provider-process@3.972.59': resolution: {integrity: sha512-DlZF2/MhLlatDdlrIy3CUCpfdbLrKx+3SMjVo+WyHnPpwzkc/M3vwAHw4OVJf7DMvO+4vfRqSCMc/E9I1auN0g==} engines: {node: '>=20.0.0'} + '@aws-sdk/credential-provider-process@3.972.70': + resolution: {integrity: sha512-2ry03fGRJr4sV3jI+ocjj5JqALnFD6ymM5KiNCDZMvq8bX2GSbE0vji4aM43TVCl2nXqqLRZaUxdq/KeWRAY4Q==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/credential-provider-sso@3.973.14': + resolution: {integrity: sha512-jkhg/8ocAAoc0RFyLMhCw+/zZh7gystQgd4F4hznNa8P4Cc501PQmxd+jGLiMHodPJ+7Zv/3znM62gZojyasmA==} + engines: {node: '>=20.0.0'} + '@aws-sdk/credential-provider-sso@3.973.3': resolution: {integrity: sha512-hmdDHoy2G5Es2e8IgelNMYUuSQI6uCIAKZMJ2u2PdKDhxvbk1uWD/g4+R7R5c/tJfKEB1+KjjWiaoCr/S+ZTiQ==} engines: {node: '>=20.0.0'} @@ -64,6 +103,10 @@ packages: resolution: {integrity: sha512-gHQb/Kt0chjk/JQDa/GJDqmAvEuVn8n7z10wK2h0LFM9TUDRkohgOO4aEF+s2sBLM0br7Cl5W6P7phgjrrJvLQ==} engines: {node: '>=20.0.0'} + '@aws-sdk/credential-provider-web-identity@3.972.76': + resolution: {integrity: sha512-d3AGyVu759PGr35mEB2s22xxlNEA5rpdxtSPJthfPFJvoQ8dt357iVPECqWfUxXp1toJAvKmbtcIYVGigaGsCA==} + engines: {node: '>=20.0.0'} + '@aws-sdk/eventstream-handler-node@3.972.29': resolution: {integrity: sha512-t3tKQRTVXsI2QNPE3CaNjHl0wRO9Xi3acZkAyti2RQsiFmZ9Gi0kArX2ighlRJ1BtDVuul413gThAgzyTfgmWA==} engines: {node: '>=20.0.0'} @@ -80,10 +123,18 @@ packages: resolution: {integrity: sha512-dVZOroI/r3/ENvqNGgjMPul+jjlz9GddfVusgTXlVjfZj5isibOxecLkGQbRPp8XOuX+RAfjXLFgPkD1JS5xrw==} engines: {node: '>=20.0.0'} + '@aws-sdk/nested-clients@3.997.44': + resolution: {integrity: sha512-NhEgryjlBF9w38ZXqGymQV28IhkYa1mKhlbYnqIis57AYwWGVYfUPgg/qC2rLRqOUfblxx++irvju10kVTa8Vw==} + engines: {node: '>=20.0.0'} + '@aws-sdk/signature-v4-multi-region@3.996.41': resolution: {integrity: sha512-QMUytg+FQMGouc8gHS00KoYih3+N6cqmVI/pQGOIo7Nr7OpQaiXjSYOuL+vsPZ1tymY4LAQ8MYcHJmws5LRxng==} engines: {node: '>=20.0.0'} + '@aws-sdk/signature-v4-multi-region@3.996.46': + resolution: {integrity: sha512-L+2xZTye/2T96f3lwCws0Zw6GG2JHZW9e8FpVgGBeeExSKyeoZ6CWRpBml/7DNiK/O26jrgPM9F+Ay8VkgzUWQ==} + engines: {node: '>=20.0.0'} + '@aws-sdk/token-providers@3.1088.0': resolution: {integrity: sha512-4ObatWt2qpJg5FBk4LOOKrTQYzaqeewAtdO3r9ZO8lH9YqLtpTzLyIdy0mJ+nVdfYOnqISkKNfmzP22bNDhwyw==} engines: {node: '>=20.0.0'} @@ -92,14 +143,26 @@ packages: resolution: {integrity: sha512-uwPWr8zRBL1YLiWpIWoEOVj0bgGJ0M4Gfi151lEJroQq/7Lc/phrJVIEnCTEHHa1BIse525BfunmOZLMBfD9OQ==} engines: {node: '>=20.0.0'} + '@aws-sdk/token-providers@3.1116.0': + resolution: {integrity: sha512-ygIivKqh8aHzNkucOCXHyIBgBpLPfrSI0mCqXF+vLBsPTUKqj0VSqAY0GFPe7lQl4HntjOcQ+KSyS7oUV2C54Q==} + engines: {node: '>=20.0.0'} + '@aws-sdk/types@3.974.2': resolution: {integrity: sha512-3W6IUtSxFbH6X7Wb7DzGCV5QiFQsd0g8bOfntpmDxQlzBoKWUMBu/JPQR0DwkE+Hpnxd6db1tXbOwdeHddG6cA==} engines: {node: '>=20.0.0'} + '@aws-sdk/types@3.974.5': + resolution: {integrity: sha512-LkwLL2BLbC6wNNm4JaH9mbEqBMdOZCct6VAYqhdN4U1xrWM+fUJQEfbHwQgDypapOWTRtlk25akb5afM0P8CIQ==} + engines: {node: '>=20.0.0'} + '@aws-sdk/xml-builder@3.972.36': resolution: {integrity: sha512-RdGmS1GLrtaTOLE1ElSluMldNrpk9Emq6uYs8SS8iHlu5xTAmM9rRkM91o48+rIRryBtyO9t+uLYCoMG6jVMVA==} engines: {node: '>=20.0.0'} + '@aws-sdk/xml-builder@3.972.40': + resolution: {integrity: sha512-wlFmCIGUlwF4zx/kncw+bmxTQh1HeSJq4mYV/V5cZUSJadDP3kXvGW8Rn21cimj/7y9ju+47oYWXi97vF7czaA==} + engines: {node: '>=20.0.0'} + '@aws/lambda-invoke-store@0.3.0': resolution: {integrity: sha512-sl4Bm6yiMNYrZKkqqDFWN0UfnWhlS8ivKxrYl+6t0gCLrqr8y3B2IqZZbFRkfaVVp7C/baApyh71P+LeE1A2sQ==} engines: {node: '>=18.0.0'} @@ -112,14 +175,30 @@ packages: resolution: {integrity: sha512-i0dk2t5B+CwV/dcJdUHILYkOQF5lof8f44dFCfDWToGCxjT9YQ+CgHqTAvJxzc3+zqQwm2QtVoJ5IqiNar/CnQ==} engines: {node: '>=18.0.0'} + '@smithy/core@3.33.3': + resolution: {integrity: sha512-CsOeKq/9kA3y6VJHt+/+VTCtBaxJ4OTFpgrjIUhPpDIKxBci1k2bJaQASF2h/ELWrulGp+t97DZ0mevfAD8idg==} + engines: {node: '>=18.0.0'} + '@smithy/credential-provider-imds@4.4.10': resolution: {integrity: sha512-MJenAe4OKRZUo1LdYYFDCsSHxaHvInIU/z52GsheO9vl1/VSySVCr0zkyKD6TFiGkSUaWGxvKZ/70OvgUZR5HQ==} engines: {node: '>=18.0.0'} + '@smithy/credential-provider-imds@4.5.2': + resolution: {integrity: sha512-A9uSdn72ozbRUSit0eib0TW7nXuNPlaeM0zcGkJ+nE6tFcSDbnmtwoxbTCFBukVQcszDAyvsd7+rTduPTXpygg==} + engines: {node: '>=18.0.0'} + '@smithy/fetch-http-handler@5.6.7': resolution: {integrity: sha512-3zpg8yqqyXzoK2TsRDdkqVOj2RDBFfLXwCczOZ5c7TWB4eiaebfSCsbMjDPYB3PJ9ihV62QaeadZ+wLadZtNGA==} engines: {node: '>=18.0.0'} + '@smithy/fetch-http-handler@5.7.2': + resolution: {integrity: sha512-nZyWTmSpJEXl6VtWVMBJve/7x12DZu6sIX1z1a+ZMaHlQQRs9Zpu6NbTe/gmxYXVRpkjxyDYpZ5gx2IM6f/Wkw==} + engines: {node: '>=18.0.0'} + + '@smithy/node-http-handler@4.11.3': + resolution: {integrity: sha512-2jY1tSpERfPfWqyBV2pH+iGFaghVsIJszJNsT7hxtQYhVJpWDyc0LqOWI+nXOxOAHaEfZ4PXXtp1wW1TGpHhkA==} + engines: {node: '>=18.0.0'} + '@smithy/node-http-handler@4.9.7': resolution: {integrity: sha512-wCU8HCLjAtAVqxxe0j2xff9LcEPw3yjBbg5IdQDIYFnxnPxbxcSLc7rgex7kqm9L/WYOnJEgaWQlfDkZleozMA==} engines: {node: '>=18.0.0'} @@ -128,10 +207,18 @@ packages: resolution: {integrity: sha512-efP6DN3UTFrzIsGO42/xcabv8jU7+9nwEdphFUH7yL0k010ERyAWaO41KFQIDLcFZLZ8xzIQr4wplFxNzslSGQ==} engines: {node: '>=18.0.0'} + '@smithy/signature-v4@5.7.3': + resolution: {integrity: sha512-7ImGm+FkHRLcBaRttIAMZ6bzJZWb2cJGoYjq46F2UjycujWzrL9GEN9h4w7eQyXJYnltrUhxbbieBAIRrdqpow==} + engines: {node: '>=18.0.0'} + '@smithy/types@4.16.1': resolution: {integrity: sha512-0JFs3V2y2M9tKW5na/qxe69Zv+uxLMO7QBbhxF/FHu/Gp2NFZAAL9tWl9PU02xxo07pb3G9FTyjNc6D5uZrJIg==} engines: {node: '>=18.0.0'} + '@smithy/types@4.17.2': + resolution: {integrity: sha512-FOKpVZob9MPTn2znRzGrnsMHv7BOsKVw3XiP/cOyYLDVZ9qKp4nifIiSCuUU/fIj5Vu0UOAxCFr+qRAtG0NUkA==} + engines: {node: '>=18.0.0'} + '@types/node@24.13.3': resolution: {integrity: sha512-Dh8vAsV36ig5wa9OX4pXvMc9D3Veibfw2wix0CUwYODLD8nkj9UsLjASr49nPg+2eKzxhBV+v7L8pXvT4e639Q==} @@ -161,6 +248,17 @@ packages: snapshots: + '@aws-sdk/client-bedrock-agentcore@3.1120.0': + dependencies: + '@aws-sdk/core': 3.977.9 + '@aws-sdk/credential-provider-node': 3.972.81 + '@aws-sdk/types': 3.974.5 + '@smithy/core': 3.33.3 + '@smithy/fetch-http-handler': 5.7.2 + '@smithy/node-http-handler': 4.11.3 + '@smithy/types': 4.17.2 + tslib: 2.8.1 + '@aws-sdk/client-bedrock-runtime@3.1090.0': dependencies: '@aws-sdk/core': 3.975.3 @@ -187,27 +285,72 @@ snapshots: bowser: 2.14.1 tslib: 2.8.1 + '@aws-sdk/core@3.977.9': + dependencies: + '@aws-sdk/types': 3.974.5 + '@aws-sdk/xml-builder': 3.972.40 + '@aws/lambda-invoke-store': 0.3.0 + '@smithy/core': 3.33.3 + '@smithy/signature-v4': 5.7.3 + '@smithy/types': 4.17.2 + bowser: 2.14.1 + tslib: 2.8.1 + '@aws-sdk/credential-provider-env@3.972.59': dependencies: - '@aws-sdk/core': 3.975.3 - '@aws-sdk/types': 3.974.2 - '@smithy/core': 3.29.5 - '@smithy/types': 4.16.1 + '@aws-sdk/core': 3.977.9 + '@aws-sdk/types': 3.974.5 + '@smithy/core': 3.33.3 + '@smithy/types': 4.17.2 + tslib: 2.8.1 + + '@aws-sdk/credential-provider-env@3.972.70': + dependencies: + '@aws-sdk/core': 3.977.9 + '@aws-sdk/types': 3.974.5 + '@smithy/core': 3.33.3 + '@smithy/types': 4.17.2 tslib: 2.8.1 '@aws-sdk/credential-provider-http@3.972.61': dependencies: - '@aws-sdk/core': 3.975.3 - '@aws-sdk/types': 3.974.2 - '@smithy/core': 3.29.5 - '@smithy/fetch-http-handler': 5.6.7 - '@smithy/node-http-handler': 4.9.7 - '@smithy/types': 4.16.1 + '@aws-sdk/core': 3.977.9 + '@aws-sdk/types': 3.974.5 + '@smithy/core': 3.33.3 + '@smithy/fetch-http-handler': 5.7.2 + '@smithy/node-http-handler': 4.11.3 + '@smithy/types': 4.17.2 + tslib: 2.8.1 + + '@aws-sdk/credential-provider-http@3.972.72': + dependencies: + '@aws-sdk/core': 3.977.9 + '@aws-sdk/types': 3.974.5 + '@smithy/core': 3.33.3 + '@smithy/fetch-http-handler': 5.7.2 + '@smithy/node-http-handler': 4.11.3 + '@smithy/types': 4.17.2 + tslib: 2.8.1 + + '@aws-sdk/credential-provider-ini@3.973.15': + dependencies: + '@aws-sdk/core': 3.977.9 + '@aws-sdk/credential-provider-env': 3.972.70 + '@aws-sdk/credential-provider-http': 3.972.72 + '@aws-sdk/credential-provider-login': 3.972.77 + '@aws-sdk/credential-provider-process': 3.972.70 + '@aws-sdk/credential-provider-sso': 3.973.14 + '@aws-sdk/credential-provider-web-identity': 3.972.76 + '@aws-sdk/nested-clients': 3.997.44 + '@aws-sdk/types': 3.974.5 + '@smithy/core': 3.33.3 + '@smithy/credential-provider-imds': 4.5.2 + '@smithy/types': 4.17.2 tslib: 2.8.1 '@aws-sdk/credential-provider-ini@3.973.4': dependencies: - '@aws-sdk/core': 3.975.3 + '@aws-sdk/core': 3.977.9 '@aws-sdk/credential-provider-env': 3.972.59 '@aws-sdk/credential-provider-http': 3.972.61 '@aws-sdk/credential-provider-login': 3.972.66 @@ -215,19 +358,28 @@ snapshots: '@aws-sdk/credential-provider-sso': 3.973.3 '@aws-sdk/credential-provider-web-identity': 3.972.65 '@aws-sdk/nested-clients': 3.997.33 - '@aws-sdk/types': 3.974.2 - '@smithy/core': 3.29.5 + '@aws-sdk/types': 3.974.5 + '@smithy/core': 3.33.3 '@smithy/credential-provider-imds': 4.4.10 - '@smithy/types': 4.16.1 + '@smithy/types': 4.17.2 tslib: 2.8.1 '@aws-sdk/credential-provider-login@3.972.66': dependencies: - '@aws-sdk/core': 3.975.3 - '@aws-sdk/nested-clients': 3.997.33 - '@aws-sdk/types': 3.974.2 - '@smithy/core': 3.29.5 - '@smithy/types': 4.16.1 + '@aws-sdk/core': 3.977.9 + '@aws-sdk/nested-clients': 3.997.44 + '@aws-sdk/types': 3.974.5 + '@smithy/core': 3.33.3 + '@smithy/types': 4.17.2 + tslib: 2.8.1 + + '@aws-sdk/credential-provider-login@3.972.77': + dependencies: + '@aws-sdk/core': 3.977.9 + '@aws-sdk/nested-clients': 3.997.44 + '@aws-sdk/types': 3.974.5 + '@smithy/core': 3.33.3 + '@smithy/types': 4.17.2 tslib: 2.8.1 '@aws-sdk/credential-provider-node@3.972.70': @@ -244,31 +396,72 @@ snapshots: '@smithy/types': 4.16.1 tslib: 2.8.1 + '@aws-sdk/credential-provider-node@3.972.81': + dependencies: + '@aws-sdk/credential-provider-env': 3.972.70 + '@aws-sdk/credential-provider-http': 3.972.72 + '@aws-sdk/credential-provider-ini': 3.973.15 + '@aws-sdk/credential-provider-process': 3.972.70 + '@aws-sdk/credential-provider-sso': 3.973.14 + '@aws-sdk/credential-provider-web-identity': 3.972.76 + '@aws-sdk/types': 3.974.5 + '@smithy/core': 3.33.3 + '@smithy/credential-provider-imds': 4.5.2 + '@smithy/types': 4.17.2 + tslib: 2.8.1 + '@aws-sdk/credential-provider-process@3.972.59': dependencies: - '@aws-sdk/core': 3.975.3 - '@aws-sdk/types': 3.974.2 - '@smithy/core': 3.29.5 - '@smithy/types': 4.16.1 + '@aws-sdk/core': 3.977.9 + '@aws-sdk/types': 3.974.5 + '@smithy/core': 3.33.3 + '@smithy/types': 4.17.2 + tslib: 2.8.1 + + '@aws-sdk/credential-provider-process@3.972.70': + dependencies: + '@aws-sdk/core': 3.977.9 + '@aws-sdk/types': 3.974.5 + '@smithy/core': 3.33.3 + '@smithy/types': 4.17.2 + tslib: 2.8.1 + + '@aws-sdk/credential-provider-sso@3.973.14': + dependencies: + '@aws-sdk/core': 3.977.9 + '@aws-sdk/nested-clients': 3.997.44 + '@aws-sdk/token-providers': 3.1116.0 + '@aws-sdk/types': 3.974.5 + '@smithy/core': 3.33.3 + '@smithy/types': 4.17.2 tslib: 2.8.1 '@aws-sdk/credential-provider-sso@3.973.3': dependencies: - '@aws-sdk/core': 3.975.3 + '@aws-sdk/core': 3.977.9 '@aws-sdk/nested-clients': 3.997.33 '@aws-sdk/token-providers': 3.1088.0 - '@aws-sdk/types': 3.974.2 - '@smithy/core': 3.29.5 - '@smithy/types': 4.16.1 + '@aws-sdk/types': 3.974.5 + '@smithy/core': 3.33.3 + '@smithy/types': 4.17.2 tslib: 2.8.1 '@aws-sdk/credential-provider-web-identity@3.972.65': dependencies: - '@aws-sdk/core': 3.975.3 + '@aws-sdk/core': 3.977.9 '@aws-sdk/nested-clients': 3.997.33 - '@aws-sdk/types': 3.974.2 - '@smithy/core': 3.29.5 - '@smithy/types': 4.16.1 + '@aws-sdk/types': 3.974.5 + '@smithy/core': 3.33.3 + '@smithy/types': 4.17.2 + tslib: 2.8.1 + + '@aws-sdk/credential-provider-web-identity@3.972.76': + dependencies: + '@aws-sdk/core': 3.977.9 + '@aws-sdk/nested-clients': 3.997.44 + '@aws-sdk/types': 3.974.5 + '@smithy/core': 3.33.3 + '@smithy/types': 4.17.2 tslib: 2.8.1 '@aws-sdk/eventstream-handler-node@3.972.29': @@ -297,29 +490,47 @@ snapshots: '@aws-sdk/nested-clients@3.997.33': dependencies: - '@aws-sdk/core': 3.975.3 + '@aws-sdk/core': 3.977.9 '@aws-sdk/signature-v4-multi-region': 3.996.41 - '@aws-sdk/types': 3.974.2 - '@smithy/core': 3.29.5 - '@smithy/fetch-http-handler': 5.6.7 - '@smithy/node-http-handler': 4.9.7 - '@smithy/types': 4.16.1 + '@aws-sdk/types': 3.974.5 + '@smithy/core': 3.33.3 + '@smithy/fetch-http-handler': 5.7.2 + '@smithy/node-http-handler': 4.11.3 + '@smithy/types': 4.17.2 + tslib: 2.8.1 + + '@aws-sdk/nested-clients@3.997.44': + dependencies: + '@aws-sdk/core': 3.977.9 + '@aws-sdk/signature-v4-multi-region': 3.996.46 + '@aws-sdk/types': 3.974.5 + '@smithy/core': 3.33.3 + '@smithy/fetch-http-handler': 5.7.2 + '@smithy/node-http-handler': 4.11.3 + '@smithy/types': 4.17.2 tslib: 2.8.1 '@aws-sdk/signature-v4-multi-region@3.996.41': dependencies: - '@aws-sdk/types': 3.974.2 - '@smithy/signature-v4': 5.6.6 - '@smithy/types': 4.16.1 + '@aws-sdk/types': 3.974.5 + '@smithy/signature-v4': 5.7.3 + '@smithy/types': 4.17.2 + tslib: 2.8.1 + + '@aws-sdk/signature-v4-multi-region@3.996.46': + dependencies: + '@aws-sdk/types': 3.974.5 + '@smithy/signature-v4': 5.7.3 + '@smithy/types': 4.17.2 tslib: 2.8.1 '@aws-sdk/token-providers@3.1088.0': dependencies: - '@aws-sdk/core': 3.975.3 - '@aws-sdk/nested-clients': 3.997.33 - '@aws-sdk/types': 3.974.2 - '@smithy/core': 3.29.5 - '@smithy/types': 4.16.1 + '@aws-sdk/core': 3.977.9 + '@aws-sdk/nested-clients': 3.997.44 + '@aws-sdk/types': 3.974.5 + '@smithy/core': 3.33.3 + '@smithy/types': 4.17.2 tslib: 2.8.1 '@aws-sdk/token-providers@3.1090.0': @@ -331,14 +542,33 @@ snapshots: '@smithy/types': 4.16.1 tslib: 2.8.1 + '@aws-sdk/token-providers@3.1116.0': + dependencies: + '@aws-sdk/core': 3.977.9 + '@aws-sdk/nested-clients': 3.997.44 + '@aws-sdk/types': 3.974.5 + '@smithy/core': 3.33.3 + '@smithy/types': 4.17.2 + tslib: 2.8.1 + '@aws-sdk/types@3.974.2': dependencies: '@smithy/types': 4.16.1 tslib: 2.8.1 + '@aws-sdk/types@3.974.5': + dependencies: + '@smithy/types': 4.17.2 + tslib: 2.8.1 + '@aws-sdk/xml-builder@3.972.36': dependencies: - '@smithy/types': 4.16.1 + '@smithy/types': 4.17.2 + tslib: 2.8.1 + + '@aws-sdk/xml-builder@3.972.40': + dependencies: + '@smithy/types': 4.17.2 tslib: 2.8.1 '@aws/lambda-invoke-store@0.3.0': {} @@ -350,10 +580,21 @@ snapshots: '@smithy/types': 4.16.1 tslib: 2.8.1 + '@smithy/core@3.33.3': + dependencies: + '@smithy/types': 4.17.2 + tslib: 2.8.1 + '@smithy/credential-provider-imds@4.4.10': dependencies: - '@smithy/core': 3.29.5 - '@smithy/types': 4.16.1 + '@smithy/core': 3.33.3 + '@smithy/types': 4.17.2 + tslib: 2.8.1 + + '@smithy/credential-provider-imds@4.5.2': + dependencies: + '@smithy/core': 3.33.3 + '@smithy/types': 4.17.2 tslib: 2.8.1 '@smithy/fetch-http-handler@5.6.7': @@ -362,6 +603,18 @@ snapshots: '@smithy/types': 4.16.1 tslib: 2.8.1 + '@smithy/fetch-http-handler@5.7.2': + dependencies: + '@smithy/core': 3.33.3 + '@smithy/types': 4.17.2 + tslib: 2.8.1 + + '@smithy/node-http-handler@4.11.3': + dependencies: + '@smithy/core': 3.33.3 + '@smithy/types': 4.17.2 + tslib: 2.8.1 + '@smithy/node-http-handler@4.9.7': dependencies: '@smithy/core': 3.29.5 @@ -370,14 +623,24 @@ snapshots: '@smithy/signature-v4@5.6.6': dependencies: - '@smithy/core': 3.29.5 - '@smithy/types': 4.16.1 + '@smithy/core': 3.33.3 + '@smithy/types': 4.17.2 + tslib: 2.8.1 + + '@smithy/signature-v4@5.7.3': + dependencies: + '@smithy/core': 3.33.3 + '@smithy/types': 4.17.2 tslib: 2.8.1 '@smithy/types@4.16.1': dependencies: tslib: 2.8.1 + '@smithy/types@4.17.2': + dependencies: + tslib: 2.8.1 + '@types/node@24.13.3': dependencies: undici-types: 7.18.2 diff --git a/providers/aws/app/api/src/clients/agentCoreEvaluationClient.ts b/providers/aws/app/api/src/clients/agentCoreEvaluationClient.ts new file mode 100644 index 0000000..ab4e770 --- /dev/null +++ b/providers/aws/app/api/src/clients/agentCoreEvaluationClient.ts @@ -0,0 +1,17 @@ +import { + BedrockAgentCoreClient, + EvaluateCommand, + type EvaluateCommandInput +} from "@aws-sdk/client-bedrock-agentcore"; + +import type { AgentCoreEvaluateClient } from "../evals/agentCoreEvaluationProviderTypes.js"; + +export function createAwsAgentCoreEvaluateClient(region: string): AgentCoreEvaluateClient { + const client = new BedrockAgentCoreClient({ region, maxAttempts: 2 }); + return { + async evaluate(request) { + const input: EvaluateCommandInput = request; + return client.send(new EvaluateCommand(input)); + } + }; +} diff --git a/providers/aws/app/api/src/evals/agentCoreEvaluationProviderGate.ts b/providers/aws/app/api/src/evals/agentCoreEvaluationProviderGate.ts new file mode 100644 index 0000000..651273f --- /dev/null +++ b/providers/aws/app/api/src/evals/agentCoreEvaluationProviderGate.ts @@ -0,0 +1,461 @@ +import { + ProviderParityError, + type ProviderEvaluationLevel, + type ProviderEvaluationRequest, + type ProviderEvaluationResponse, + type ProviderEvaluatorId, + type ProviderParityPolicy, + type ProviderParityReport, + type ProviderParityResult +} from "./agentCoreEvaluationProviderTypes.js"; +import type { EvaluationConvention } from "./agentEvaluationTelemetryTypes.js"; + +const EVALUATORS: readonly ProviderEvaluatorId[] = [ + "Builtin.Correctness", + "Builtin.ToolSelectionAccuracy", + "Builtin.GoalSuccessRate" +]; + +const CONVENTIONS: readonly EvaluationConvention[] = ["otel-genai", "openinference"]; + +const EXPECTED_KEYS = new Set( + CONVENTIONS.flatMap((convention) => EVALUATORS.map((evaluatorId) => `${convention}:${evaluatorId}`)) +); + +const RESULT_KEYS = [ + "convention", "evaluatorId", "level", "score", "label", "threshold", "passed", "reasonCode", "tokenUsage" +]; + +const PARITY_KEYS = [ + "evaluatorId", "otelGenaiScore", "openInferenceScore", "absoluteDelta", "maximumDelta", "passed" +]; + +const REPORT_KEYS = [ + "contractVersion", "thresholdVersion", "evidenceLevel", "generatedAt", "sourceCommit", "githubRunId", + "regionLabel", "scenarioId", "status", "providerCallCount", "durationBucket", "aggregateTokenUsage", + "results", "parity" +]; + +const DURATION_BUCKETS = new Set(["under-1m", "under-5m", "under-15m", "15m-or-more"]); +const SAFE_PROVIDER_LABEL = "provider_result"; + +export type ProviderEvaluationPair = { + convention: EvaluationConvention; + request: ProviderEvaluationRequest; + response: ProviderEvaluationResponse; +}; + +export type ProviderParityReportInput = { + pairs: ProviderEvaluationPair[]; + policy: ProviderParityPolicy; + generatedAt: string; + sourceCommit: string; + githubRunId: string; + durationBucket: ProviderParityReport["durationBucket"]; +}; + +export function sanitizeProviderResult( + pair: ProviderEvaluationPair, + policy: ProviderParityPolicy +): ProviderParityResult { + validatePolicy(policy); + if (!isRecord(pair)) throw new ProviderParityError("provider_result_coverage_invalid"); + if (!isConvention(pair.convention)) throw new ProviderParityError("provider_result_coverage_invalid"); + if (!isRecord(pair.request)) throw new ProviderParityError("provider_context_mismatch"); + + const evaluatorId = pair.request.evaluatorId; + if (!isEvaluatorId(evaluatorId)) throw new ProviderParityError("provider_evaluator_unexpected"); + const expectedContext = deriveProviderResultContext(pair.request); + if (!hasExpectedRequestShape(pair.request, evaluatorId, expectedContext)) { + throw new ProviderParityError("provider_context_mismatch"); + } + const level = levelFor(evaluatorId); + if (!isRecord(pair.response)) throw new ProviderParityError("provider_result_missing"); + const evaluationResults = pair.response.evaluationResults; + if (!Array.isArray(evaluationResults) || evaluationResults.length === 0) { + throw new ProviderParityError("provider_result_missing"); + } + if (evaluationResults.length !== 1) throw new ProviderParityError("provider_result_duplicate"); + + const raw = evaluationResults[0]; + if (!isRecord(raw)) throw new ProviderParityError("provider_result_missing"); + if (raw.evaluatorId !== evaluatorId) throw new ProviderParityError("provider_evaluator_unexpected"); + if (raw.errorCode !== undefined) throw new ProviderParityError("provider_result_failed"); + if (raw.ignoredReferenceInputFields !== undefined && + (!Array.isArray(raw.ignoredReferenceInputFields) || raw.ignoredReferenceInputFields.length !== 0)) { + throw new ProviderParityError("provider_reference_input_ignored"); + } + if (!isScore(raw.value)) throw new ProviderParityError("provider_score_invalid"); + if (!sameContext(raw.context, expectedContext)) throw new ProviderParityError("provider_context_mismatch"); + if (!isProviderLabel(raw.label)) throw new ProviderParityError("provider_label_invalid"); + if (!isTokenUsage(raw.tokenUsage)) throw new ProviderParityError("provider_token_usage_invalid"); + + const threshold = policy.evaluatorThresholds[evaluatorId]; + if (raw.value < threshold) throw new ProviderParityError("provider_score_below_threshold"); + + return { + convention: pair.convention, + evaluatorId, + level, + score: raw.value, + label: SAFE_PROVIDER_LABEL, + threshold, + passed: true, + reasonCode: "passed", + tokenUsage: { + inputTokens: raw.tokenUsage.inputTokens, + outputTokens: raw.tokenUsage.outputTokens, + totalTokens: raw.tokenUsage.totalTokens + } + }; +} + +export function buildProviderParityReport(input: ProviderParityReportInput): ProviderParityReport { + if (!isRecord(input)) throw new ProviderParityError("provider_policy_invalid"); + validatePolicy(input.policy); + if (!Array.isArray(input.pairs) || input.pairs.length !== 6) { + throw new ProviderParityError("provider_call_count_invalid"); + } + + const results = input.pairs.map((pair) => sanitizeProviderResult(pair, input.policy)); + assertResultCoverage(results); + const aggregateTokenUsage = aggregateTokens(results); + const parity = EVALUATORS.map((evaluatorId) => { + const otel = resultFor(results, "otel-genai", evaluatorId); + const openInference = resultFor(results, "openinference", evaluatorId); + const absoluteDelta = Math.abs(otel.score - openInference.score); + const maximumDelta = input.policy.maximumParityDelta; + return { + evaluatorId, + otelGenaiScore: otel.score, + openInferenceScore: openInference.score, + absoluteDelta, + maximumDelta, + passed: absoluteDelta <= maximumDelta + }; + }); + + const report: ProviderParityReport = { + contractVersion: "1.0", + thresholdVersion: "1.0", + evidenceLevel: "provider-direct", + generatedAt: input.generatedAt, + sourceCommit: input.sourceCommit, + githubRunId: input.githubRunId, + regionLabel: "ap-southeast-2", + scenarioId: "synthetic-cited-answer", + status: parity.every((row) => row.passed) ? "passed" : "failed", + providerCallCount: 6, + durationBucket: input.durationBucket, + aggregateTokenUsage, + results, + parity + }; + assertReportEnvelope(report); + return report; +} + +export function assertProviderParityGate(report: ProviderParityReport): void { + assertReportEnvelope(report); + if (report.providerCallCount !== 6 || !Array.isArray(report.results) || report.results.length !== 6) { + throw new ProviderParityError("provider_call_count_invalid"); + } + + assertResultCoverage(report.results); + for (const result of report.results) assertReportResult(result); + const aggregate = aggregateTokens(report.results); + if (!sameTokenUsage(report.aggregateTokenUsage, aggregate)) { + throw new ProviderParityError("provider_token_usage_invalid"); + } + + if (!Array.isArray(report.parity) || report.parity.length !== 3) { + throw new ProviderParityError("provider_result_coverage_invalid"); + } + const parityIds = new Set(); + for (const row of report.parity) { + if (!hasOnlyKeys(row, PARITY_KEYS) || !isEvaluatorId(row.evaluatorId) || parityIds.has(row.evaluatorId)) { + throw new ProviderParityError("provider_result_coverage_invalid"); + } + parityIds.add(row.evaluatorId); + const otel = resultFor(report.results, "otel-genai", row.evaluatorId); + const openInference = resultFor(report.results, "openinference", row.evaluatorId); + const expectedDelta = Math.abs(otel.score - openInference.score); + if (!isScore(row.otelGenaiScore) || !isScore(row.openInferenceScore) || + !isScore(row.absoluteDelta) || row.otelGenaiScore !== otel.score || + row.openInferenceScore !== openInference.score || row.absoluteDelta !== expectedDelta || + row.maximumDelta !== 0.20 || row.passed !== (expectedDelta <= row.maximumDelta)) { + throw new ProviderParityError("provider_parity_delta_exceeded"); + } + if (expectedDelta > 0.20) throw new ProviderParityError("provider_parity_delta_exceeded"); + } + if (parityIds.size !== EVALUATORS.length || report.status !== "passed") { + throw new ProviderParityError("provider_parity_delta_exceeded"); + } +} + +function validatePolicy(policy: ProviderParityPolicy): void { + const thresholds = policy?.evaluatorThresholds as unknown; + if (!policy || policy.contractVersion !== "1.0" || policy.profileId !== "provider-parity-v1" || + policy.scenarioId !== "synthetic-cited-answer" || policy.maximumProviderCalls !== 6) { + throw new ProviderParityError("provider_policy_invalid"); + } + if (policy.maximumParityDelta !== 0.20 || !isRecord(thresholds) || + !sameStrings(Object.keys(thresholds), EVALUATORS) || + EVALUATORS.some((evaluatorId) => thresholds[evaluatorId] !== 0.70)) { + throw new ProviderParityError("provider_policy_invalid"); + } +} + +export function deriveProviderResultContext( + request: ProviderEvaluationRequest +): { sessionId: string; traceId?: string; spanId?: string } { + if (!isRecord(request.evaluationInput) || !Array.isArray(request.evaluationInput.sessionSpans) || + request.evaluationInput.sessionSpans.length === 0) { + throw new ProviderParityError("provider_context_mismatch"); + } + + const spans = request.evaluationInput.sessionSpans.map((span) => providerSpanContext(span)); + const sessionIds = new Set(spans.map((span) => span.sessionId)); + if (sessionIds.size !== 1) throw new ProviderParityError("provider_context_mismatch"); + const sessionId = spans[0]!.sessionId; + + switch (request.evaluatorId) { + case "Builtin.Correctness": { + const traceId = singleTargetId(request.evaluationTarget, "traceIds"); + if (!spans.some((span) => span.traceId === traceId)) { + throw new ProviderParityError("provider_context_mismatch"); + } + return { sessionId, traceId }; + } + case "Builtin.ToolSelectionAccuracy": { + const spanId = singleTargetId(request.evaluationTarget, "spanIds"); + const matches = spans.filter((span) => span.spanId === spanId); + if (matches.length !== 1) throw new ProviderParityError("provider_context_mismatch"); + return { sessionId, traceId: matches[0]!.traceId, spanId }; + } + case "Builtin.GoalSuccessRate": + if (request.evaluationTarget !== undefined) throw new ProviderParityError("provider_context_mismatch"); + return { sessionId }; + default: + throw new ProviderParityError("provider_evaluator_unexpected"); + } +} + +function hasExpectedRequestShape( + request: ProviderEvaluationRequest, + evaluatorId: ProviderEvaluatorId, + expectedContext: { sessionId: string; traceId?: string; spanId?: string } +): boolean { + const target = request.evaluationTarget; + switch (evaluatorId) { + case "Builtin.Correctness": + return isProviderTarget(target, "traceIds") && + hasExactReference(request.evaluationReferenceInputs, "expectedResponse", expectedContext); + case "Builtin.ToolSelectionAccuracy": + return isProviderTarget(target, "spanIds") && request.evaluationReferenceInputs === undefined; + case "Builtin.GoalSuccessRate": + return target === undefined && + hasExactReference(request.evaluationReferenceInputs, "assertions", expectedContext); + } +} + +function hasExactReference( + references: ProviderEvaluationRequest["evaluationReferenceInputs"], + field: "expectedResponse" | "assertions", + expectedContext: { sessionId: string; traceId?: string; spanId?: string } +): boolean { + if (!Array.isArray(references) || references.length !== 1 || !isRecord(references[0]) || + !hasOnlyKeys(references[0], ["context", field]) || !sameContext(references[0].context, expectedContext)) { + return false; + } + if (field === "expectedResponse") { + return isTextContent(references[0].expectedResponse); + } + return Array.isArray(references[0].assertions) && references[0].assertions.length === 1 && + isTextContent(references[0].assertions[0]); +} + +function isTextContent(value: unknown): boolean { + return isRecord(value) && hasOnlyKeys(value, ["text"]) && + typeof value.text === "string" && value.text.length > 0; +} + +function providerSpanContext(span: unknown): { sessionId: string; traceId: string; spanId: string } { + if (!isRecord(span) || typeof span.traceId !== "string" || span.traceId.length === 0 || + typeof span.spanId !== "string" || span.spanId.length === 0 || !isRecord(span.attributes) || + typeof span.attributes["session.id"] !== "string" || span.attributes["session.id"].length === 0) { + throw new ProviderParityError("provider_context_mismatch"); + } + return { + sessionId: span.attributes["session.id"], + traceId: span.traceId, + spanId: span.spanId + }; +} + +function singleTargetId(target: unknown, key: "traceIds" | "spanIds"): string { + if (!isProviderTarget(target, key)) throw new ProviderParityError("provider_context_mismatch"); + return (target as Record<"traceIds" | "spanIds", string[]>)[key][0]!; +} + +function isProviderTarget(target: unknown, key: "traceIds" | "spanIds"): boolean { + return isRecord(target) && hasOnlyKeys(target, [key]) && + Array.isArray(target[key]) && target[key].length === 1 && + target[key].every((value) => typeof value === "string" && value.length > 0); +} + +function assertResultCoverage(results: ProviderParityResult[]): void { + const keys = new Set(); + for (const result of results) { + if (!isRecord(result)) throw new ProviderParityError("provider_result_coverage_invalid"); + if (!isConvention(result.convention) || !isEvaluatorId(result.evaluatorId)) { + throw new ProviderParityError("provider_result_coverage_invalid"); + } + const key = `${result.convention}:${result.evaluatorId}`; + if (!EXPECTED_KEYS.has(key) || keys.has(key)) throw new ProviderParityError("provider_result_coverage_invalid"); + keys.add(key); + } + if (keys.size !== EXPECTED_KEYS.size) throw new ProviderParityError("provider_result_coverage_invalid"); +} + +function assertReportResult(result: ProviderParityResult): void { + if (!hasOnlyKeys(result, RESULT_KEYS) || result.level !== levelFor(result.evaluatorId)) { + throw new ProviderParityError("provider_result_coverage_invalid"); + } + if (!isScore(result.score)) throw new ProviderParityError("provider_score_invalid"); + if (result.threshold !== 0.70) { + throw new ProviderParityError("provider_score_below_threshold"); + } + if (result.label !== SAFE_PROVIDER_LABEL) throw new ProviderParityError("provider_label_invalid"); + if (result.reasonCode !== "passed") throw new ProviderParityError("provider_result_coverage_invalid"); + if (!isTokenUsage(result.tokenUsage)) throw new ProviderParityError("provider_token_usage_invalid"); + if (result.score < result.threshold || result.passed !== true) { + throw new ProviderParityError("provider_score_below_threshold"); + } +} + +function aggregateTokens(results: ProviderParityResult[]): ProviderParityReport["aggregateTokenUsage"] { + const totals = { inputTokens: 0, outputTokens: 0, totalTokens: 0 }; + for (const result of results) { + if (!isTokenUsage(result.tokenUsage)) throw new ProviderParityError("provider_token_usage_invalid"); + totals.inputTokens = checkedSum(totals.inputTokens, result.tokenUsage.inputTokens); + totals.outputTokens = checkedSum(totals.outputTokens, result.tokenUsage.outputTokens); + totals.totalTokens = checkedSum(totals.totalTokens, result.tokenUsage.totalTokens); + } + return totals; +} + +function resultFor( + results: ProviderParityResult[], + convention: EvaluationConvention, + evaluatorId: ProviderEvaluatorId +): ProviderParityResult { + const result = results.find((candidate) => candidate.convention === convention && candidate.evaluatorId === evaluatorId); + if (!result) throw new ProviderParityError("provider_result_coverage_invalid"); + return result; +} + +function levelFor(evaluatorId: ProviderEvaluatorId): ProviderEvaluationLevel { + switch (evaluatorId) { + case "Builtin.Correctness": return "trace"; + case "Builtin.ToolSelectionAccuracy": return "tool-call"; + case "Builtin.GoalSuccessRate": return "session"; + } +} + +function sameContext( + actual: unknown, + expected: { sessionId: string; traceId?: string; spanId?: string } +): boolean { + if (!isRecord(actual) || !isRecord(actual.spanContext)) return false; + const spanContext = actual.spanContext; + if (typeof spanContext.sessionId !== "string") return false; + const actualKeys = Object.keys(spanContext).sort(); + const expectedKeys = Object.keys(expected).sort(); + return actualKeys.length === expectedKeys.length && actualKeys.every((key, index) => key === expectedKeys[index]) && + spanContext.sessionId === expected.sessionId && spanContext.traceId === expected.traceId && spanContext.spanId === expected.spanId; +} + +function isScore(value: unknown): value is number { + return typeof value === "number" && Number.isFinite(value) && value >= 0 && value <= 1; +} + +function isTokenUsage(value: unknown): value is { inputTokens: number; outputTokens: number; totalTokens: number } { + return isRecord(value) && isSafeTokenCount(value.inputTokens) && + isSafeTokenCount(value.outputTokens) && isSafeTokenCount(value.totalTokens) && + hasOnlyKeys(value, ["inputTokens", "outputTokens", "totalTokens"]); +} + +function isSafeTokenCount(value: unknown): value is number { + return typeof value === "number" && Number.isSafeInteger(value) && value >= 0; +} + +function assertReportEnvelope(report: unknown): asserts report is ProviderParityReport { + if (!isRecord(report) || !hasOnlyKeys(report, REPORT_KEYS)) { + throw new ProviderParityError("provider_result_coverage_invalid"); + } + if (report.contractVersion !== "1.0" || report.thresholdVersion !== "1.0" || + report.evidenceLevel !== "provider-direct" || report.scenarioId !== "synthetic-cited-answer") { + throw new ProviderParityError("provider_policy_invalid"); + } + if (!isIsoDateTime(report.generatedAt) || !isGithubRunId(report.githubRunId) || + !isDurationBucket(report.durationBucket) || (report.status !== "passed" && report.status !== "failed")) { + throw new ProviderParityError("provider_result_coverage_invalid"); + } + if (!isSourceCommit(report.sourceCommit)) throw new ProviderParityError("provider_source_commit_invalid"); + if (report.regionLabel !== "ap-southeast-2") throw new ProviderParityError("provider_region_invalid"); + if (!isTokenUsage(report.aggregateTokenUsage)) throw new ProviderParityError("provider_token_usage_invalid"); +} + +function isProviderLabel(value: unknown): value is string { + return typeof value === "string" && value.length > 0 && value.length <= 80 && /^[\x20-\x7e]+$/.test(value); +} + +function isIsoDateTime(value: unknown): value is string { + if (typeof value !== "string") return false; + const timestamp = new Date(value); + return Number.isFinite(timestamp.getTime()) && timestamp.toISOString() === value; +} + +function isSourceCommit(value: unknown): value is string { + return typeof value === "string" && /^[0-9a-f]{40}$/.test(value); +} + +function isGithubRunId(value: unknown): value is string { + return typeof value === "string" && /^[0-9]+$/.test(value); +} + +function isDurationBucket(value: unknown): value is ProviderParityReport["durationBucket"] { + return typeof value === "string" && DURATION_BUCKETS.has(value); +} + +function checkedSum(left: number, right: number): number { + const total = left + right; + if (!Number.isSafeInteger(total)) throw new ProviderParityError("provider_token_usage_invalid"); + return total; +} + +function sameTokenUsage(left: unknown, right: unknown): boolean { + return isTokenUsage(left) && isTokenUsage(right) && + left.inputTokens === right.inputTokens && left.outputTokens === right.outputTokens && left.totalTokens === right.totalTokens; +} + +function isEvaluatorId(value: unknown): value is ProviderEvaluatorId { + return typeof value === "string" && EVALUATORS.includes(value as ProviderEvaluatorId); +} + +function isConvention(value: unknown): value is EvaluationConvention { + return typeof value === "string" && CONVENTIONS.includes(value as EvaluationConvention); +} + +function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value); +} + +function hasOnlyKeys(value: unknown, keys: string[]): boolean { + return isRecord(value) && sameStrings(Object.keys(value), keys); +} + +function sameStrings(actual: string[], expected: readonly string[]): boolean { + return actual.length === expected.length && actual.every((value) => expected.includes(value)); +} diff --git a/providers/aws/app/api/src/evals/agentCoreEvaluationProviderTypes.ts b/providers/aws/app/api/src/evals/agentCoreEvaluationProviderTypes.ts new file mode 100644 index 0000000..e5d5157 --- /dev/null +++ b/providers/aws/app/api/src/evals/agentCoreEvaluationProviderTypes.ts @@ -0,0 +1,131 @@ +import type { EvaluationConvention } from "./agentEvaluationTelemetryTypes.js"; + +export type ProviderEvaluatorId = + | "Builtin.Correctness" + | "Builtin.ToolSelectionAccuracy" + | "Builtin.GoalSuccessRate"; + +export type ProviderEvaluationLevel = "trace" | "tool-call" | "session"; + +export type ProviderParityErrorCode = + | "provider_fixture_not_synthetic" + | "provider_scenario_not_allowed" + | "provider_scope_not_allowed" + | "provider_required_span_missing" + | "provider_session_count_invalid" + | "provider_attribute_not_allowed" + | "provider_policy_invalid" + | "provider_call_count_invalid" + | "provider_result_missing" + | "provider_result_duplicate" + | "provider_evaluator_unexpected" + | "provider_result_failed" + | "provider_reference_input_ignored" + | "provider_score_invalid" + | "provider_context_mismatch" + | "provider_token_usage_invalid" + | "provider_label_invalid" + | "provider_score_below_threshold" + | "provider_parity_delta_exceeded" + | "provider_result_coverage_invalid" + | "provider_confirmation_required" + | "provider_readiness_required" + | "provider_call_budget_invalid" + | "provider_region_invalid" + | "provider_source_ref_invalid" + | "provider_source_commit_invalid" + | "provider_output_path_required" + | "provider_input_file_invalid" + | "provider_request_failed" + | "provider_artifact_write_failed"; + +export class ProviderParityError extends Error { + constructor(public readonly code: ProviderParityErrorCode) { + super(code); + this.name = "ProviderParityError"; + } +} + +export type ProviderParityPolicy = { + contractVersion: "1.0"; + profileId: "provider-parity-v1"; + scenarioId: "synthetic-cited-answer"; + evaluatorThresholds: Record; + maximumParityDelta: number; + maximumProviderCalls: 6; +}; + +export type ProviderDocument = + | null + | boolean + | number + | string + | ProviderDocument[] + | { [key: string]: ProviderDocument }; + +export type ProviderDocumentObject = { [key: string]: ProviderDocument }; + +export type ProviderEvaluationRequest = { + evaluatorId: ProviderEvaluatorId; + evaluationInput: { sessionSpans: ProviderDocumentObject[] }; + evaluationTarget?: { traceIds: string[] } | { spanIds: string[] }; + evaluationReferenceInputs?: Array<{ + context: { spanContext: { sessionId: string; traceId?: string; spanId?: string } }; + expectedResponse?: { text: string }; + assertions?: Array<{ text: string }>; + }>; +}; + +export type ProviderEvaluationResponse = { + evaluationResults?: Array<{ + evaluatorId?: string; + value?: number; + label?: string; + errorCode?: string; + errorMessage?: string; + explanation?: string; + ignoredReferenceInputFields?: string[]; + context?: { spanContext?: { sessionId?: string; traceId?: string; spanId?: string } }; + tokenUsage?: { inputTokens?: number; outputTokens?: number; totalTokens?: number }; + }>; +}; + +export interface AgentCoreEvaluateClient { + evaluate(request: ProviderEvaluationRequest): Promise; +} + +export type ProviderParityResult = { + convention: EvaluationConvention; + evaluatorId: ProviderEvaluatorId; + level: ProviderEvaluationLevel; + score: number; + label: string; + threshold: number; + passed: boolean; + reasonCode: string; + tokenUsage: { inputTokens: number; outputTokens: number; totalTokens: number }; +}; + +export type ProviderParityReport = { + contractVersion: "1.0"; + thresholdVersion: "1.0"; + evidenceLevel: "provider-direct"; + generatedAt: string; + sourceCommit: string; + githubRunId: string; + regionLabel: "ap-southeast-2"; + scenarioId: "synthetic-cited-answer"; + status: "passed" | "failed"; + providerCallCount: 6; + durationBucket: "under-1m" | "under-5m" | "under-15m" | "15m-or-more"; + aggregateTokenUsage: { inputTokens: number; outputTokens: number; totalTokens: number }; + results: ProviderParityResult[]; + parity: Array<{ + evaluatorId: ProviderEvaluatorId; + otelGenaiScore: number; + openInferenceScore: number; + absoluteDelta: number; + maximumDelta: number; + passed: boolean; + }>; +}; diff --git a/providers/aws/app/api/src/evals/agentCoreEvaluationRequestBuilder.ts b/providers/aws/app/api/src/evals/agentCoreEvaluationRequestBuilder.ts new file mode 100644 index 0000000..2f29f01 --- /dev/null +++ b/providers/aws/app/api/src/evals/agentCoreEvaluationRequestBuilder.ts @@ -0,0 +1,357 @@ +import { createHash } from "node:crypto"; + +import { + ProviderParityError, + type ProviderDocument, + type ProviderDocumentObject, + type ProviderEvaluationRequest, + type ProviderParityPolicy +} from "./agentCoreEvaluationProviderTypes.js"; +import type { + EvaluationConvention, + EvaluationScenario, + EvaluationTelemetrySpan, + TelemetryFixture +} from "./agentEvaluationTelemetryTypes.js"; + +const PROVIDER_ASSERTION_TEXT = { + "citation-present": "The final answer cites the approved synthetic source." +} as const; + +const EXPECTED_EVALUATORS = [ + "Builtin.Correctness", + "Builtin.ToolSelectionAccuracy", + "Builtin.GoalSuccessRate" +] as const; + +const ATTRIBUTE_ALLOWLISTS = { + "otel-genai": { + "invoke-agent": new Set([ + "session.id", + "gen_ai.operation.name", + "gen_ai.input.messages", + "gen_ai.output.messages" + ]), + inference: new Set([ + "session.id", + "gen_ai.operation.name", + "gen_ai.input.messages", + "gen_ai.output.messages" + ]), + "execute-tool": new Set([ + "session.id", + "gen_ai.operation.name", + "gen_ai.tool.name", + "gen_ai.tool.call.id", + "gen_ai.tool.call.arguments", + "gen_ai.tool.call.result" + ]) + }, + openinference: { + "invoke-agent": new Set([ + "session.id", + "openinference.span.kind", + "input.value", + "output.value" + ]), + inference: new Set([ + "session.id", + "openinference.span.kind", + "llm.input_messages.0.message.role", + "llm.input_messages.0.message.content", + "llm.output_messages.0.message.role", + "llm.output_messages.0.message.tool_calls.0.tool_call.function.name", + "llm.output_messages.0.message.tool_calls.0.tool_call.function.arguments", + "llm.output_messages.0.message.tool_calls.0.tool_call.id" + ]), + "execute-tool": new Set([ + "session.id", + "openinference.span.kind", + "tool.name", + "input.value", + "output.value" + ]) + } +} as const; + +type SpanRole = "invoke-agent" | "inference" | "execute-tool"; + +type DerivedProviderIds = { + traceId: string; + spanIds: Record; +}; + +export function buildProviderEvaluationRequests( + fixture: TelemetryFixture, + scenario: EvaluationScenario, + policy: ProviderParityPolicy +): ProviderEvaluationRequest[] { + validatePolicy(policy); + validateScenario(scenario); + const reviewed = validateFixture(fixture); + const ids = deriveProviderIds(fixture.scenarioId, fixture.convention, fixture.spans.map((span) => span.spanId)); + const sessionSpans = fixture.spans.map((span) => mapSpan(span, reviewed.sessionId, ids)); + const toolSpanId = ids.spanIds[reviewed.toolSpan.spanId]!; + + return [ + { + evaluatorId: "Builtin.Correctness", + evaluationInput: { sessionSpans }, + evaluationTarget: { traceIds: [ids.traceId] }, + evaluationReferenceInputs: [{ + context: { spanContext: { sessionId: reviewed.sessionId, traceId: ids.traceId } }, + expectedResponse: { text: scenario.expectedResponse } + }] + }, + { + evaluatorId: "Builtin.ToolSelectionAccuracy", + evaluationInput: { sessionSpans }, + evaluationTarget: { spanIds: [toolSpanId] } + }, + { + evaluatorId: "Builtin.GoalSuccessRate", + evaluationInput: { sessionSpans }, + evaluationReferenceInputs: [{ + context: { spanContext: { sessionId: reviewed.sessionId } }, + assertions: [{ text: PROVIDER_ASSERTION_TEXT["citation-present"] }] + }] + } + ]; +} + +export function deriveProviderIds( + scenarioId: string, + convention: EvaluationConvention, + originalSpanIds: string[] +): DerivedProviderIds { + return { + traceId: hexId(32, "cloudai-provider-parity-trace-v1", scenarioId, convention), + spanIds: Object.fromEntries(originalSpanIds.map((spanId) => [ + spanId, + hexId(16, "cloudai-provider-parity-span-v1", scenarioId, convention, spanId) + ])) + }; +} + +function validatePolicy(policy: ProviderParityPolicy): void { + if (policy.maximumProviderCalls !== 6) { + throw new ProviderParityError("provider_call_count_invalid"); + } + + const thresholds = policy.evaluatorThresholds as unknown; + if (!isRecord(thresholds) || + policy.contractVersion !== "1.0" || + policy.profileId !== "provider-parity-v1" || + policy.scenarioId !== "synthetic-cited-answer" || + policy.maximumParityDelta !== 0.2 || + !sameStrings(Object.keys(thresholds), EXPECTED_EVALUATORS) || + thresholds["Builtin.Correctness"] !== 0.7 || + thresholds["Builtin.ToolSelectionAccuracy"] !== 0.7 || + thresholds["Builtin.GoalSuccessRate"] !== 0.7) { + throw new ProviderParityError("provider_policy_invalid"); + } +} + +function validateScenario(scenario: EvaluationScenario): void { + if (scenario.contractVersion !== "1.0" || + scenario.scenarioId !== "synthetic-cited-answer" || + scenario.syntheticOnly !== true || + typeof scenario.expectedResponse !== "string" || + scenario.expectedResponse.length === 0 || + scenario.expectedToolTrajectory.length !== 1 || + scenario.expectedToolTrajectory[0]!.name !== "knowledge_search" || + scenario.assertions.length !== 1 || + scenario.assertions[0] !== "citation-present") { + throw new ProviderParityError("provider_scenario_not_allowed"); + } +} + +function validateFixture(fixture: TelemetryFixture): { + sessionId: string; + toolSpan: EvaluationTelemetrySpan; +} { + if (fixture.contractVersion !== "1.0" || + fixture.scenarioId !== "synthetic-cited-answer" || + (fixture.convention !== "otel-genai" && fixture.convention !== "openinference")) { + throw new ProviderParityError("provider_fixture_not_synthetic"); + } + + const scopePrefix = fixture.convention === "otel-genai" + ? "opentelemetry.instrumentation." + : "openinference.instrumentation."; + if (fixture.spans.length === 0 || fixture.spans.some((span) => !span.scopeName.startsWith(scopePrefix))) { + throw new ProviderParityError("provider_scope_not_allowed"); + } + + const sessionIds = fixture.spans.map((span) => span.attributes["session.id"]); + if (sessionIds.some((sessionId) => typeof sessionId !== "string" || sessionId.length === 0) || + new Set(sessionIds).size !== 1) { + throw new ProviderParityError("provider_session_count_invalid"); + } + + const classified = fixture.spans.map((span) => ({ span, role: classifySpan(fixture.convention, span) })); + for (const { span, role } of classified) { + if (role) validateAttributes(fixture.convention, role, span); + } + + const agentSpans = classified.filter(({ role }) => role === "invoke-agent").map(({ span }) => span); + const inferenceSpans = classified.filter(({ role }) => role === "inference").map(({ span }) => span); + const toolSpans = classified.filter(({ role }) => role === "execute-tool").map(({ span }) => span); + if (agentSpans.length !== 1 || inferenceSpans.length !== 1 || toolSpans.length !== 1 || + fixture.spans.length !== 3 || new Set(fixture.spans.map((span) => span.spanId)).size !== fixture.spans.length || + !hasReviewedParents(agentSpans[0]!, inferenceSpans[0]!, toolSpans[0]!)) { + throw new ProviderParityError("provider_required_span_missing"); + } + + return { sessionId: sessionIds[0] as string, toolSpan: toolSpans[0]! }; +} + +function validateAttributes( + convention: EvaluationConvention, + role: SpanRole, + span: EvaluationTelemetrySpan +): void { + const allowed = ATTRIBUTE_ALLOWLISTS[convention][role]; + if (Object.keys(span.attributes).some((key) => !allowed.has(key)) || + [...allowed].some((key) => !(key in span.attributes))) { + throw new ProviderParityError("provider_attribute_not_allowed"); + } +} + +function classifySpan(convention: EvaluationConvention, span: EvaluationTelemetrySpan): SpanRole | null { + if (convention === "otel-genai") { + switch (span.attributes["gen_ai.operation.name"]) { + case "invoke_agent": return "invoke-agent"; + case "chat": return "inference"; + case "execute_tool": return "execute-tool"; + default: return null; + } + } + + switch (span.attributes["openinference.span.kind"]) { + case "AGENT": return "invoke-agent"; + case "LLM": return "inference"; + case "TOOL": return "execute-tool"; + default: return null; + } +} + +function hasReviewedParents( + agent: EvaluationTelemetrySpan, + inference: EvaluationTelemetrySpan, + tool: EvaluationTelemetrySpan +): boolean { + return agent.parentSpanId === null && + inference.parentSpanId === agent.spanId && + tool.parentSpanId === agent.spanId; +} + +function mapSpan( + source: EvaluationTelemetrySpan, + sessionId: string, + ids: DerivedProviderIds +): ProviderDocumentObject { + const role = classifySpanFromKnownSource(source); + const parentSpanId = source.parentSpanId === null ? undefined : ids.spanIds[source.parentSpanId]; + return { + traceId: ids.traceId, + spanId: ids.spanIds[source.spanId], + ...(parentSpanId ? { parentSpanId } : {}), + name: role, + kind: 1, + startTimeUnixNano: source.startTimeUnixNano, + endTimeUnixNano: (BigInt(source.startTimeUnixNano) + 1n).toString(), + attributes: mapAttributes(source, role, sessionId), + scope: { name: source.scopeName, version: "1.0.0" }, + resource: { + attributes: { + "service.name": "cloudai-provider-parity-synthetic", + "cloudai.data.scope": "synthetic-only" + } + }, + status: { code: 1 } + }; +} + +function mapAttributes( + source: EvaluationTelemetrySpan, + role: SpanRole, + sessionId: string +): ProviderDocumentObject { + if (role !== "invoke-agent" || source.attributes["gen_ai.operation.name"] !== "invoke_agent") { + return { ...copyProviderDocument(source.attributes), "session.id": sessionId }; + } + + const prompt = extractSingleTextMessage(source.attributes["gen_ai.input.messages"], "user"); + const finalResponse = extractSingleTextMessage(source.attributes["gen_ai.output.messages"], "assistant"); + return { + "session.id": sessionId, + "gen_ai.operation.name": "invoke_agent", + "gen_ai.task.input": prompt, + "gen_ai.task.output": finalResponse + }; +} + +function copyProviderDocument(value: Record): ProviderDocumentObject { + const copied: ProviderDocumentObject = {}; + for (const [key, candidate] of Object.entries(value)) { + if (!isProviderDocument(candidate)) throw new ProviderParityError("provider_attribute_not_allowed"); + copied[key] = candidate; + } + return copied; +} + +function isProviderDocument(value: unknown): value is ProviderDocument { + if (value === null || typeof value === "string" || typeof value === "boolean") return true; + if (typeof value === "number") return Number.isFinite(value); + if (Array.isArray(value)) return value.every(isProviderDocument); + return isRecord(value) && Object.values(value).every(isProviderDocument); +} + +function extractSingleTextMessage(value: unknown, expectedRole: "user" | "assistant"): string { + if (typeof value !== "string") throw new ProviderParityError("provider_attribute_not_allowed"); + + let parsed: unknown; + try { + parsed = JSON.parse(value) as unknown; + } catch { + throw new ProviderParityError("provider_attribute_not_allowed"); + } + if (!Array.isArray(parsed) || parsed.length !== 1 || !isRecord(parsed[0]) || + !hasOnlyKeys(parsed[0], ["role", "parts"]) || parsed[0].role !== expectedRole || + !Array.isArray(parsed[0].parts) || parsed[0].parts.length !== 1 || !isRecord(parsed[0].parts[0]) || + !hasOnlyKeys(parsed[0].parts[0], ["type", "content"]) || parsed[0].parts[0].type !== "text" || + typeof parsed[0].parts[0].content !== "string" || parsed[0].parts[0].content.length === 0 || + parsed[0].parts[0].content.trim() !== parsed[0].parts[0].content) { + throw new ProviderParityError("provider_attribute_not_allowed"); + } + return parsed[0].parts[0].content; +} + +function classifySpanFromKnownSource(source: EvaluationTelemetrySpan): SpanRole { + if (source.attributes["gen_ai.operation.name"] === "invoke_agent" || source.attributes["openinference.span.kind"] === "AGENT") { + return "invoke-agent"; + } + if (source.attributes["gen_ai.operation.name"] === "chat" || source.attributes["openinference.span.kind"] === "LLM") { + return "inference"; + } + return "execute-tool"; +} + +function hexId(width: 16 | 32, ...parts: string[]): string { + return createHash("sha256").update(parts.join("\u0000")).digest("hex").slice(0, width); +} + +function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value); +} + +function hasOnlyKeys(value: Record, keys: string[]): boolean { + const actual = Object.keys(value).sort(); + const expected = [...keys].sort(); + return actual.length === expected.length && actual.every((key, index) => key === expected[index]); +} + +function sameStrings(actual: string[], expected: readonly string[]): boolean { + return actual.length === expected.length && actual.sort().every((value, index) => value === [...expected].sort()[index]); +} diff --git a/providers/aws/app/api/src/scripts/runAgentCoreEvaluationProviderParity.ts b/providers/aws/app/api/src/scripts/runAgentCoreEvaluationProviderParity.ts new file mode 100644 index 0000000..ceaeff8 --- /dev/null +++ b/providers/aws/app/api/src/scripts/runAgentCoreEvaluationProviderParity.ts @@ -0,0 +1,324 @@ +import { readFile, writeFile } from "node:fs/promises"; +import { resolve } from "node:path"; +import { pathToFileURL } from "node:url"; + +import { createAwsAgentCoreEvaluateClient } from "../clients/agentCoreEvaluationClient.js"; +import { + assertProviderParityGate, + buildProviderParityReport, + deriveProviderResultContext, + sanitizeProviderResult, + type ProviderEvaluationPair +} from "../evals/agentCoreEvaluationProviderGate.js"; +import { buildProviderEvaluationRequests } from "../evals/agentCoreEvaluationRequestBuilder.js"; +import { + ProviderParityError, + type AgentCoreEvaluateClient, + type ProviderEvaluationRequest, + type ProviderParityErrorCode, + type ProviderParityPolicy, + type ProviderParityReport +} from "../evals/agentCoreEvaluationProviderTypes.js"; +import type { + EvaluationConvention, + EvaluationScenario, + TelemetryFixture +} from "../evals/agentEvaluationTelemetryTypes.js"; + +export type ProviderParityMode = "validate" | "direct-spans"; +export type ProviderParityRunOptions = { + mode: ProviderParityMode; + scenarioPath: string; + fixturePaths: [string, string]; + policyPath: string; + outputPath?: string; + generatedAt: string; + githubRunId: string; + environment: NodeJS.ProcessEnv; +}; +export type ProviderParityRunResult = + | { mode: "validate"; status: "passed"; requestCount: 6 } + | { mode: "direct-spans"; status: "passed"; report: ProviderParityReport }; +export type AgentCoreEvaluateClientFactory = () => AgentCoreEvaluateClient; + +export type ProviderParityEnvironmentConfiguration = + | { mode: "validate"; sourceCommit: string } + | { + mode: "direct-spans"; + sourceCommit: string; + region: "ap-southeast-2"; + }; + +const CONFIRMATION = "I_UNDERSTAND_AGENTCORE_EVALUATION_PROVIDER_PARITY"; +const CALL_BUDGET = 6; +const EVALUATORS = [ + "Builtin.Correctness", + "Builtin.ToolSelectionAccuracy", + "Builtin.GoalSuccessRate" +] as const; + +export function validateProviderParityEnvironment( + environment: NodeJS.ProcessEnv, + mode: ProviderParityMode = environment.PROVIDER_PARITY_MODE === "direct-spans" ? "direct-spans" : "validate" +): ProviderParityEnvironmentConfiguration { + if (mode === "validate") { + const sourceCommit = environment.GITHUB_SHA; + if (sourceCommit !== "local" && !isCommitSha(sourceCommit)) { + throw new ProviderParityError("provider_source_commit_invalid"); + } + return { mode, sourceCommit }; + } + + if (environment.PROVIDER_PARITY_MODE !== "direct-spans" || environment.CONFIRMATION !== CONFIRMATION) { + throw new ProviderParityError("provider_confirmation_required"); + } + if (environment.AGENTCORE_EVALUATION_READY !== "true") { + throw new ProviderParityError("provider_readiness_required"); + } + if (environment.AGENTCORE_EVALUATION_MAX_CALLS !== String(CALL_BUDGET)) { + throw new ProviderParityError("provider_call_budget_invalid"); + } + if (environment.AWS_REGION !== "ap-southeast-2") { + throw new ProviderParityError("provider_region_invalid"); + } + if (environment.GITHUB_REF !== "refs/heads/main") { + throw new ProviderParityError("provider_source_ref_invalid"); + } + if (!isCommitSha(environment.GITHUB_SHA)) { + throw new ProviderParityError("provider_source_commit_invalid"); + } + return { + mode, + sourceCommit: environment.GITHUB_SHA, + region: "ap-southeast-2" + }; +} + +export async function runProviderParityEvaluation( + options: ProviderParityRunOptions, + clientFactory: AgentCoreEvaluateClientFactory +): Promise { + const configuration = validateProviderParityEnvironment(options.environment, options.mode); + if (options.mode === "direct-spans" && !options.outputPath) { + throw new ProviderParityError("provider_output_path_required"); + } + + const { scenario, fixtures, policy } = await loadReviewedInputs(options); + let requests: ProviderEvaluationRequest[]; + try { + requests = fixtures.flatMap((fixture) => buildProviderEvaluationRequests(fixture, scenario, policy)); + } catch (error: unknown) { + if (error instanceof ProviderParityError) throw error; + throw new ProviderParityError("provider_input_file_invalid"); + } + if (requests.length !== CALL_BUDGET) { + throw new ProviderParityError("provider_call_count_invalid"); + } + + const startedAt = Date.now(); + let client: AgentCoreEvaluateClient; + try { + client = clientFactory(); + } catch { + throw new ProviderParityError("provider_request_failed"); + } + + if (options.mode === "direct-spans") { + console.info(`agentcore-provider-parity-start mode=${options.mode} call_budget=6`); + } + const pairs: ProviderEvaluationPair[] = []; + for (const [index, request] of requests.entries()) { + const convention = fixtures[Math.floor(index / EVALUATORS.length)]!.convention; + let response; + try { + response = await client.evaluate(request); + } catch { + throw new ProviderParityError("provider_request_failed"); + } + const pair = { convention, request, response }; + sanitizeProviderResult(pair, policy); + pairs.push(pair); + if (options.mode === "direct-spans") { + console.info( + `agentcore-provider-evaluation-complete convention=${convention} evaluator=${request.evaluatorId}` + ); + } + } + + if (options.mode === "validate") { + assertValidatedPairCoverage(pairs, policy); + return { mode: "validate", status: "passed", requestCount: 6 }; + } + + const report = buildProviderParityReport({ + pairs, + policy, + generatedAt: options.generatedAt, + sourceCommit: configuration.sourceCommit, + githubRunId: options.githubRunId, + durationBucket: durationBucket(Date.now() - startedAt) + }); + assertProviderParityGate(report); + try { + await writeFile(options.outputPath!, `${JSON.stringify(report, null, 2)}\n`, "utf8"); + } catch { + throw new ProviderParityError("provider_artifact_write_failed"); + } + console.info("agentcore-provider-parity-passed evidence_level=provider-direct calls=6"); + return { mode: "direct-spans", status: "passed", report }; +} + +async function loadReviewedInputs(options: ProviderParityRunOptions): Promise<{ + scenario: EvaluationScenario; + fixtures: [TelemetryFixture, TelemetryFixture]; + policy: ProviderParityPolicy; +}> { + let scenarios: unknown; + let fixtureGroups: [unknown, unknown]; + let policy: unknown; + try { + [scenarios, fixtureGroups, policy] = await Promise.all([ + readJson(options.scenarioPath), + Promise.all([readJson(options.fixturePaths[0]), readJson(options.fixturePaths[1])]), + readJson(options.policyPath) + ]); + } catch { + throw new ProviderParityError("provider_input_file_invalid"); + } + if (!Array.isArray(scenarios) || !Array.isArray(fixtureGroups[0]) || !Array.isArray(fixtureGroups[1])) { + throw new ProviderParityError("provider_input_file_invalid"); + } + + const matchingScenarios = scenarios.filter(isFixedScenario); + const otelFixtures = fixtureGroups[0].filter((value): value is TelemetryFixture => + isFixedFixture(value, "otel-genai")); + const openInferenceFixtures = fixtureGroups[1].filter((value): value is TelemetryFixture => + isFixedFixture(value, "openinference")); + if (matchingScenarios.length !== 1 || otelFixtures.length !== 1 || openInferenceFixtures.length !== 1 || + !isRecord(policy)) { + throw new ProviderParityError("provider_input_file_invalid"); + } + return { + scenario: matchingScenarios[0], + fixtures: [otelFixtures[0], openInferenceFixtures[0]], + policy: policy as ProviderParityPolicy + }; +} + +function assertValidatedPairCoverage(pairs: ProviderEvaluationPair[], policy: ProviderParityPolicy): void { + if (pairs.length !== CALL_BUDGET) throw new ProviderParityError("provider_call_count_invalid"); + const results = pairs.map((pair) => sanitizeProviderResult(pair, policy)); + for (const evaluatorId of EVALUATORS) { + const otel = results.find((result) => result.convention === "otel-genai" && result.evaluatorId === evaluatorId); + const openInference = results.find((result) => + result.convention === "openinference" && result.evaluatorId === evaluatorId); + if (!otel || !openInference) throw new ProviderParityError("provider_result_coverage_invalid"); + if (Math.abs(otel.score - openInference.score) > policy.maximumParityDelta) { + throw new ProviderParityError("provider_parity_delta_exceeded"); + } + } +} + +function isFixedScenario(value: unknown): value is EvaluationScenario { + return isRecord(value) && value.scenarioId === "synthetic-cited-answer"; +} + +function isFixedFixture(value: unknown, convention: EvaluationConvention): value is TelemetryFixture { + return isRecord(value) && value.scenarioId === "synthetic-cited-answer" && value.convention === convention; +} + +function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value); +} + +function isCommitSha(value: unknown): value is string { + return typeof value === "string" && /^[0-9a-f]{40}$/.test(value); +} + +function durationBucket(milliseconds: number): ProviderParityReport["durationBucket"] { + if (milliseconds < 60_000) return "under-1m"; + if (milliseconds < 300_000) return "under-5m"; + if (milliseconds < 900_000) return "under-15m"; + return "15m-or-more"; +} + +async function readJson(filePath: string): Promise { + return JSON.parse(await readFile(filePath, "utf8")) as unknown; +} + +type ParsedCli = { mode: ProviderParityMode; outputPath?: string }; + +export function parseProviderParityArguments(arguments_: string[]): ParsedCli { + const argumentsWithoutSeparator = arguments_[0] === "--" ? arguments_.slice(1) : arguments_; + if (argumentsWithoutSeparator.length === 2 && argumentsWithoutSeparator[0] === "--mode" && + argumentsWithoutSeparator[1] === "validate") { + return { mode: "validate" }; + } + if (argumentsWithoutSeparator.length === 4 && argumentsWithoutSeparator[0] === "--mode" && + argumentsWithoutSeparator[1] === "direct-spans" && argumentsWithoutSeparator[2] === "--output" && + isCliValue(argumentsWithoutSeparator[3])) { + return { mode: "direct-spans", outputPath: resolve(argumentsWithoutSeparator[3]) }; + } + throw new ProviderParityError(argumentsWithoutSeparator[1] === "direct-spans" + ? "provider_output_path_required" + : "provider_policy_invalid"); +} + +function isCliValue(value: string | undefined): value is string { + return typeof value === "string" && value.length > 0 && !value.startsWith("-"); +} + +async function main(): Promise { + const cli = parseProviderParityArguments(process.argv.slice(2)); + const exampleDirectory = resolve(process.cwd(), "../../../../shared/examples/agent-evaluation-telemetry"); + const environment = cli.mode === "validate" && process.env.GITHUB_SHA === undefined + ? { ...process.env, GITHUB_SHA: "local" } + : process.env; + const options: ProviderParityRunOptions = { + mode: cli.mode, + scenarioPath: resolve(exampleDirectory, "scenarios.v1.json"), + fixturePaths: [ + resolve(exampleDirectory, "otel-genai.traces.v1.json"), + resolve(exampleDirectory, "openinference.traces.v1.json") + ], + policyPath: resolve(exampleDirectory, "provider-parity-thresholds.v1.json"), + ...(cli.outputPath ? { outputPath: cli.outputPath } : {}), + generatedAt: new Date().toISOString(), + githubRunId: process.env.GITHUB_RUN_ID ?? "local", + environment + }; + const factory = cli.mode === "validate" + ? () => deterministicValidationClient() + : () => createAwsAgentCoreEvaluateClient("ap-southeast-2"); + await runProviderParityEvaluation(options, factory); + if (cli.mode === "validate") { + console.info("agentcore-provider-parity-passed evidence_level=local-contract calls=6"); + } +} + +function deterministicValidationClient(): AgentCoreEvaluateClient { + return { + async evaluate(request: ProviderEvaluationRequest) { + return { + evaluationResults: [{ + evaluatorId: request.evaluatorId, + value: 0.9, + label: "validated", + context: { spanContext: deriveProviderResultContext(request) }, + tokenUsage: { inputTokens: 1, outputTokens: 1, totalTokens: 2 } + }] + }; + } + }; +} + +const entryPoint = process.argv[1] ? pathToFileURL(process.argv[1]).href : null; +if (entryPoint === import.meta.url) { + main().catch((error: unknown) => { + const code: ProviderParityErrorCode = error instanceof ProviderParityError + ? error.code + : "provider_request_failed"; + console.error(`agentcore-provider-parity-failed code=${code}`); + process.exitCode = 1; + }); +} diff --git a/providers/aws/app/api/tests/agentCoreEvaluationProviderContracts.test.ts b/providers/aws/app/api/tests/agentCoreEvaluationProviderContracts.test.ts new file mode 100644 index 0000000..35f1ba3 --- /dev/null +++ b/providers/aws/app/api/tests/agentCoreEvaluationProviderContracts.test.ts @@ -0,0 +1,38 @@ +import assert from "node:assert/strict"; +import { readFile } from "node:fs/promises"; +import { resolve } from "node:path"; +import test from "node:test"; + +const ROOT = resolve(process.cwd(), "../../../.."); +const EXAMPLES = resolve(ROOT, "shared/examples/agent-evaluation-telemetry"); +const SCHEMAS = resolve(ROOT, "shared/schemas/agent-evaluation-telemetry"); + +test("provider-parity-v1 fixes three evaluators, thresholds, tolerance, and six calls", async () => { + const policy = JSON.parse(await readFile( + resolve(EXAMPLES, "provider-parity-thresholds.v1.json"), "utf8")); + assert.deepEqual(policy, { + contractVersion: "1.0", + profileId: "provider-parity-v1", + scenarioId: "synthetic-cited-answer", + evaluatorThresholds: { + "Builtin.Correctness": 0.70, + "Builtin.ToolSelectionAccuracy": 0.70, + "Builtin.GoalSuccessRate": 0.70 + }, + maximumParityDelta: 0.20, + maximumProviderCalls: 6 + }); +}); + +test("provider-direct schema is closed and contains no raw-content fields", async () => { + const schema = JSON.parse(await readFile( + resolve(SCHEMAS, "provider-parity-report.schema.json"), "utf8")); + assert.equal(schema.additionalProperties, false); + assert.equal(schema.properties.evidenceLevel.const, "provider-direct"); + const serialized = JSON.stringify(schema).toLowerCase(); + for (const forbidden of [ + "prompt", "response", "assertion", "trajectory", "toolarguments", + "toolresult", "sessionspans", "explanation", "errormessage", + "accountid", "resourcearn", "endpoint" + ]) assert.equal(serialized.includes(`\"${forbidden}\"`), false, forbidden); +}); diff --git a/providers/aws/app/api/tests/agentCoreEvaluationProviderGate.test.ts b/providers/aws/app/api/tests/agentCoreEvaluationProviderGate.test.ts new file mode 100644 index 0000000..287b8fb --- /dev/null +++ b/providers/aws/app/api/tests/agentCoreEvaluationProviderGate.test.ts @@ -0,0 +1,507 @@ +import assert from "node:assert/strict"; +import test from "node:test"; + +import { + assertProviderParityGate, + buildProviderParityReport, + sanitizeProviderResult, + type ProviderEvaluationPair +} from "../src/evals/agentCoreEvaluationProviderGate.js"; +import { ProviderParityError } from "../src/evals/agentCoreEvaluationProviderTypes.js"; +import type { + ProviderEvaluationRequest, + ProviderEvaluationResponse, + ProviderEvaluatorId, + ProviderParityPolicy +} from "../src/evals/agentCoreEvaluationProviderTypes.js"; +import type { EvaluationConvention } from "../src/evals/agentEvaluationTelemetryTypes.js"; + +const POLICY: ProviderParityPolicy = { + contractVersion: "1.0", + profileId: "provider-parity-v1", + scenarioId: "synthetic-cited-answer", + evaluatorThresholds: { + "Builtin.Correctness": 0.70, + "Builtin.ToolSelectionAccuracy": 0.70, + "Builtin.GoalSuccessRate": 0.70 + }, + maximumParityDelta: 0.20, + maximumProviderCalls: 6 +}; + +const EVALUATORS: ProviderEvaluatorId[] = [ + "Builtin.Correctness", + "Builtin.ToolSelectionAccuracy", + "Builtin.GoalSuccessRate" +]; + +test("builds metadata-only provider-direct evidence from six validated provider results", () => { + const report = buildProviderParityReport({ + pairs: [ + ...makeConventionPairs("otel-genai", [0.90, 0.85, 0.80]), + ...makeConventionPairs("openinference", [0.88, 0.82, 0.78]) + ], + policy: POLICY, + generatedAt: "2026-08-29T00:00:00.000Z", + sourceCommit: "a".repeat(40), + githubRunId: "123456", + durationBucket: "under-1m" + }); + + assert.equal(report.evidenceLevel, "provider-direct"); + assert.equal(report.providerCallCount, 6); + assert.equal(report.results.length, 6); + assert.equal(report.parity.length, 3); + assert.equal(report.status, "passed"); + assert.doesNotThrow(() => assertProviderParityGate(report)); + const serialized = JSON.stringify(report); + assert.equal(serialized.includes("provider explanation"), false); + assert.equal(serialized.includes("arn:aws"), false); + assert.equal(serialized.includes("Which controls"), false); +}); + +test("rejects malformed provider results with bounded codes and no diagnostic leakage", () => { + const cases: Array<{ + name: string; + mutate: (response: ProviderEvaluationResponse) => void; + code: ProviderParityError["code"]; + }> = [ + { + name: "missing results", + mutate: (response) => { delete response.evaluationResults; }, + code: "provider_result_missing" + }, + { + name: "empty results", + mutate: (response) => { response.evaluationResults = []; }, + code: "provider_result_missing" + }, + { + name: "duplicate results", + mutate: (response) => { response.evaluationResults!.push(structuredClone(response.evaluationResults![0]!)); }, + code: "provider_result_duplicate" + }, + { + name: "unexpected evaluator", + mutate: (response) => { response.evaluationResults![0]!.evaluatorId = "Builtin.Other"; }, + code: "provider_evaluator_unexpected" + }, + { + name: "provider error code", + mutate: (response) => { response.evaluationResults![0]!.errorCode = "provider diagnostic"; }, + code: "provider_result_failed" + }, + { + name: "ignored reference input", + mutate: (response) => { response.evaluationResults![0]!.ignoredReferenceInputFields = ["expectedResponse"]; }, + code: "provider_reference_input_ignored" + }, + { + name: "missing score", + mutate: (response) => { delete response.evaluationResults![0]!.value; }, + code: "provider_score_invalid" + }, + { + name: "NaN score", + mutate: (response) => { response.evaluationResults![0]!.value = Number.NaN; }, + code: "provider_score_invalid" + }, + { + name: "infinite score", + mutate: (response) => { response.evaluationResults![0]!.value = Number.POSITIVE_INFINITY; }, + code: "provider_score_invalid" + }, + { + name: "out of range score", + mutate: (response) => { response.evaluationResults![0]!.value = 1.01; }, + code: "provider_score_invalid" + }, + { + name: "missing context", + mutate: (response) => { delete response.evaluationResults![0]!.context; }, + code: "provider_context_mismatch" + }, + { + name: "wrong context", + mutate: (response) => { response.evaluationResults![0]!.context!.spanContext!.sessionId = "wrong"; }, + code: "provider_context_mismatch" + }, + { + name: "negative token count", + mutate: (response) => { response.evaluationResults![0]!.tokenUsage!.inputTokens = -1; }, + code: "provider_token_usage_invalid" + }, + { + name: "fractional token count", + mutate: (response) => { response.evaluationResults![0]!.tokenUsage!.outputTokens = 0.5; }, + code: "provider_token_usage_invalid" + }, + { + name: "empty label", + mutate: (response) => { response.evaluationResults![0]!.label = ""; }, + code: "provider_label_invalid" + }, + { + name: "overlong label", + mutate: (response) => { response.evaluationResults![0]!.label = "a".repeat(81); }, + code: "provider_label_invalid" + }, + { + name: "non-printable label", + mutate: (response) => { response.evaluationResults![0]!.label = "not\nprintable"; }, + code: "provider_label_invalid" + } + ]; + + for (const { name, mutate, code } of cases) { + const pair = makePair("otel-genai", "Builtin.Correctness", 0.90); + mutate(pair.response); + assertProviderError(() => sanitizeProviderResult(pair, POLICY), code, name); + } +}); + +test("fails closed for below-threshold scores, parity deltas, and incomplete result coverage", () => { + assertProviderError( + () => sanitizeProviderResult(makePair("otel-genai", "Builtin.Correctness", 0.69), POLICY), + "provider_score_below_threshold" + ); + + assertProviderError(() => buildProviderParityReport({ + pairs: [ + ...makeConventionPairs("otel-genai", [0.90, 0.85, 0.80]), + ...makeConventionPairs("openinference", [0.60, 0.82, 0.78]) + ], + policy: POLICY, + generatedAt: "2026-08-29T00:00:00.000Z", + sourceCommit: "a".repeat(40), + githubRunId: "123456", + durationBucket: "under-1m" + }), "provider_score_below_threshold"); + + const parityMismatch = buildProviderParityReport({ + pairs: [ + ...makeConventionPairs("otel-genai", [0.99, 0.85, 0.80]), + ...makeConventionPairs("openinference", [0.78, 0.82, 0.78]) + ], + policy: POLICY, + generatedAt: "2026-08-29T00:00:00.000Z", + sourceCommit: "a".repeat(40), + githubRunId: "123456", + durationBucket: "under-1m" + }); + assert.equal(parityMismatch.status, "failed"); + assertProviderError(() => assertProviderParityGate(parityMismatch), "provider_parity_delta_exceeded"); + + assertProviderError(() => buildProviderParityReport({ + pairs: [ + ...makeConventionPairs("otel-genai", [0.90, 0.85, 0.80]), + makePair("openinference", "Builtin.Correctness", 0.88), + makePair("openinference", "Builtin.ToolSelectionAccuracy", 0.82), + makePair("otel-genai", "Builtin.Correctness", 0.90) + ], + policy: POLICY, + generatedAt: "2026-08-29T00:00:00.000Z", + sourceCommit: "a".repeat(40), + githubRunId: "123456", + durationBucket: "under-1m" + }), "provider_result_coverage_invalid"); + + assertProviderError(() => buildProviderParityReport({ + pairs: [ + ...makeConventionPairs("otel-genai", [0.90, 0.85, 0.80]), + ...makeConventionPairs("openinference", [0.88, 0.82, 0.78]), + makePair("otel-genai", "Builtin.Correctness", 0.90) + ], + policy: POLICY, + generatedAt: "2026-08-29T00:00:00.000Z", + sourceCommit: "a".repeat(40), + githubRunId: "123456", + durationBucket: "under-1m" + }), "provider_call_count_invalid"); +}); + +test("revalidates report rows instead of trusting passed fields or status", () => { + const report = buildPassingReport(); + report.results[0]!.passed = false; + assertProviderError(() => assertProviderParityGate(report), "provider_score_below_threshold"); + + const malformed = buildPassingReport(); + malformed.parity[0]!.absoluteDelta = 0; + assertProviderError(() => assertProviderParityGate(malformed), "provider_parity_delta_exceeded"); + + const wrongCount = buildPassingReport(); + (wrongCount as { providerCallCount: number }).providerCallCount = 5; + assertProviderError(() => assertProviderParityGate(wrongCount), "provider_call_count_invalid"); +}); + +test("maps all printable provider labels to a code-owned evidence label", () => { + const ordinary = makePair("otel-genai", "Builtin.Correctness", 0.90); + ordinary.response.evaluationResults![0]!.label = "Passed with citation"; + const sensitive = makePair("openinference", "Builtin.Correctness", 0.90); + sensitive.response.evaluationResults![0]!.label = "123456789012"; + + const ordinaryResult = sanitizeProviderResult(ordinary, POLICY); + const sensitiveResult = sanitizeProviderResult(sensitive, POLICY); + assert.equal(ordinaryResult.label, "provider_result"); + assert.equal(sensitiveResult.label, "provider_result"); + const serialized = JSON.stringify([ordinaryResult, sensitiveResult]); + assert.equal(serialized.includes("Passed with citation"), false); + assert.equal(serialized.includes("123456789012"), false); +}); + +test("rejects malformed top-level evidence and unknown fields", () => { + const cases: Array<{ + name: string; + mutate: (report: ReturnType & Record) => void; + code: ProviderParityError["code"]; + }> = [ + { + name: "contract version", + mutate: (report) => { report.contractVersion = "2.0" as never; }, + code: "provider_policy_invalid" + }, + { + name: "threshold version", + mutate: (report) => { report.thresholdVersion = "2.0" as never; }, + code: "provider_policy_invalid" + }, + { + name: "evidence level", + mutate: (report) => { report.evidenceLevel = "local" as never; }, + code: "provider_policy_invalid" + }, + { + name: "invalid timestamp", + mutate: (report) => { report.generatedAt = "not-a-timestamp"; }, + code: "provider_result_coverage_invalid" + }, + { + name: "invalid source commit", + mutate: (report) => { report.sourceCommit = "a".repeat(39); }, + code: "provider_source_commit_invalid" + }, + { + name: "invalid GitHub run ID", + mutate: (report) => { report.githubRunId = "run-123"; }, + code: "provider_result_coverage_invalid" + }, + { + name: "region", + mutate: (report) => { report.regionLabel = "us-east-1" as never; }, + code: "provider_region_invalid" + }, + { + name: "scenario", + mutate: (report) => { report.scenarioId = "other" as never; }, + code: "provider_policy_invalid" + }, + { + name: "duration bucket", + mutate: (report) => { report.durationBucket = "forever" as never; }, + code: "provider_result_coverage_invalid" + }, + { + name: "unknown provider response field", + mutate: (report) => { report.providerResponse = { explanation: "do not retain" }; }, + code: "provider_result_coverage_invalid" + } + ]; + + for (const { name, mutate, code } of cases) { + const report = buildPassingReport() as ReturnType & Record; + mutate(report); + assertProviderError(() => assertProviderParityGate(report), code, name); + } +}); + +test("rejects malformed aggregate, result, and parity data without trusting stored flags", () => { + const aggregateUnknown = buildPassingReport() as ReturnType & { + aggregateTokenUsage: Record; + }; + aggregateUnknown.aggregateTokenUsage.unexpected = "secret"; + assertProviderError(() => assertProviderParityGate(aggregateUnknown), "provider_token_usage_invalid"); + + const aggregateMismatch = buildPassingReport(); + aggregateMismatch.aggregateTokenUsage.totalTokens += 1; + assertProviderError(() => assertProviderParityGate(aggregateMismatch), "provider_token_usage_invalid"); + + const badResultScore = buildPassingReport(); + badResultScore.results[0]!.score = Number.NaN; + assertProviderError(() => assertProviderParityGate(badResultScore), "provider_score_invalid"); + + const unknownParity = buildPassingReport(); + unknownParity.parity[0]!.evaluatorId = "Builtin.Unknown" as never; + assertProviderError(() => assertProviderParityGate(unknownParity), "provider_result_coverage_invalid"); + + const duplicateParity = buildPassingReport(); + duplicateParity.parity[1]!.evaluatorId = duplicateParity.parity[0]!.evaluatorId; + assertProviderError(() => assertProviderParityGate(duplicateParity), "provider_result_coverage_invalid"); + + const badParityScore = buildPassingReport(); + badParityScore.parity[0]!.otelGenaiScore = Number.NaN; + assertProviderError(() => assertProviderParityGate(badParityScore), "provider_parity_delta_exceeded"); + + const parityPassedMismatch = buildPassingReport(); + parityPassedMismatch.parity[0]!.passed = false; + assertProviderError(() => assertProviderParityGate(parityPassedMismatch), "provider_parity_delta_exceeded"); + + const statusMismatch = buildPassingReport(); + statusMismatch.status = "failed"; + assertProviderError(() => assertProviderParityGate(statusMismatch), "provider_parity_delta_exceeded"); +}); + +test("uses a bounded error for an invalid report-builder input", () => { + assertProviderError( + () => buildProviderParityReport(null as never), + "provider_policy_invalid" + ); +}); + +test("rejects malformed provider pair records with bounded errors", () => { + const nullPair = makeConventionPairs("otel-genai", [0.90, 0.85, 0.80]); + nullPair[0] = null as never; + assertProviderError(() => buildProviderParityReport({ + pairs: [...nullPair, ...makeConventionPairs("openinference", [0.88, 0.82, 0.78])], + policy: POLICY, + generatedAt: "2026-08-29T00:00:00.000Z", + sourceCommit: "a".repeat(40), + githubRunId: "123456", + durationBucket: "under-1m" + }), "provider_result_coverage_invalid"); + + const malformedTarget = makeConventionPairs("otel-genai", [0.90, 0.85, 0.80]); + malformedTarget[0]!.request.evaluationTarget = { traceIds: "not-an-array" } as never; + assertProviderError(() => buildProviderParityReport({ + pairs: [...malformedTarget, ...makeConventionPairs("openinference", [0.88, 0.82, 0.78])], + policy: POLICY, + generatedAt: "2026-08-29T00:00:00.000Z", + sourceCommit: "a".repeat(40), + githubRunId: "123456", + durationBucket: "under-1m" + }), "provider_context_mismatch"); +}); + +test("derives tool result context from the targeted session span without reference input", () => { + const pair = makePair("otel-genai", "Builtin.ToolSelectionAccuracy", 0.90); + assert.equal(pair.request.evaluationReferenceInputs, undefined); + assert.doesNotThrow(() => sanitizeProviderResult(pair, POLICY)); + + pair.response.evaluationResults![0]!.context!.spanContext!.traceId = "wrong-trace"; + assertProviderError(() => sanitizeProviderResult(pair, POLICY), "provider_context_mismatch"); +}); + +test("rejects evaluator reference fields outside the fixed provider profile", () => { + const tool = makePair("otel-genai", "Builtin.ToolSelectionAccuracy", 0.90); + tool.request.evaluationReferenceInputs = [{ + context: { spanContext: { sessionId: "synthetic-session", traceId: "trace-id", spanId: "tool-span-id" } } + }]; + assertProviderError(() => sanitizeProviderResult(tool, POLICY), "provider_context_mismatch"); + + const correctness = makePair("otel-genai", "Builtin.Correctness", 0.90); + delete correctness.request.evaluationReferenceInputs; + assertProviderError(() => sanitizeProviderResult(correctness, POLICY), "provider_context_mismatch"); + + const goal = makePair("otel-genai", "Builtin.GoalSuccessRate", 0.90); + goal.request.evaluationReferenceInputs![0]!.expectedResponse = { text: "unsupported for fixed goal evaluator" }; + assertProviderError(() => sanitizeProviderResult(goal, POLICY), "provider_context_mismatch"); +}); + +function buildPassingReport() { + return buildProviderParityReport({ + pairs: [ + ...makeConventionPairs("otel-genai", [0.90, 0.85, 0.80]), + ...makeConventionPairs("openinference", [0.88, 0.82, 0.78]) + ], + policy: POLICY, + generatedAt: "2026-08-29T00:00:00.000Z", + sourceCommit: "a".repeat(40), + githubRunId: "123456", + durationBucket: "under-1m" + }); +} + +function makeConventionPairs(convention: EvaluationConvention, scores: number[]): ProviderEvaluationPair[] { + return EVALUATORS.map((evaluatorId, index) => makePair(convention, evaluatorId, scores[index]!)); +} + +function makePair( + convention: EvaluationConvention, + evaluatorId: ProviderEvaluatorId, + score: number +): ProviderEvaluationPair { + const request: ProviderEvaluationRequest = { + evaluatorId, + evaluationInput: { + sessionSpans: [ + { + traceId: "trace-id", + spanId: "agent-span-id", + attributes: { "session.id": "synthetic-session" } + }, + { + traceId: "trace-id", + spanId: "tool-span-id", + attributes: { "session.id": "synthetic-session" } + } + ] + }, + ...(evaluatorId === "Builtin.Correctness" + ? { evaluationTarget: { traceIds: ["trace-id"] } } + : evaluatorId === "Builtin.ToolSelectionAccuracy" + ? { evaluationTarget: { spanIds: ["tool-span-id"] } } + : {}), + ...(evaluatorId === "Builtin.ToolSelectionAccuracy" + ? {} + : { + evaluationReferenceInputs: [{ + context: { + spanContext: { + sessionId: "synthetic-session", + ...(evaluatorId === "Builtin.Correctness" ? { traceId: "trace-id" } : {}) + } + }, + ...(evaluatorId === "Builtin.Correctness" + ? { expectedResponse: { text: "expected synthetic response" } } + : { assertions: [{ text: "expected synthetic outcome" }] }) + }] + }) + }; + const rawProviderResult = { + evaluatorId, + value: score, + label: "pass", + explanation: "provider explanation: Which controls were selected", + errorMessage: "provider diagnostic: do not expose", + ignoredReferenceInputFields: [], + context: { + spanContext: { + sessionId: "synthetic-session", + ...(evaluatorId === "Builtin.GoalSuccessRate" ? {} : { traceId: "trace-id" }), + ...(evaluatorId === "Builtin.ToolSelectionAccuracy" ? { spanId: "tool-span-id" } : {}) + } + }, + tokenUsage: { inputTokens: 11, outputTokens: 7, totalTokens: 18 }, + evaluatorArn: "arn:aws:bedrock-agentcore:ap-southeast-2:123:evaluator/example" + }; + return { + convention, + request, + response: { + evaluationResults: [rawProviderResult] as unknown as NonNullable + } + }; +} + +function assertProviderError( + operation: () => unknown, + code: ProviderParityError["code"], + label?: string +): void { + assert.throws(operation, (error: unknown) => { + assert.ok(error instanceof ProviderParityError, label); + assert.equal(error.code, code, label); + assert.equal(error.message, code, label); + assert.equal(error.message.includes("provider diagnostic"), false, label); + return true; + }); +} diff --git a/providers/aws/app/api/tests/agentCoreEvaluationProviderWorkflow.test.ts b/providers/aws/app/api/tests/agentCoreEvaluationProviderWorkflow.test.ts new file mode 100644 index 0000000..1c54058 --- /dev/null +++ b/providers/aws/app/api/tests/agentCoreEvaluationProviderWorkflow.test.ts @@ -0,0 +1,58 @@ +import assert from "node:assert/strict"; +import { readFile } from "node:fs/promises"; +import { resolve } from "node:path"; +import test from "node:test"; + +const ROOT = resolve(process.cwd(), "../../../.."); +const WORKFLOW_PATH = resolve(ROOT, ".github/workflows/agentcore-evaluation-provider-parity.yml"); +const CI_PATH = resolve(ROOT, ".github/workflows/ci.yml"); + +test("manual provider-parity workflow keeps ordinary CI cloud-free and protects direct spans", async () => { + const [source, ciSource] = await Promise.all([ + readFile(WORKFLOW_PATH, "utf8"), + readFile(CI_PATH, "utf8") + ]); + const validateJob = job(source, "validate", "direct-spans"); + const directJob = job(source, "direct-spans"); + + assert.match(source, /workflow_dispatch:/); + assert.doesNotMatch(source, /\n\s+(pull_request|push|schedule|workflow_call):/); + assert.match(source, /mode:\n\s+type:\s*choice\n\s+options:\s*\[validate, direct-spans\]\n\s+default:\s*validate/); + assert.match(source, /confirmation:\n\s+type:\s*string\n\s+required:\s*false/); + assert.match(source, /permissions:\n\s+contents:\s*read/); + assert.match(source, /group:\s*cloudai-agentcore-evaluation-provider-parity/); + assert.match(source, /cancel-in-progress:\s*false/); + assert.match(validateJob, /--mode validate/); + assert.doesNotMatch(validateJob, /environment:|id-token:\s*write|configure-aws-credentials|upload-artifact/); + assert.match(directJob, /environment:\s*aws-sandbox/); + assert.match(directJob, /id-token:\s*write/); + assert.match(directJob, /I_UNDERSTAND_AGENTCORE_EVALUATION_PROVIDER_PARITY/); + assert.match(directJob, /AGENTCORE_EVALUATION_MAX_CALLS/); + assert.match(directJob, /retention-days:\s*7/); + assert.doesNotMatch(source, /strategy:|matrix:|continue-on-error:\s*true/); + + assert.match(directJob, /AGENTCORE_EVALUATION_READY/); + assert.match(directJob, /AWS_AGENTCORE_EVALUATION_ROLE_TO_ASSUME/); + assert.match(directJob, /PROVIDER_PARITY_MODE:\s*direct-spans/); + assert.match(directJob, /GITHUB_REF.*refs\/heads\/main/s); + assert.match(directJob, /GITHUB_SHA.*\^\[0-9a-f\]\{40\}\$/s); + assert.match(directJob, /role-to-assume:\s*\$\{\{ env\.AWS_AGENTCORE_EVALUATION_ROLE_TO_ASSUME \}\}/); + assert.match(directJob, /mask-aws-account-id:\s*true/); + assert.match(directJob, /if-no-files-found:\s*error/); + assert.ok( + directJob.indexOf("Preflight protected direct evaluation") < directJob.indexOf("configure-aws-credentials"), + "the preflight must run before AWS credentials are configured" + ); + + const requiredApiJob = job(ciSource, "mock-genai-api", "agentcore-rag-runtime"); + assert.match(requiredApiJob, /pnpm test/); + assert.doesNotMatch(ciSource, /configure-aws-credentials|direct-spans/); +}); + +function job(source: string, name: string, nextName?: string): string { + const start = source.indexOf(` ${name}:`); + assert.notEqual(start, -1, `missing ${name} job`); + const end = nextName === undefined ? source.length : source.indexOf(` ${nextName}:`, start + 1); + assert.notEqual(end, -1, `missing ${nextName} job`); + return source.slice(start, end === -1 ? source.length : end); +} diff --git a/providers/aws/app/api/tests/agentCoreEvaluationRequestBuilder.test.ts b/providers/aws/app/api/tests/agentCoreEvaluationRequestBuilder.test.ts new file mode 100644 index 0000000..ca8d53e --- /dev/null +++ b/providers/aws/app/api/tests/agentCoreEvaluationRequestBuilder.test.ts @@ -0,0 +1,298 @@ +import assert from "node:assert/strict"; +import { readFile } from "node:fs/promises"; +import { resolve } from "node:path"; +import test from "node:test"; + +import { + buildProviderEvaluationRequests, + deriveProviderIds +} from "../src/evals/agentCoreEvaluationRequestBuilder.js"; +import { ProviderParityError } from "../src/evals/agentCoreEvaluationProviderTypes.js"; +import type { + ProviderEvaluationRequest, + ProviderParityPolicy +} from "../src/evals/agentCoreEvaluationProviderTypes.js"; +import type { + EvaluationScenario, + TelemetryFixture +} from "../src/evals/agentEvaluationTelemetryTypes.js"; + +const EXAMPLE_DIR = resolve(process.cwd(), "../../../../shared/examples/agent-evaluation-telemetry"); + +test("builds equivalent deterministic direct-span requests for both cited-answer conventions", async () => { + const { otel, openInference, scenario, policy } = await loadInputs(); + const otelRequests = buildProviderEvaluationRequests(otel, scenario, policy); + const openInferenceRequests = buildProviderEvaluationRequests(openInference, scenario, policy); + + assert.equal(otelRequests.length, 3); + assert.equal(openInferenceRequests.length, 3); + assert.deepEqual(otelRequests.map((request) => request.evaluatorId), [ + "Builtin.Correctness", + "Builtin.ToolSelectionAccuracy", + "Builtin.GoalSuccessRate" + ]); + assert.deepEqual(summarizeSemantics(otelRequests), summarizeSemantics(openInferenceRequests)); + assert.deepEqual(extractProviderVisibleSemantics(otelRequests[0]!), { + prompt: "Which controls protect the synthetic platform?", + finalResponse: "Governed access protects the synthetic platform [source:platform-handbook].", + toolName: "knowledge_search", + toolArguments: "{\"source\":\"platform-handbook\"}", + toolResult: "{\"status\":\"succeeded\",\"citation\":\"source:platform-handbook\"}" + }); + assert.deepEqual( + extractProviderVisibleSemantics(otelRequests[0]!), + extractProviderVisibleSemantics(openInferenceRequests[0]!) + ); + + const otelAgent = (otelRequests[0]!.evaluationInput.sessionSpans as ProviderSpan[]) + .find((span) => span.name === "invoke-agent")!; + assert.equal(otelAgent.attributes["gen_ai.task.input"], "Which controls protect the synthetic platform?"); + assert.equal( + otelAgent.attributes["gen_ai.task.output"], + "Governed access protects the synthetic platform [source:platform-handbook]." + ); + assert.equal("gen_ai.input.messages" in otelAgent.attributes, false); + assert.equal("gen_ai.output.messages" in otelAgent.attributes, false); + + for (const [fixture, requests] of [[otel, otelRequests], [openInference, openInferenceRequests]] as const) { + const spans = requests[0]!.evaluationInput.sessionSpans as ProviderSpan[]; + const ids = deriveProviderIds(fixture.scenarioId, fixture.convention, fixture.spans.map((span) => span.spanId)); + assert.match(ids.traceId, /^[0-9a-f]{32}$/); + assert.deepEqual(Object.values(ids.spanIds).sort(), [...new Set(Object.values(ids.spanIds))].sort()); + + const spanIds = new Set(spans.map((span) => span.spanId)); + for (const [index, span] of spans.entries()) { + const source = fixture.spans[index]!; + assert.match(span.traceId, /^[0-9a-f]{32}$/); + assert.match(span.spanId, /^[0-9a-f]{16}$/); + if (span.parentSpanId) assert.ok(spanIds.has(span.parentSpanId)); + assert.equal(span.attributes["session.id"], fixture.spans[0]!.attributes["session.id"]); + assert.ok(span.scope.name.startsWith(fixture.convention === "otel-genai" + ? "opentelemetry.instrumentation." + : "openinference.instrumentation.")); + assert.equal(span.scope.version, "1.0.0"); + assert.deepEqual(span.resource, { + attributes: { + "service.name": "cloudai-provider-parity-synthetic", + "cloudai.data.scope": "synthetic-only" + } + }); + assert.deepEqual(span.status, { code: 1 }); + assert.equal(span.endTimeUnixNano, (BigInt(source.startTimeUnixNano) + 1n).toString()); + } + } +}); + +test("uses evaluator-specific direct-span targets and reference contexts", async () => { + const { otel, scenario, policy } = await loadInputs(); + const [correctness, toolSelection, goalSuccess] = buildProviderEvaluationRequests(otel, scenario, policy); + const spans = correctness!.evaluationInput.sessionSpans as ProviderSpan[]; + const generatedTraceId = spans[0]!.traceId; + const generatedToolSpanId = spans.find((span) => span.name === "execute-tool")!.spanId; + const sessionId = otel.spans[0]!.attributes["session.id"] as string; + + assert.deepEqual(correctness!.evaluationTarget, { traceIds: [generatedTraceId] }); + assert.deepEqual(toolSelection!.evaluationTarget, { spanIds: [generatedToolSpanId] }); + assert.equal(goalSuccess!.evaluationTarget, undefined); + + const correctnessReference = correctness!.evaluationReferenceInputs![0]!; + const goalReference = goalSuccess!.evaluationReferenceInputs![0]!; + assert.deepEqual(correctnessReference.expectedResponse, { text: scenario.expectedResponse }); + assert.equal(toolSelection!.evaluationReferenceInputs, undefined); + assert.deepEqual(goalReference.assertions, [ + { text: "The final answer cites the approved synthetic source." } + ]); + assert.deepEqual(correctnessReference.context, { spanContext: { sessionId, traceId: generatedTraceId } }); + assert.deepEqual(goalReference.context, { spanContext: { sessionId } }); + assert.deepEqual(Object.keys(correctnessReference).sort(), ["context", "expectedResponse"]); + assert.deepEqual(Object.keys(goalReference).sort(), ["assertions", "context"]); +}); + +test("rejects malformed or ambiguous reviewed GenAI agent messages", async () => { + const { otel, scenario, policy } = await loadInputs(); + + const malformed = structuredClone(otel); + malformed.spans[0]!.attributes["gen_ai.input.messages"] = "not-json"; + assertBuilderError( + () => buildProviderEvaluationRequests(malformed, scenario, policy), + "provider_attribute_not_allowed" + ); + + const ambiguous = structuredClone(otel); + ambiguous.spans[0]!.attributes["gen_ai.output.messages"] = JSON.stringify([ + { role: "assistant", parts: [{ type: "text", content: "first" }] }, + { role: "assistant", parts: [{ type: "text", content: "second" }] } + ]); + assertBuilderError( + () => buildProviderEvaluationRequests(ambiguous, scenario, policy), + "provider_attribute_not_allowed" + ); + + const nonText = structuredClone(otel); + nonText.spans[0]!.attributes["gen_ai.output.messages"] = JSON.stringify([ + { role: "assistant", parts: [{ type: "tool_call", content: "not-final-text" }] } + ]); + assertBuilderError( + () => buildProviderEvaluationRequests(nonText, scenario, policy), + "provider_attribute_not_allowed" + ); +}); + +test("rejects inputs that are outside the reviewed provider request boundary", async () => { + const { otel, scenario, policy } = await loadInputs(); + + const nonSyntheticFixture = structuredClone(otel); + nonSyntheticFixture.scenarioId = "synthetic-citation-missing"; + assertBuilderError(() => buildProviderEvaluationRequests(nonSyntheticFixture, scenario, policy), "provider_fixture_not_synthetic"); + + const wrongScenario = structuredClone(scenario); + wrongScenario.scenarioId = "synthetic-citation-missing"; + assertBuilderError(() => buildProviderEvaluationRequests(otel, wrongScenario, policy), "provider_scenario_not_allowed"); + + const unknownScopeFixture = structuredClone(otel); + unknownScopeFixture.spans[0]!.scopeName = "custom.agent.tracing"; + assertBuilderError(() => buildProviderEvaluationRequests(unknownScopeFixture, scenario, policy), "provider_scope_not_allowed"); + + const missingAgentSpan = structuredClone(otel); + missingAgentSpan.spans = missingAgentSpan.spans.filter((span) => span.attributes["gen_ai.operation.name"] !== "invoke_agent"); + assertBuilderError(() => buildProviderEvaluationRequests(missingAgentSpan, scenario, policy), "provider_required_span_missing"); + + const secondSessionFixture = structuredClone(otel); + secondSessionFixture.spans[1]!.attributes["session.id"] = "second-reviewed-session"; + assertBuilderError(() => buildProviderEvaluationRequests(secondSessionFixture, scenario, policy), "provider_session_count_invalid"); + + const unknownAttributeFixture = structuredClone(otel); + unknownAttributeFixture.spans[0]!.attributes["customer.account.id"] = "not-allowed"; + assertBuilderError(() => buildProviderEvaluationRequests(unknownAttributeFixture, scenario, policy), "provider_attribute_not_allowed"); +}); + +test("rejects missing reviewed GenAI and OpenInference message and tool attributes", async () => { + const { otel, openInference, scenario, policy } = await loadInputs(); + const missingOtelMessage = structuredClone(otel); + delete missingOtelMessage.spans[0]!.attributes["gen_ai.input.messages"]; + assertBuilderError(() => buildProviderEvaluationRequests( + missingOtelMessage, scenario, policy), "provider_attribute_not_allowed"); + + const missingOtelTool = structuredClone(otel); + delete missingOtelTool.spans[2]!.attributes["gen_ai.tool.call.result"]; + assertBuilderError(() => buildProviderEvaluationRequests( + missingOtelTool, scenario, policy), "provider_attribute_not_allowed"); + + const missingOpenInferenceMessage = structuredClone(openInference); + delete missingOpenInferenceMessage.spans[0]!.attributes["output.value"]; + assertBuilderError(() => buildProviderEvaluationRequests( + missingOpenInferenceMessage, scenario, policy), "provider_attribute_not_allowed"); + + const missingOpenInferenceTool = structuredClone(openInference); + delete missingOpenInferenceTool.spans[2]!.attributes["tool.name"]; + assertBuilderError(() => buildProviderEvaluationRequests( + missingOpenInferenceTool, scenario, policy), "provider_attribute_not_allowed"); +}); + +test("rejects mutable policy and scenario values that would change the fixed request matrix", async () => { + const { otel, scenario, policy } = await loadInputs(); + const fourthEvaluatorPolicy = structuredClone(policy) as ProviderParityPolicy & { + evaluatorThresholds: Record; + }; + fourthEvaluatorPolicy.evaluatorThresholds["Builtin.NewEvaluator"] = 0.7; + assertBuilderError(() => buildProviderEvaluationRequests(otel, scenario, fourthEvaluatorPolicy), "provider_policy_invalid"); + + const changedThresholdPolicy = structuredClone(policy); + changedThresholdPolicy.evaluatorThresholds["Builtin.Correctness"] = 0.8; + assertBuilderError(() => buildProviderEvaluationRequests(otel, scenario, changedThresholdPolicy), "provider_policy_invalid"); + + const wrongCallCapPolicy = structuredClone(policy) as unknown as { maximumProviderCalls: number }; + wrongCallCapPolicy.maximumProviderCalls = 3; + assertBuilderError(() => buildProviderEvaluationRequests( + otel, scenario, wrongCallCapPolicy as ProviderParityPolicy), "provider_call_count_invalid"); + + const emptyExpectedResponse = structuredClone(scenario); + emptyExpectedResponse.expectedResponse = ""; + assertBuilderError(() => buildProviderEvaluationRequests(otel, emptyExpectedResponse, policy), "provider_scenario_not_allowed"); + + const multipleTools = structuredClone(scenario); + multipleTools.expectedToolTrajectory.push({ name: "another_tool", argumentsSubset: {} }); + assertBuilderError(() => buildProviderEvaluationRequests(otel, multipleTools, policy), "provider_scenario_not_allowed"); +}); + +type ProviderSpan = { + traceId: string; + spanId: string; + parentSpanId?: string; + name: string; + attributes: Record; + startTimeUnixNano: string; + endTimeUnixNano: string; + scope: { name: string; version: string }; + resource: { attributes: Record }; + status: { code: number }; +}; + +function summarizeSemantics(requests: ProviderEvaluationRequest[]) { + return requests.map((request) => ({ + evaluatorId: request.evaluatorId, + targetType: request.evaluationTarget && "traceIds" in request.evaluationTarget + ? "trace" + : request.evaluationTarget && "spanIds" in request.evaluationTarget + ? "tool" + : "session", + spanNames: (request.evaluationInput.sessionSpans as ProviderSpan[]).map((span) => span.name), + referenceFields: request.evaluationReferenceInputs + ? Object.keys(request.evaluationReferenceInputs[0]!).filter((key) => key !== "context").sort() + : [] + })); +} + +function extractProviderVisibleSemantics(request: ProviderEvaluationRequest) { + const spans = request.evaluationInput.sessionSpans as ProviderSpan[]; + const agent = spans.find((span) => span.name === "invoke-agent")!; + const tool = spans.find((span) => span.name === "execute-tool")!; + if ("gen_ai.operation.name" in agent.attributes) { + return { + prompt: agent.attributes["gen_ai.task.input"], + finalResponse: agent.attributes["gen_ai.task.output"], + toolName: tool.attributes["gen_ai.tool.name"], + toolArguments: tool.attributes["gen_ai.tool.call.arguments"], + toolResult: tool.attributes["gen_ai.tool.call.result"] + }; + } + return { + prompt: agent.attributes["input.value"], + finalResponse: agent.attributes["output.value"], + toolName: tool.attributes["tool.name"], + toolArguments: tool.attributes["input.value"], + toolResult: tool.attributes["output.value"] + }; +} + +async function loadInputs(): Promise<{ + otel: TelemetryFixture; + openInference: TelemetryFixture; + scenario: EvaluationScenario; + policy: ProviderParityPolicy; +}> { + const [otel, openInference, scenarios, policy] = await Promise.all([ + loadFixture("otel-genai.traces.v1.json"), + loadFixture("openinference.traces.v1.json"), + readJson("scenarios.v1.json"), + readJson("provider-parity-thresholds.v1.json") + ]); + const scenario = scenarios.find((candidate) => candidate.scenarioId === "synthetic-cited-answer"); + assert.ok(scenario); + return { otel, openInference, scenario, policy }; +} + +async function loadFixture(fileName: string): Promise { + const fixtures = await readJson(fileName); + const fixture = fixtures.find((candidate) => candidate.scenarioId === "synthetic-cited-answer"); + assert.ok(fixture); + return structuredClone(fixture); +} + +async function readJson(fileName: string): Promise { + return JSON.parse(await readFile(resolve(EXAMPLE_DIR, fileName), "utf8")) as T; +} + +function assertBuilderError(operation: () => unknown, code: ProviderParityError["code"]): void { + assert.throws(operation, (error: unknown) => error instanceof ProviderParityError && error.code === code); +} diff --git a/providers/aws/app/api/tests/agentEvaluationTelemetryDocumentation.test.ts b/providers/aws/app/api/tests/agentEvaluationTelemetryDocumentation.test.ts index 24bf689..42e3eb4 100644 --- a/providers/aws/app/api/tests/agentEvaluationTelemetryDocumentation.test.ts +++ b/providers/aws/app/api/tests/agentEvaluationTelemetryDocumentation.test.ts @@ -9,6 +9,8 @@ test("agent evaluation telemetry documentation records standards, scores, and ev const documentation = (await Promise.all([ "docs/solutions/agent-evaluation-telemetry-runbook.md", "docs/architecture/agentcore-governed-rag-poc.md", + "docs/solutions/p8i-agentcore-rag-key-process-record.md", + "docs/practices/current-status.md", "providers/aws/app/api/README.md" ].map((path) => readFile(resolve(ROOT, path), "utf8")))).join("\n"); @@ -21,12 +23,37 @@ test("agent evaluation telemetry documentation records standards, scores, and ev "local.tool_trajectory_accuracy", "locally contract-tested", "protected provider-parity lane", - "does not call AWS" + "does not call AWS", + "provider-parity-v1", + "provider-direct", + "I_UNDERSTAND_AGENTCORE_EVALUATION_PROVIDER_PARITY", + "AWS_AGENTCORE_EVALUATION_ROLE_TO_ASSUME", + "AGENTCORE_EVALUATION_MAX_CALLS=6", + "Builtin.ToolSelectionAccuracy", + "Stage B", + "Runtime-to-CloudWatch" ]) { assert.match(documentation, new RegExp(escapeRegExp(required), "i")); } }); +test("managed-score boundaries reserve admission and approval for deterministic controls", async () => { + const scoreBoundaryDocuments = [ + "docs/solutions/agent-evaluation-telemetry-runbook.md", + "docs/architecture/agentcore-governed-rag-poc.md", + "docs/solutions/p8i-agentcore-rag-key-process-record.md" + ]; + + for (const path of scoreBoundaryDocuments) { + const document = await readFile(resolve(ROOT, path), "utf8"); + assert.match( + document, + /Managed scores supplement\s+deterministic\s+controls[\s\S]{0,120}never authorize IAM,\s+admission or approval,\s+tool execution,\s+deployment,\s+remediation,\s+rollback,\s+or\s+deletion/i, + `${path} must reserve admission and approval for deterministic controls` + ); + } +}); + test("current status does not overstate the local agent evaluation evidence", async () => { const currentStatus = await readFile( resolve(ROOT, "docs/practices/current-status.md"), @@ -34,13 +61,16 @@ test("current status does not overstate the local agent evaluation evidence", as ); const telemetryRow = currentStatus .split("\n") - .find((line) => line.includes("Framework-neutral agent evaluation telemetry")); + .find((line) => line.startsWith("| Framework-neutral agent evaluation telemetry |")); assert.ok(telemetryRow, "current status must include the telemetry gate"); assert.doesNotMatch(telemetryRow, /live validated/i); assert.doesNotMatch(telemetryRow, /managed evaluation/i); + assert.doesNotMatch(telemetryRow, /provider validated/i); + assert.doesNotMatch(telemetryRow, /runtime validated/i); assert.doesNotMatch(telemetryRow, /production evaluation/i); - assert.match(telemetryRow, /locally contract-tested/i); + assert.match(telemetryRow, /source implemented/i); + assert.match(telemetryRow, /provider validation pending/i); }); function escapeRegExp(value: string): string { diff --git a/providers/aws/app/api/tests/runAgentCoreEvaluationProviderParity.test.ts b/providers/aws/app/api/tests/runAgentCoreEvaluationProviderParity.test.ts new file mode 100644 index 0000000..69aa9ae --- /dev/null +++ b/providers/aws/app/api/tests/runAgentCoreEvaluationProviderParity.test.ts @@ -0,0 +1,287 @@ +import assert from "node:assert/strict"; +import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { resolve } from "node:path"; +import test from "node:test"; + +import { + parseProviderParityArguments, + runProviderParityEvaluation, + validateProviderParityEnvironment, + type ProviderParityRunOptions +} from "../src/scripts/runAgentCoreEvaluationProviderParity.js"; +import { + ProviderParityError, + type AgentCoreEvaluateClient, + type ProviderEvaluationRequest, + type ProviderEvaluationResponse +} from "../src/evals/agentCoreEvaluationProviderTypes.js"; + +const ROOT = resolve(process.cwd(), "../../../.."); +const EXAMPLE_DIR = resolve(ROOT, "shared/examples/agent-evaluation-telemetry"); +const VALID_DIRECT_ENVIRONMENT: NodeJS.ProcessEnv = { + PROVIDER_PARITY_MODE: "direct-spans", + CONFIRMATION: "I_UNDERSTAND_AGENTCORE_EVALUATION_PROVIDER_PARITY", + AGENTCORE_EVALUATION_READY: "true", + AGENTCORE_EVALUATION_MAX_CALLS: "6", + AWS_REGION: "ap-southeast-2", + GITHUB_REF: "refs/heads/main", + GITHUB_SHA: "a".repeat(40) +}; + +test("rejects every invalid direct-mode preflight before constructing the client", async (t) => { + const temporaryDirectory = await mkdtemp(resolve(tmpdir(), "agentcore-provider-parity-preflight-")); + t.after(() => rm(temporaryDirectory, { recursive: true, force: true })); + const base = directOptions(resolve(temporaryDirectory, "report.json")); + const wrongPolicyPath = resolve(temporaryDirectory, "wrong-policy.json"); + const malformedScenarioPath = resolve(temporaryDirectory, "malformed-scenarios.json"); + const malformedFixturePath = resolve(temporaryDirectory, "malformed-fixtures.json"); + const policy = JSON.parse(await readFile(base.policyPath, "utf8")) as Record; + policy.maximumProviderCalls = 5; + await writeFile(wrongPolicyPath, JSON.stringify(policy), "utf8"); + const scenarios = JSON.parse(await readFile(base.scenarioPath, "utf8")) as Array>; + const matchingScenario = scenarios.find((scenario) => scenario.scenarioId === "synthetic-cited-answer")!; + delete matchingScenario.expectedToolTrajectory; + await writeFile(malformedScenarioPath, JSON.stringify(scenarios), "utf8"); + const fixtures = JSON.parse(await readFile(base.fixturePaths[0], "utf8")) as Array>; + const matchingFixture = fixtures.find((fixture) => fixture.scenarioId === "synthetic-cited-answer")!; + delete matchingFixture.spans; + await writeFile(malformedFixturePath, JSON.stringify(fixtures), "utf8"); + + const cases: Array<{ + name: string; + mutate: (options: ProviderParityRunOptions) => void; + code: ProviderParityError["code"]; + }> = [ + { name: "missing confirmation", mutate: (options) => { delete options.environment.CONFIRMATION; }, code: "provider_confirmation_required" }, + { name: "wrong confirmation", mutate: (options) => { options.environment.CONFIRMATION = "yes"; }, code: "provider_confirmation_required" }, + { name: "readiness is not exactly true", mutate: (options) => { options.environment.AGENTCORE_EVALUATION_READY = "TRUE"; }, code: "provider_readiness_required" }, + { name: "call cap is not exactly six", mutate: (options) => { options.environment.AGENTCORE_EVALUATION_MAX_CALLS = "7"; }, code: "provider_call_budget_invalid" }, + { name: "region is outside the review", mutate: (options) => { options.environment.AWS_REGION = "us-east-1"; }, code: "provider_region_invalid" }, + { name: "source ref is not main", mutate: (options) => { options.environment.GITHUB_REF = "refs/heads/feature"; }, code: "provider_source_ref_invalid" }, + { name: "source commit is malformed", mutate: (options) => { options.environment.GITHUB_SHA = "abc123"; }, code: "provider_source_commit_invalid" }, + { name: "output path is missing", mutate: (options) => { delete options.outputPath; }, code: "provider_output_path_required" }, + { name: "scenario file is missing", mutate: (options) => { options.scenarioPath = resolve(temporaryDirectory, "missing-scenarios.json"); }, code: "provider_input_file_invalid" }, + { name: "first fixture file is missing", mutate: (options) => { options.fixturePaths[0] = resolve(temporaryDirectory, "missing-fixture.json"); }, code: "provider_input_file_invalid" }, + { name: "policy file is missing", mutate: (options) => { options.policyPath = resolve(temporaryDirectory, "missing-policy.json"); }, code: "provider_input_file_invalid" }, + { name: "matching scenario has a malformed shape", mutate: (options) => { options.scenarioPath = malformedScenarioPath; }, code: "provider_input_file_invalid" }, + { name: "matching fixture has a malformed shape", mutate: (options) => { options.fixturePaths[0] = malformedFixturePath; }, code: "provider_input_file_invalid" }, + { name: "policy maximum is not six", mutate: (options) => { options.policyPath = wrongPolicyPath; }, code: "provider_call_count_invalid" } + ]; + + for (const { name, mutate, code } of cases) { + const options = structuredClone(base); + mutate(options); + let factoryCalls = 0; + await assert.rejects( + runProviderParityEvaluation(options, () => { + factoryCalls += 1; + return passingClient(); + }), + (error: unknown) => error instanceof ProviderParityError && error.code === code, + name + ); + assert.equal(factoryCalls, 0, `${name}: client factory was called`); + } +}); + +test("CLI parser accepts only exact grammar with non-option values", () => { + assert.deepEqual(parseProviderParityArguments(["--", "--mode", "validate"]), { mode: "validate" }); + + for (const { arguments_, code } of [ + { arguments_: ["--mode", "--", "validate"], code: "provider_policy_invalid" }, + { arguments_: ["--", "--", "--mode", "validate"], code: "provider_policy_invalid" }, + { arguments_: ["--", "--mode", "validate", "--"], code: "provider_policy_invalid" }, + { arguments_: ["--mode", "direct-spans", "--output", ""], code: "provider_output_path_required" }, + { arguments_: ["--mode", "direct-spans", "--output", "--"], code: "provider_output_path_required" }, + { arguments_: ["--mode", "direct-spans", "--output", "--mode"], code: "provider_output_path_required" }, + { arguments_: ["--mode", "direct-spans", "--output", "-relative.json"], code: "provider_output_path_required" }, + { arguments_: ["--mode", "--output"], code: "provider_policy_invalid" }, + { arguments_: ["--mode", "--"], code: "provider_policy_invalid" } + ]) { + assert.throws( + () => parseProviderParityArguments(arguments_), + (error: unknown) => error instanceof ProviderParityError && error.code === code + ); + } +}); + +test("validate mode runs six deterministic fake evaluations without protected-cloud fields or an artifact", async (t) => { + const temporaryDirectory = await mkdtemp(resolve(tmpdir(), "agentcore-provider-parity-validate-")); + t.after(() => rm(temporaryDirectory, { recursive: true, force: true })); + const outputPath = resolve(temporaryDirectory, "must-not-exist.json"); + const requests: ProviderEvaluationRequest[] = []; + let activeCalls = 0; + let maximumActiveCalls = 0; + const client: AgentCoreEvaluateClient = { + async evaluate(request) { + activeCalls += 1; + maximumActiveCalls = Math.max(maximumActiveCalls, activeCalls); + requests.push(request); + await new Promise((done) => setImmediate(done)); + activeCalls -= 1; + return passingResponse(request); + } + }; + let factoryCalls = 0; + + assert.deepEqual(validateProviderParityEnvironment({ GITHUB_SHA: "local" }, "validate"), { + mode: "validate", + sourceCommit: "local" + }); + const result = await runProviderParityEvaluation({ + ...baseOptions("validate", { GITHUB_SHA: "local" }), + outputPath + }, () => { + factoryCalls += 1; + return client; + }); + + assert.deepEqual(result, { mode: "validate", status: "passed", requestCount: 6 }); + assert.equal(factoryCalls, 1); + assert.equal(maximumActiveCalls, 1); + assert.deepEqual(requests.map((request) => request.evaluatorId), [ + "Builtin.Correctness", + "Builtin.ToolSelectionAccuracy", + "Builtin.GoalSuccessRate", + "Builtin.Correctness", + "Builtin.ToolSelectionAccuracy", + "Builtin.GoalSuccessRate" + ]); + await assert.rejects(readFile(outputPath), { code: "ENOENT" }); + assert.equal(JSON.stringify(result).includes("provider-direct"), false); +}); + +test("direct mode performs exactly six serial calls in convention and evaluator order", async (t) => { + const temporaryDirectory = await mkdtemp(resolve(tmpdir(), "agentcore-provider-parity-direct-")); + t.after(() => rm(temporaryDirectory, { recursive: true, force: true })); + const outputPath = resolve(temporaryDirectory, "report.json"); + const calls: Array<{ convention: string; evaluatorId: string }> = []; + let activeCalls = 0; + let maximumActiveCalls = 0; + let factoryCalls = 0; + + const result = await runProviderParityEvaluation(directOptions(outputPath), () => { + factoryCalls += 1; + return { + async evaluate(request) { + activeCalls += 1; + maximumActiveCalls = Math.max(maximumActiveCalls, activeCalls); + calls.push({ + convention: conventionFor(request), + evaluatorId: request.evaluatorId + }); + await new Promise((done) => setImmediate(done)); + activeCalls -= 1; + return passingResponse(request); + } + }; + }); + + assert.equal(factoryCalls, 1); + assert.equal(maximumActiveCalls, 1); + assert.deepEqual(calls, [ + { convention: "otel-genai", evaluatorId: "Builtin.Correctness" }, + { convention: "otel-genai", evaluatorId: "Builtin.ToolSelectionAccuracy" }, + { convention: "otel-genai", evaluatorId: "Builtin.GoalSuccessRate" }, + { convention: "openinference", evaluatorId: "Builtin.Correctness" }, + { convention: "openinference", evaluatorId: "Builtin.ToolSelectionAccuracy" }, + { convention: "openinference", evaluatorId: "Builtin.GoalSuccessRate" } + ]); + assert.equal(result.mode, "direct-spans"); + assert.equal(result.report.providerCallCount, 6); + assert.equal(result.report.evidenceLevel, "provider-direct"); + assert.deepEqual(JSON.parse(await readFile(outputPath, "utf8")), result.report); +}); + +test("sanitizes a provider exception on call four and makes a seventh call impossible", async (t) => { + const temporaryDirectory = await mkdtemp(resolve(tmpdir(), "agentcore-provider-parity-error-")); + t.after(() => rm(temporaryDirectory, { recursive: true, force: true })); + const outputPath = resolve(temporaryDirectory, "report.json"); + let callCount = 0; + const providerMessage = "sensitive provider response and account 123456789012"; + + await assert.rejects( + runProviderParityEvaluation(directOptions(outputPath), () => ({ + async evaluate(request) { + callCount += 1; + if (callCount === 4) throw new Error(providerMessage); + return passingResponse(request); + } + })), + (error: unknown) => { + assert.ok(error instanceof ProviderParityError); + assert.equal(error.code, "provider_request_failed"); + assert.equal(error.message, "provider_request_failed"); + assert.equal(String(error).includes(providerMessage), false); + return true; + } + ); + assert.equal(callCount, 4); + assert.ok(callCount < 7); + await assert.rejects(readFile(outputPath), { code: "ENOENT" }); +}); + +function baseOptions(mode: ProviderParityRunOptions["mode"], environment: NodeJS.ProcessEnv): ProviderParityRunOptions { + return { + mode, + scenarioPath: resolve(EXAMPLE_DIR, "scenarios.v1.json"), + fixturePaths: [ + resolve(EXAMPLE_DIR, "otel-genai.traces.v1.json"), + resolve(EXAMPLE_DIR, "openinference.traces.v1.json") + ], + policyPath: resolve(EXAMPLE_DIR, "provider-parity-thresholds.v1.json"), + generatedAt: "2026-08-29T00:00:00.000Z", + githubRunId: "123456", + environment + }; +} + +function directOptions(outputPath: string): ProviderParityRunOptions { + return { + ...baseOptions("direct-spans", structuredClone(VALID_DIRECT_ENVIRONMENT)), + outputPath + }; +} + +function passingClient(): AgentCoreEvaluateClient { + return { evaluate: async (request) => passingResponse(request) }; +} + +function passingResponse(request: ProviderEvaluationRequest): ProviderEvaluationResponse { + return { + evaluationResults: [{ + evaluatorId: request.evaluatorId, + value: 0.9, + label: "pass", + context: { spanContext: expectedResultContext(request) }, + tokenUsage: { inputTokens: 10, outputTokens: 2, totalTokens: 12 } + }] + }; +} + +function expectedResultContext(request: ProviderEvaluationRequest) { + const spans = request.evaluationInput.sessionSpans as Array<{ + traceId: string; + spanId: string; + attributes: Record; + }>; + const sessionId = spans[0]!.attributes["session.id"] as string; + if (request.evaluatorId === "Builtin.GoalSuccessRate") return { sessionId }; + if (request.evaluatorId === "Builtin.Correctness") { + return { sessionId, traceId: request.evaluationTarget && "traceIds" in request.evaluationTarget + ? request.evaluationTarget.traceIds[0]! + : "missing" }; + } + const spanId = request.evaluationTarget && "spanIds" in request.evaluationTarget + ? request.evaluationTarget.spanIds[0]! + : "missing"; + return { sessionId, traceId: spans.find((span) => span.spanId === spanId)!.traceId, spanId }; +} + +function conventionFor(request: ProviderEvaluationRequest): string { + const scope = request.evaluationInput.sessionSpans[0]?.scope as { name?: unknown } | undefined; + return typeof scope?.name === "string" && scope.name.startsWith("opentelemetry.") + ? "otel-genai" + : "openinference"; +} diff --git a/providers/aws/infra/bootstrap/README.md b/providers/aws/infra/bootstrap/README.md index bf81c76..d470ebf 100644 --- a/providers/aws/infra/bootstrap/README.md +++ b/providers/aws/infra/bootstrap/README.md @@ -9,6 +9,7 @@ The bootstrap layer exists because Terraform needs a backend before it can safel - S3 bucket for Terraform state. - DynamoDB table for Terraform state locking. - IAM role for GitHub Actions. +- Separate evaluate-only IAM role for AgentCore provider-parity evidence. - Separate IAM role for the protected private-EKS network foundation. - Separate IAM role for the VPC-connected CodeBuild runner Terraform state and lifecycle. - IAM policy scoped to the sandbox bootstrap and Terraform plan/apply needs. @@ -35,7 +36,7 @@ Replace `` only in your private local command or AWS console. Do not `AWS_ROLE_TO_ASSUME`. 3. Use the protected `update-aws-bootstrap` GitHub Actions workflow to create a CloudFormation change set, review it, then explicitly apply it. -4. Store the resulting Terraform role ARN as a GitHub environment variable or secret named `AWS_ROLE_TO_ASSUME` in the matching environment. The private-network role is emitted as `PrivateEKSNetworkRoleArn`; the separate runner-state role is emitted as `PrivateEKSRunnerRoleArn`. Both belong only in `aws-private-eks`, under different variable names. +4. Store the resulting Terraform role ARN as a GitHub environment variable or secret named `AWS_ROLE_TO_ASSUME` in the matching environment. The evaluate-only role is emitted as `AgentCoreEvaluationRoleArn` and belongs only in protected `aws-sandbox` as `AWS_AGENTCORE_EVALUATION_ROLE_TO_ASSUME`. The private-network role is emitted as `PrivateEKSNetworkRoleArn`; the separate runner-state role is emitted as `PrivateEKSRunnerRoleArn`. Both belong only in `aws-private-eks`, under different variable names. 5. Use the `aws-sandbox` GitHub environment for manual approval. 6. Run Terraform validate/plan first. 7. Run future apply, deploy, GitOps update, and teardown through GitHub Actions rather than laptop-local commands. @@ -76,6 +77,18 @@ CloudFormation template is the only supported place to evolve these Terraform execution permissions. It does not grant general IAM administration, model invocation, Knowledge Base reads, browser access, or autonomous write actions. +The AgentCore evaluation role is deliberately separate from the Terraform, +Runtime, and bootstrap execution identities. It trusts only the existing +account-level GitHub OIDC provider with the `sts.amazonaws.com` audience and +the protected `aws-sandbox` GitHub Environment subject. Its only data-plane +permission is `bedrock-agentcore:Evaluate`: it cannot create, update, or delete +evaluators; invoke an AgentCore Runtime; query CloudWatch or logs; pass an IAM +role; read S3; or call another service. `Resource: "*"` is the reviewed, +isolated exception because this project has no proven evaluator ARN scope for +the current data-plane action. The compensating controls are a fixed evaluator +allowlist, a six-call cap, and a protected, manually dispatched, main-only +evaluation workflow. + ## CI/CD-Only Bootstrap Updates Use `.github/workflows/update-aws-bootstrap.yml` for every update to the @@ -90,6 +103,7 @@ Configure these values in the protected `aws-sandbox` GitHub Environment: | `AWS_BOOTSTRAP_ROLE_TO_ASSUME` | Separate GitHub OIDC trust-root role that may update the existing bootstrap stack. | | `AWS_BOOTSTRAP_STACK_NAME` | Existing CloudFormation stack name; this workflow updates, never creates, the stack. | | `AWS_OIDC_PROVIDER_ARN` | Existing account-level GitHub Actions OIDC provider ARN. | +| `AWS_AGENTCORE_EVALUATION_ROLE_TO_ASSUME` | Dedicated evaluate-only role output copied into this protected Environment after the separately confirmed bootstrap apply. | | `TF_BACKEND_BUCKET` | Existing Terraform state bucket name, passed back to the bootstrap stack as a private parameter. | | `TF_BACKEND_LOCK_TABLE` | Existing Terraform lock-table name, passed back to the bootstrap stack as a private parameter. | @@ -111,10 +125,19 @@ Execution order: 3. Run it with `mode=apply`, supply that exact reviewed change-set name, and enter `I_UNDERSTAND_AWS_BOOTSTRAP_APPLY`. GitHub Environment approval still applies. -4. Run `terraform-agentcore-rag-sandbox` with `mode=bootstrap-plan`. -5. Review the Terraform plan, then run `bootstrap-apply` with +4. After a successful apply, copy the masked `AgentCoreEvaluationRoleArn` + handoff into the protected `aws-sandbox` Environment setting + `AWS_AGENTCORE_EVALUATION_ROLE_TO_ASSUME`. The workflow fails this handoff + if the stack output is absent and does not print the ARN to normal logs. +5. Run `terraform-agentcore-rag-sandbox` with `mode=bootstrap-plan`. +6. Review the Terraform plan, then run `bootstrap-apply` with `I_UNDERSTAND_AGENTCORE_RAG_BOOTSTRAP_APPLY`. +Merging this source does not create the role or change any protected GitHub +Environment setting. The role is created only after the cloud-free validation, +change-set plan, human review, and separately confirmed apply sequence above; +the protected setting handoff remains a distinct manual action. + No AWS Console inline-policy attachment and no laptop-local AWS deployment is part of this path. diff --git a/providers/aws/infra/bootstrap/github-oidc-terraform-backend.yaml b/providers/aws/infra/bootstrap/github-oidc-terraform-backend.yaml index a8edda6..bb8d189 100644 --- a/providers/aws/infra/bootstrap/github-oidc-terraform-backend.yaml +++ b/providers/aws/infra/bootstrap/github-oidc-terraform-backend.yaml @@ -900,6 +900,39 @@ Resources: - Key: ManagedBy Value: cloudformation + GitHubActionsAgentCoreEvaluationRole: + Type: AWS::IAM::Role + DependsOn: GitHubActionsBootstrapRole + Properties: + RoleName: !Sub "${GitHubRepo}-${GitHubEnvironment}-agentcore-evaluation" + AssumeRolePolicyDocument: + Version: "2012-10-17" + Statement: + - Effect: Allow + Principal: + Federated: !Ref ExistingGitHubOidcProviderArn + Action: sts:AssumeRoleWithWebIdentity + Condition: + StringEquals: + token.actions.githubusercontent.com:aud: sts.amazonaws.com + StringLike: + token.actions.githubusercontent.com:sub: !Sub "repo:${GitHubOrg}/${GitHubRepo}:environment:${GitHubEnvironment}" + Policies: + - PolicyName: AgentCoreEvaluationDataPlanePolicy + PolicyDocument: + Version: "2012-10-17" + Statement: + - Sid: EvaluateOnlySyntheticProviderParity + Effect: Allow + Action: + - bedrock-agentcore:Evaluate + Resource: "*" + Tags: + - { Key: Project, Value: cloudai-platform } + - { Key: Environment, Value: aws-sandbox } + - { Key: ManagedBy, Value: cloudformation } + - { Key: DataScope, Value: synthetic-only } + GitHubActionsBootstrapRole: Type: AWS::IAM::Role Properties: @@ -960,6 +993,7 @@ Resources: Resource: - !Sub "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/${GitHubRepo}-${GitHubEnvironment}-terraform" - !Sub "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/${GitHubRepo}-${GitHubEnvironment}-budget-guardrails" + - !Sub "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/${GitHubRepo}-${GitHubEnvironment}-agentcore-evaluation" - !Sub "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/${GitHubRepo}-aws-private-eks-terraform" - !Sub "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/${GitHubRepo}-aws-private-eks-runner-terraform" - !Sub "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/${BootstrapRoleName}" @@ -996,6 +1030,9 @@ Outputs: BudgetGuardrailsRoleArn: Description: Store as AWS_BUDGET_GUARDRAILS_ROLE_TO_ASSUME in the aws-sandbox GitHub environment. Value: !GetAtt GitHubActionsBudgetGuardrailsRole.Arn + AgentCoreEvaluationRoleArn: + Description: Store as AWS_AGENTCORE_EVALUATION_ROLE_TO_ASSUME in the aws-sandbox GitHub environment. + Value: !GetAtt GitHubActionsAgentCoreEvaluationRole.Arn BudgetGuardrailsStateKey: Description: Fixed state key used only by the dedicated Budget Guardrails Terraform workflow. Value: !Ref BudgetGuardrailsStateKey diff --git a/providers/aws/infra/bootstrap/test_github_oidc_terraform_backend.rb b/providers/aws/infra/bootstrap/test_github_oidc_terraform_backend.rb index 96ae4e2..e5903ce 100644 --- a/providers/aws/infra/bootstrap/test_github_oidc_terraform_backend.rb +++ b/providers/aws/infra/bootstrap/test_github_oidc_terraform_backend.rb @@ -36,6 +36,24 @@ def test_includes_dedicated_budget_guardrails_role assert_includes template, "BudgetGuardrailsRoleArn:" end + def test_includes_dedicated_agentcore_evaluation_role + assert_includes template, "GitHubActionsAgentCoreEvaluationRole:" + assert_includes agentcore_evaluation_role, + 'RoleName: !Sub "${GitHubRepo}-${GitHubEnvironment}-agentcore-evaluation"' + assert_includes agentcore_evaluation_role, + 'token.actions.githubusercontent.com:sub: !Sub "repo:${GitHubOrg}/${GitHubRepo}:environment:${GitHubEnvironment}"' + assert_includes agentcore_evaluation_role, "bedrock-agentcore:Evaluate" + assert_includes template, "AgentCoreEvaluationRoleArn:" + end + + def test_agentcore_evaluation_role_cannot_mutate_or_invoke_other_services + %w[ + bedrock-agentcore:CreateEvaluator bedrock-agentcore:UpdateEvaluator + bedrock-agentcore:DeleteEvaluator bedrock-agentcore:InvokeAgentRuntime + logs:StartQuery cloudwatch:GetMetricData iam:PassRole s3:GetObject + ].each { |action| refute_includes agentcore_evaluation_role, action } + end + def test_includes_dedicated_private_eks_network_role assert_includes template, "GitHubActionsPrivateEKSNetworkRole:" assert_includes private_network_role, 'RoleName: !Sub "${GitHubRepo}-aws-private-eks-terraform"' @@ -57,7 +75,9 @@ def test_bootstrap_role_can_inspect_only_the_recovery_contract assert_includes bootstrap_role_management_statement, "iam:DeleteRole" assert_includes bootstrap_role_management_statement, "iam:DeleteRolePolicy" assert_includes bootstrap_role_management_statement, 'role/${GitHubRepo}-${GitHubEnvironment}-budget-guardrails' + assert_includes bootstrap_role_management_statement, 'role/${GitHubRepo}-${GitHubEnvironment}-agentcore-evaluation' assert_includes bootstrap_role_management_statement, 'role/${GitHubRepo}-aws-private-eks-terraform' + refute_includes bootstrap_role_management_statement, "role/*" refute_includes bootstrap_role_management_statement, "iam:DeleteUser" refute_includes bootstrap_role_management_statement, "iam:DeletePolicy" end @@ -131,8 +151,22 @@ def budget_role template.split("GitHubActionsBudgetGuardrailsRole:", 2).fetch(1, "").split("AgentCoreRagTerraformPolicy:", 2).first.to_s end + def agentcore_evaluation_role + template + .split("GitHubActionsAgentCoreEvaluationRole:", 2) + .fetch(1, "") + .split(/\n\s{2}[A-Z][A-Za-z0-9]+:/, 2) + .first + .to_s + end + def private_network_role - template.split("GitHubActionsPrivateEKSNetworkRole:", 2).fetch(1, "").split("AgentCoreRagTerraformPolicy:", 2).first.to_s + template + .split("GitHubActionsPrivateEKSNetworkRole:", 2) + .fetch(1, "") + .split(/\n\s{2}[A-Z][A-Za-z0-9]+:/, 2) + .first + .to_s end def bootstrap_role diff --git a/shared/examples/agent-evaluation-telemetry/provider-parity-thresholds.v1.json b/shared/examples/agent-evaluation-telemetry/provider-parity-thresholds.v1.json new file mode 100644 index 0000000..655e7b6 --- /dev/null +++ b/shared/examples/agent-evaluation-telemetry/provider-parity-thresholds.v1.json @@ -0,0 +1,12 @@ +{ + "contractVersion": "1.0", + "profileId": "provider-parity-v1", + "scenarioId": "synthetic-cited-answer", + "evaluatorThresholds": { + "Builtin.Correctness": 0.7, + "Builtin.ToolSelectionAccuracy": 0.7, + "Builtin.GoalSuccessRate": 0.7 + }, + "maximumParityDelta": 0.2, + "maximumProviderCalls": 6 +} diff --git a/shared/schemas/agent-evaluation-telemetry/provider-parity-report.schema.json b/shared/schemas/agent-evaluation-telemetry/provider-parity-report.schema.json new file mode 100644 index 0000000..c5693a1 --- /dev/null +++ b/shared/schemas/agent-evaluation-telemetry/provider-parity-report.schema.json @@ -0,0 +1,120 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "title": "Metadata-only provider parity evaluation report", + "type": "object", + "additionalProperties": false, + "required": [ + "contractVersion", + "thresholdVersion", + "evidenceLevel", + "generatedAt", + "sourceCommit", + "githubRunId", + "regionLabel", + "scenarioId", + "status", + "providerCallCount", + "durationBucket", + "aggregateTokenUsage", + "results", + "parity" + ], + "properties": { + "contractVersion": { "const": "1.0" }, + "thresholdVersion": { "const": "1.0" }, + "evidenceLevel": { "const": "provider-direct" }, + "generatedAt": { "type": "string", "format": "date-time" }, + "sourceCommit": { "type": "string", "pattern": "^[0-9a-f]{40}$" }, + "githubRunId": { "type": "string", "pattern": "^[0-9]+$" }, + "regionLabel": { "const": "ap-southeast-2" }, + "scenarioId": { "const": "synthetic-cited-answer" }, + "status": { "enum": ["passed", "failed"] }, + "providerCallCount": { "const": 6 }, + "durationBucket": { + "enum": ["under-1m", "under-5m", "under-15m", "15m-or-more"] + }, + "aggregateTokenUsage": { + "type": "object", + "additionalProperties": false, + "required": ["inputTokens", "outputTokens", "totalTokens"], + "properties": { + "inputTokens": { "type": "integer", "minimum": 0 }, + "outputTokens": { "type": "integer", "minimum": 0 }, + "totalTokens": { "type": "integer", "minimum": 0 } + } + }, + "results": { + "type": "array", + "items": { + "type": "object", + "additionalProperties": false, + "required": [ + "convention", + "evaluatorId", + "level", + "score", + "label", + "threshold", + "passed", + "reasonCode", + "tokenUsage" + ], + "properties": { + "convention": { "enum": ["otel-genai", "openinference"] }, + "evaluatorId": { + "enum": [ + "Builtin.Correctness", + "Builtin.ToolSelectionAccuracy", + "Builtin.GoalSuccessRate" + ] + }, + "level": { "enum": ["trace", "tool-call", "session"] }, + "score": { "type": "number", "minimum": 0, "maximum": 1 }, + "label": { "type": "string", "pattern": "^[a-z0-9_]+$" }, + "threshold": { "type": "number", "minimum": 0, "maximum": 1 }, + "passed": { "type": "boolean" }, + "reasonCode": { "type": "string", "pattern": "^[a-z0-9_]+$" }, + "tokenUsage": { + "type": "object", + "additionalProperties": false, + "required": ["inputTokens", "outputTokens", "totalTokens"], + "properties": { + "inputTokens": { "type": "integer", "minimum": 0 }, + "outputTokens": { "type": "integer", "minimum": 0 }, + "totalTokens": { "type": "integer", "minimum": 0 } + } + } + } + } + }, + "parity": { + "type": "array", + "items": { + "type": "object", + "additionalProperties": false, + "required": [ + "evaluatorId", + "otelGenaiScore", + "openInferenceScore", + "absoluteDelta", + "maximumDelta", + "passed" + ], + "properties": { + "evaluatorId": { + "enum": [ + "Builtin.Correctness", + "Builtin.ToolSelectionAccuracy", + "Builtin.GoalSuccessRate" + ] + }, + "otelGenaiScore": { "type": "number", "minimum": 0, "maximum": 1 }, + "openInferenceScore": { "type": "number", "minimum": 0, "maximum": 1 }, + "absoluteDelta": { "type": "number", "minimum": 0, "maximum": 1 }, + "maximumDelta": { "type": "number", "minimum": 0, "maximum": 1 }, + "passed": { "type": "boolean" } + } + } + } + } +}