feat: add protected AgentCore evaluation provider parity

.github/workflows/agentcore-evaluation-provider-parity.yml

@@ -0,0 +1,137 @@
+name: AgentCore evaluation provider parity
+
+on:
+  workflow_dispatch:
+    inputs:
+      mode:
+        type: choice
+        options: [validate, direct-spans]
+        default: validate
+      confirmation:
+        type: string
+        required: false
+
+permissions:
+  contents: read
+
+concurrency:
+  group: cloudai-agentcore-evaluation-provider-parity
+  cancel-in-progress: false
+
+jobs:
+  validate:
+    if: ${{ inputs.mode == 'validate' }}
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Set up Node.js
+        uses: actions/setup-node@v4
+        with:
+          node-version: "22"
+
+      - name: Set up pnpm
+        uses: pnpm/action-setup@v4
+        with:
+          version: 11.7.0
+
+      - name: Install API dependencies
+        working-directory: providers/aws/app/api
+        run: pnpm install --frozen-lockfile
+
+      - name: Test API contracts
+        working-directory: providers/aws/app/api
+        run: pnpm test
+
+      - name: Validate provider parity contracts
+        working-directory: providers/aws/app/api
+        run: pnpm agentcore-eval:provider-parity -- --mode validate
+
+  direct-spans:
+    if: ${{ inputs.mode == 'direct-spans' }}
+    runs-on: ubuntu-latest
+    environment: aws-sandbox
+    permissions:
+      contents: read
+      id-token: write
+    env:
+      CONFIRMATION: ${{ inputs.confirmation }}
+      AGENTCORE_EVALUATION_READY: ${{ vars.AGENTCORE_EVALUATION_READY }}
+      AGENTCORE_EVALUATION_MAX_CALLS: ${{ vars.AGENTCORE_EVALUATION_MAX_CALLS }}
+      AWS_AGENTCORE_EVALUATION_ROLE_TO_ASSUME: ${{ vars.AWS_AGENTCORE_EVALUATION_ROLE_TO_ASSUME || secrets.AWS_AGENTCORE_EVALUATION_ROLE_TO_ASSUME }}
+      AWS_REGION: ${{ vars.AWS_REGION || 'ap-southeast-2' }}
+      PROVIDER_PARITY_MODE: direct-spans
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Preflight protected direct evaluation
+        shell: bash
+        run: |
+          set -euo pipefail
+          [[ "$CONFIRMATION" == "I_UNDERSTAND_AGENTCORE_EVALUATION_PROVIDER_PARITY" ]] || {
+            echo "exact confirmation is required" >&2
+            exit 1
+          }
+          [[ "$AGENTCORE_EVALUATION_READY" == "true" ]] || {
+            echo "evaluation readiness is not enabled" >&2
+            exit 1
+          }
+          [[ "$AGENTCORE_EVALUATION_MAX_CALLS" == "6" ]] || {
+            echo "the direct evaluation call cap must be six" >&2
+            exit 1
+          }
+          [[ "$AWS_REGION" == "ap-southeast-2" ]] || {
+            echo "the protected evaluation region must be ap-southeast-2" >&2
+            exit 1
+          }
+          [[ -n "$AWS_AGENTCORE_EVALUATION_ROLE_TO_ASSUME" ]] || {
+            echo "the protected evaluation role is required" >&2
+            exit 1
+          }
+          [[ "$GITHUB_REF" == "refs/heads/main" ]] || {
+            echo "the protected evaluation must run from main" >&2
+            exit 1
+          }
+          [[ "$GITHUB_SHA" =~ ^[0-9a-f]{40}$ ]] || {
+            echo "the protected evaluation requires a full lowercase commit SHA" >&2
+            exit 1
+          }
+
+      - name: Set up Node.js
+        uses: actions/setup-node@v4
+        with:
+          node-version: "22"
+
+      - name: Set up pnpm
+        uses: pnpm/action-setup@v4
+        with:
+          version: 11.7.0
+
+      - name: Install API dependencies
+        working-directory: providers/aws/app/api
+        run: pnpm install --frozen-lockfile
+
+      - name: Test API contracts
+        working-directory: providers/aws/app/api
+        run: pnpm test
+
+      - name: Configure protected AWS role
+        uses: aws-actions/configure-aws-credentials@v4
+        with:
+          role-to-assume: ${{ env.AWS_AGENTCORE_EVALUATION_ROLE_TO_ASSUME }}
+          aws-region: ${{ env.AWS_REGION }}
+          mask-aws-account-id: true
+
+      - name: Run direct provider parity evaluation
+        working-directory: providers/aws/app/api
+        run: pnpm agentcore-eval:provider-parity -- --mode direct-spans --output "$RUNNER_TEMP/provider-direct-evaluation-report.json"
+
+      - name: Upload sanitized provider parity report
+        uses: actions/upload-artifact@v4
+        with:
+          name: provider-direct-evaluation-report
+          path: ${{ runner.temp }}/provider-direct-evaluation-report.json
+          if-no-files-found: error
+          retention-days: 7

.github/workflows/update-aws-bootstrap.yml

@@ -163,6 +163,26 @@ jobs:
             --query 'Stacks[0].{StackStatus:StackStatus,LastUpdatedTime:LastUpdatedTime}' \
             --output json
 
+      - name: Publish AgentCore evaluation role handoff
+        if: ${{ github.event_name == 'workflow_dispatch' && inputs.mode == 'apply' && success() }}
+        run: |
+          agentcore_evaluation_role_arn="$(aws cloudformation describe-stacks \
+            --stack-name "$AWS_BOOTSTRAP_STACK_NAME" \
+            --query "Stacks[0].Outputs[?OutputKey=='AgentCoreEvaluationRoleArn'].OutputValue | [0]" \
+            --output text)"
+          if [ -z "$agentcore_evaluation_role_arn" ] || [ "$agentcore_evaluation_role_arn" = "None" ]; then
+            echo "::error::Bootstrap stack did not return AgentCoreEvaluationRoleArn."
+            exit 1
+          fi
+          echo "::add-mask::$agentcore_evaluation_role_arn"
+          {
+            printf '%s\n' '## AgentCore evaluation Environment handoff'
+            printf '%s\n' ''
+            printf '%s\n' 'Add this exact masked value as the protected aws-sandbox Environment setting `AWS_AGENTCORE_EVALUATION_ROLE_TO_ASSUME`:'
+            printf '%s\n' ''
+            printf '`%s`\n' "$agentcore_evaluation_role_arn"
+          } >> "$GITHUB_STEP_SUMMARY"
+
       - name: Publish Budget Guardrails role handoff
         if: ${{ github.event_name == 'workflow_dispatch' && inputs.mode == 'apply' && success() }}
         run: |

docs/architecture/agentcore-governed-rag-poc.md

@@ -174,25 +174,39 @@ the sandbox is currently kept deployed for demonstrations.
 
 The evaluation boundary separates an agent framework from the quality gate.
 Synthetic OpenTelemetry GenAI and OpenInference traces are normalized into one
-contract before deterministic scenario and trajectory checks are applied:
+contract before deterministic scenario and trajectory checks are applied. The
+direct evaluation path is beside, not inside, the deployed Runtime and
+CloudWatch path:
 
 ```text
-Agent framework
-  -> OpenTelemetry GenAI or OpenInference spans
-  -> framework-neutral normalizer
-  -> deterministic local dimensions
-  -> versioned thresholds
-  -> metadata-only CI evidence
-  -> optional protected AgentCore parity evaluation
+local fixtures -> direct sessionSpans -> AgentCore Evaluate -> provider-direct evidence
+
+Gateway -> Runtime -> ADOT -> CloudWatch -> AgentCore Evaluate -> provider-runtime evidence
+                                Stage B: not implemented by this change
 ```
 
 Accepted scopes are limited to `opentelemetry.instrumentation.*` and
 `openinference.instrumentation.*`. The gate checks invoke-agent, inference,
 and execute-tool evidence, including `local.telemetry_compatibility` and
-`local.tool_trajectory_accuracy`. It is locally contract-tested and its
-ordinary CI job does not call AWS. OTLP export, CloudWatch trace ingestion,
-and managed AgentCore scoring remain a future protected provider-parity lane.
-The operating procedure and non-claim boundary are in the
+`local.tool_trajectory_accuracy`. `local-contract` evidence is locally
+contract-tested and its ordinary CI job does not call AWS. Stage A source
+implements the `provider-parity-v1` fixed, six-call direct-spans request matrix
+for future `provider-direct` evidence; provider validation is pending. It
+precedes Runtime ingestion so the reviewed spans, scenario, evaluator matrix,
+and metadata boundary can be checked without claiming that the Runtime emits
+or CloudWatch receives them. Stage B Runtime-to-CloudWatch evidence is the
+future `provider-runtime` lane and is not implemented.
+
+Within Stage A, Correctness uses a trace-scoped expected response,
+ToolSelectionAccuracy uses only a targeted tool span, and GoalSuccessRate uses
+a session-scoped assertion. Managed trajectory parity is outside this fixed
+profile and would require a separately reviewed `Builtin.Trajectory*` policy.
+
+Managed scores supplement deterministic controls. They never authorize IAM,
+admission or approval, tool execution, deployment, remediation, rollback, or
+deletion. No provider, runtime, or production evaluation has been validated.
+The operating procedure
+and non-claim boundary are in the
 [agent evaluation telemetry runbook](../solutions/agent-evaluation-telemetry-runbook.md).
 
 ### 5. What this diagram does and does not claim

docs/practices/current-status.md

@@ -57,11 +57,12 @@ The repository can now demonstrate:
   IAM Gateway/Runtime target, and confirmation-gated CI ingestion/invocation
   controls deployed; direct Bedrock preflight, Gateway end-to-end evidence, and
   bounded CloudWatch observability are complete; teardown remains separately gated
-- **Framework-neutral agent evaluation telemetry:** locally contract-tested
-  OpenTelemetry GenAI and OpenInference normalization, fixed synthetic prompts,
-  expected tool trajectories, strict deterministic dimensions, and a
-  metadata-only required CI artifact; the local gate does not call AWS and the
-  protected provider-parity lane remains pending
+- **Framework-neutral agent evaluation telemetry:** Stage A source implemented;
+  provider validation pending. The protected lane is manual, synthetic-only,
+  evaluate-only, and bounded to six calls. Stage B Runtime-to-CloudWatch
+  evaluation is not implemented. `local-contract` remains the only validated
+  evidence; future `provider-direct` and `provider-runtime` evidence are not
+  provider, runtime, or production validation.
 - **P5a AI-Assisted DevSecOps Boundary:** advisory AI use, human review, CI/security checks, and release evidence
 - **P5b AI-Assisted Review Evidence:** review summaries, threat-model checklists, CI failure summaries, and release-note drafts
 - **P6f AI Platform Security and Operations Controls:** identity, data protection, AI AppSec, delivery, operations, and FinOps
@@ -122,7 +123,7 @@ It currently has six mock-first lanes:
 | P8h AgentCore knowledge-lookup readiness | Complete static gateway-first reference architecture; no AgentCore resource or call | `docs/solutions/p8h-agentcore-knowledge-lookup-readiness.md` |
 | P8i AgentCore synthetic contract pack | Complete local synthetic contract evidence; provider-neutral pack remains separate from the live AWS validation | `shared/schemas/agentcore-readiness/`, `shared/examples/agentcore-readiness/`, `providers/aws/app/api/tests/agentcoreReadinessContracts.test.ts`, and `docs/solutions/p8i-agentcore-synthetic-contract-pack.md` |
 | AgentCore governed RAG POC | Synthetic data foundation, arm64 Runtime, IAM Gateway/Runtime target, direct Bedrock preflight, Gateway end-to-end evidence, and bounded CloudWatch observability complete through protected CI; teardown remains separately gated | `providers/aws/app/agentcore-rag-runtime/`, `providers/aws/agentcore/`, `.github/workflows/terraform-agentcore-rag-sandbox.yml`, `providers/aws/infra/bootstrap/github-oidc-terraform-backend.yaml`, `docs/solutions/p8i-agentcore-rag-data-foundation.md`, `docs/solutions/p8i-agentcore-rag-key-process-record.md`, and `docs/solutions/agentcore-governed-rag-poc-runbook.md` |
-| Framework-neutral agent evaluation telemetry | Locally contract-tested with synthetic OpenTelemetry GenAI and OpenInference fixtures; required CI gate does not call AWS; protected provider-parity lane pending | `shared/schemas/agent-evaluation-telemetry/`, `shared/examples/agent-evaluation-telemetry/`, `providers/aws/app/api/src/evals/agentEvaluationTelemetryNormalizer.ts`, `providers/aws/app/api/src/evals/agentEvaluationTelemetryGate.ts`, and `docs/solutions/agent-evaluation-telemetry-runbook.md` |
+| Framework-neutral agent evaluation telemetry | Stage A source implemented; provider validation pending. The protected lane is manual, synthetic-only, evaluate-only, and bounded to six calls. Stage B Runtime-to-CloudWatch evaluation is not implemented. | `shared/schemas/agent-evaluation-telemetry/`, `shared/examples/agent-evaluation-telemetry/`, `providers/aws/app/api/src/evals/agentEvaluationTelemetryNormalizer.ts`, `providers/aws/app/api/src/evals/agentEvaluationTelemetryGate.ts`, `providers/aws/app/api/src/evals/agentCoreEvaluationProviderGate.ts`, `.github/workflows/agentcore-evaluation-provider-parity.yml`, and `docs/solutions/agent-evaluation-telemetry-runbook.md` |
 | P5a AI-assisted DevSecOps boundary | Complete | `docs/practices/ai-assisted-devsecops-pattern.md` and `.github/workflows/ai-assisted-devsecops.yml` |
 | P5b AI-assisted review evidence | Complete | `docs/evidence/ai-assisted-review-evidence.md`, `shared/schemas/ai-assisted-devsecops/`, and `shared/examples/ai-assisted-devsecops/` |
 | P6d control-plane evidence map | Complete | `docs/evidence/control-plane-evidence-map.md`, `shared/schemas/control-plane-evidence/`, and `shared/examples/control-plane-evidence/` |