diff --git a/.github/labeler.yml b/.github/labeler.yml new file mode 100644 index 0000000..23b4dfe --- /dev/null +++ b/.github/labeler.yml @@ -0,0 +1,35 @@ +# Configuration for .github/workflows/label.yml (actions/labeler@v7). +# +# The workflow has run since it was added and has never succeeded, because this +# file did not exist. actions/labeler resolves its config with +# `ref: github.context.sha`, which under `pull_request_target` is the base +# branch commit — so the file has to be on the default branch to be found. A +# contributor cannot supply it from a pull request branch. +# +# Labels below are limited to ones that already exist in this repository. +# actions/labeler creates missing labels implicitly with an arbitrary colour, so +# adding a new name here has a side effect outside this file. + +documentation: + - changed-files: + - any-glob-to-any-file: + - 'docs/**' + - '**/*.md' + +python: + - changed-files: + - any-glob-to-any-file: + - '**/*.py' + - 'requirements.txt' + +github_actions: + - changed-files: + - any-glob-to-any-file: + - '.github/workflows/**' + - '.github/labeler.yml' + +dependencies: + - changed-files: + - any-glob-to-any-file: + - 'requirements.txt' + - '**/package.json' diff --git a/.github/workflows/shell-checks.yml b/.github/workflows/shell-checks.yml index cab87d7..a70812b 100644 --- a/.github/workflows/shell-checks.yml +++ b/.github/workflows/shell-checks.yml @@ -48,3 +48,12 @@ jobs: - name: Workshop module flow (maps, explains, verifies, ordering) run: bash scripts/check-workshop-flow.sh + + # The Kiro power in kiro/ documents this repo's commands, flags and module + # sequences. It ships here so that a rename breaks the build rather than + # quietly breaking someone's session: this gate resolves every file:line + # citation against the tree, and checks the profile sequences against + # PROFILE_MODULES and each cited --flag against the script it is used with. + # Stdlib only, no AWS, no network. + - name: Kiro power matches the source it documents + run: bash scripts/check-kiro-power.sh diff --git a/README.md b/README.md index e75b623..cef2030 100644 --- a/README.md +++ b/README.md @@ -28,6 +28,10 @@ Use this five-step path to get from a starting point to a working deployment. > When something breaks, [`docs/TROUBLESHOOTING.md`](docs/TROUBLESHOOTING.md) is > organised by symptom. +> **Using Kiro?** This repository ships a [Kiro power](#drive-it-with-kiro) that +> can walk any of these five steps with you — picking a profile, deploying and +> verifying a module at a time, and diagnosing what failed. + ## Choose Your Starting Point @@ -134,6 +138,46 @@ python3 -m http.server 8888 -d dashboard/public ![AgentCore deployment dashboard monitor tab](docs/dashboard-monitor.png) +## Drive It with Kiro + +[`kiro/agentcore-enterprise-platform/`](kiro/agentcore-enterprise-platform) is a +[Kiro](https://kiro.dev) **power**: the operational knowledge about *this* repository, +packaged so an agent can drive it rather than just describe it. Ask it which profile +fits your situation, what a module actually deploys, why module 6 has been silent for +seven minutes, what is billing right now, or hand it the deployment and approve one +command at a time. + +Powers are added through the Kiro UI. **Powers panel → Add Custom Power → Local +Directory**, then paste the path this prints — the power directory itself, not the +repository root: + +```bash +echo "$(git rev-parse --show-toplevel)/kiro/agentcore-enterprise-platform" +``` + +`POWER.md` routes; the detail sits in 14 `steering/` files that load only when the +question calls for them. Six of those are **runbooks** — deploy the platform, deploy +one module, verify, recover a failed deploy, audit cost, tear down — ordered +procedures with a verify gate after each step and explicit halt conditions, for when +you want the deployment run rather than explained. They spend real money in a real +account, so anything that creates, changes, deletes, or bills is proposed one command +at a time and waits for you. + +> **It is blunt about this repository on purpose.** Security controls are opt-in and +> default off; a `-security` stack at `CREATE_COMPLETE` means the resources exist, not +> that anything is enforced; `enable_networking=true` is not an air-gapped VPC; Cedar +> ships in `LOG_ONLY`. The power says so, in the same terms as +> [`docs/SECURITY_CONTROLS.md`](docs/SECURITY_CONTROLS.md), because a facilitator who +> overstates the posture loses the room in the first security question. + +The power ships here rather than in a repository of its own so that its claims stay +tied to the code: `scripts/check-kiro-power.sh` runs in CI and fails the build when a +cited `file:line` no longer resolves, a restated profile sequence drifts from +`PROFILE_MODULES` in `scripts/deploy.sh`, or a cited `--flag` does not exist in the +script it is used with. A rename breaks the build instead of quietly breaking +someone's session. [`kiro/README.md`](kiro/README.md) covers the layout and how to +change it. + ## Clean Up Destroy resources when you no longer need them diff --git a/kiro/README.md b/kiro/README.md new file mode 100644 index 0000000..5849b32 --- /dev/null +++ b/kiro/README.md @@ -0,0 +1,81 @@ +# Kiro power for this accelerator + +[`agentcore-enterprise-platform/`](agentcore-enterprise-platform) is a +[Kiro](https://kiro.dev) **power**: operational knowledge about *this* repository, +packaged so an agent can drive it — pick a deployment profile, deploy and verify a +module, diagnose a failure, audit what is billing, tear it down. + +It carries what a checkout does not: which profile fits a given situation, what +each module actually deploys versus what its title suggests, which command proves +a layer works, why module 6 goes silent for seven minutes, which flags are sharp, +and which security controls are enforced rather than merely declared. + +## Install it + +Powers are added through the Kiro UI; there is no CLI for it. + +**Powers panel → Add Custom Power → Local Directory**, then paste the absolute +path of the `agentcore-enterprise-platform/` directory inside your clone — not the +repository root, and not this `kiro/` directory: + +```bash +echo "$(git rev-parse --show-toplevel)/kiro/agentcore-enterprise-platform" +``` + +After pulling a change to these files: **Powers panel → the power → Check for +Updates → Update Power**. + +The power bundles two MCP servers (`mcp.json`): the AgentCore MCP server, which +needs [uv](https://docs.astral.sh/uv/) on `PATH` for `uvx`, and the AWS Knowledge +server over HTTP. Nothing in either is pre-approved — Kiro strips `autoApprove` +from a power's `mcp.json` on load, so a power cannot grant itself auto-approval. +Approving the read-only tools and leaving anything that creates, updates, deletes +or invokes on manual confirmation is the posture to aim for. + +## Layout, and why it is this exact shape + +``` +agentcore-enterprise-platform/ + POWER.md router: profile picker, module map, the sharp edges, steering index + mcp.json MCP servers only, no display metadata + steering/ the detail — loaded on demand, one file per question shape + runbook-*.md six procedures, for driving rather than explaining +``` + +Kiro's power installer copies a fixed allowlist — `POWER.md`, `mcp.json`, and +`.md` files under `steering/` — and its validator **rejects** a power directory +containing anything else that looks like a script, an archive, a credential, or a +**hidden file at any depth**. So `.DS_Store` landing in here makes the power fail +to install with an error that does not mention `.DS_Store`. That is why this +README lives one level up, outside the power directory, and why +`scripts/check-kiro-power.sh` checks installability rather than trusting it. + +It is also why the six runbooks are `steering/runbook-*.md` rather than the +`skills//SKILL.md` layout Kiro's skill *reader* expects: `skills/` is not on +the installer's allowlist, so an installed power has no `skills/` directory and +every runbook read fails. `POWER.md` routes to the filenames that actually exist. + +## Changing it + +Run the gate before you commit: + +```bash +bash scripts/check-kiro-power.sh +``` + +It runs in CI on every pull request and makes no AWS calls and no network calls. +Beyond the power format, it checks the claims against the tree they are shipped +with: every `file:line` citation resolves and is in bounds, every restatement of a +profile's module sequence matches `PROFILE_MODULES` in `scripts/deploy.sh`, and +every `--flag` cited next to one of this repo's scripts exists in that script. + +That coupling is the point of shipping the power here rather than in a repository +of its own. A hallucinated flag reads exactly like a real one and only fails in +front of a user, and a renamed flag is indistinguishable from a hallucinated one +a month later. Keeping these files next to the code means a rename breaks the +build instead of quietly breaking someone's session. + +**The one rule that matters most: every claim must be verifiable in this +repository's source.** Prefer a real run over a reading — the timings and error +strings in `steering/troubleshooting.md` came from actual deployments. If you +cannot confirm something, cut it rather than hedging it. diff --git a/kiro/agentcore-enterprise-platform/POWER.md b/kiro/agentcore-enterprise-platform/POWER.md new file mode 100644 index 0000000..ea7a78e --- /dev/null +++ b/kiro/agentcore-enterprise-platform/POWER.md @@ -0,0 +1,296 @@ +--- +name: "agentcore-enterprise-platform" +displayName: "Enterprise Agentic AI Platform Accelerator" +description: "Stand up a governed multi-agent platform on Amazon Bedrock AgentCore using the aws-samples enterprise accelerator — pick a deployment profile, deploy and verify each module, and run a guided team build. Covers Gateway/MCP, A2A agents, memory, identity, networking, security controls, and observability." +keywords: ["agentcore", "enterprise agent platform", "agentcore gateway", "a2a agents", "deployment profile", "agent platform accelerator", "mcp gateway"] +author: "AWS" +--- + +# Enterprise Agentic AI Platform Accelerator + +Drives [`aws-samples/sample-agentcore-enterprise-platform`](https://github.com/aws-samples/sample-agentcore-enterprise-platform) +— a CDK accelerator that deploys a governed agent platform on Amazon Bedrock +AgentCore: Cognito identity, an MCP Gateway with tool targets, agent runtimes, +Memory, A2A sub-agents, VPC networking, security controls, and observability. + +Twelve modules, five deployment profiles, one script (`./scripts/deploy.sh`). +This power knows which profile fits a given situation, what each module actually +deploys, which command proves it worked, and which flags are sharp — so a build +session spends its time on the customer's architecture instead of on rediscovery. + +## Use this power when someone is + +- Standing up an agent platform for an organization, not a single agent +- Choosing between the accelerator's profiles (`greenfield`, `migration`, + `multi-agent`, `platform-team`, `security-focused`) +- Running or facilitating a guided team build / workshop on this accelerator +- Debugging a module: a deploy that failed, an invoke that returns + `Unauthorized` or HTTP 424, an agent that reports no tools, traces that never + appear +- Extending it: adding a gateway tool, contributing a stack through the + `use-cases/` extension point, swapping the agent framework, turning on + Cedar / guardrails / VPC mode, going multi-account + +## Use something else when + +- **Building one agent, locally, from scratch** → the `aws-agentcore` power. + That one is about `agentcore configure` / `agentcore launch` and the local dev + loop for a single agent. This one is about a platform that many teams build + *on*, deployed from an existing CDK repo. If there is no CDK accelerator + checkout in play, it is probably the other power. +- **General AWS architecture or IaC questions** → a general AWS power. +- The user is asking about Bedrock Agents (the older service) rather than + Bedrock AgentCore. + +## Hard prerequisites + +Check these before anything else. Each one has produced a failed session: + +| Requirement | Why it is hard | +|---|---| +| `python3.13` on PATH under exactly that name | the scripts invoke `python3.13`, not `python3` | +| `bash` 4+ | the deploy script uses associative arrays; macOS `/bin/bash` is 3.2 and dies on `declare -A`. `brew install bash` | +| `node` + `npm` | the CDK CLI runs through `npx` | +| **A current CDK CLI** — `npm install -g aws-cdk@latest` | `requirements.txt` has no upper bound on `aws-cdk-lib`, and the repo has no `package.json`, so pip installs the newest library while `npx` picks up whatever CLI is installed globally. A stale global CLI fails at bootstrap with a schema-version mismatch, and the prereq check passes it anyway — it only tests that `cdk` exists | +| `aws` CLI with working credentials | validated before any deploy | +| **At least one Claude model enabled in Bedrock, in the target Region** | the first agent invoke (module 6) fails with an access error otherwise | +| A Region where AgentCore *and* the model exist | `us-east-1` is the safe default | +| Docker | **not** required — images build in AWS CodeBuild | + +```bash +git clone https://github.com/aws-samples/sample-agentcore-enterprise-platform.git +cd sample-agentcore-enterprise-platform +python3.13 -m venv .venv && source .venv/bin/activate +pip install -r requirements.txt +./scripts/deploy.sh workshop --dry-run # zero AWS calls; prints the whole plan +``` + +Always start with `workshop --dry-run`. It makes no AWS calls at all — no +credential check, no bootstrap — and prints every module, its stacks, and its +verify command. One exception: `--profile security-focused` hits the `ORG_ID` gate +before the plan prints, even in dry-run. + +**`--dry-run` only exists for `workshop`.** The `deploy` action parses the flag and +then ignores it: `deploy --module 3 --dry-run` bootstraps the Region and deploys +module 3 for real. Measured — a "dry run" aimed at an untouched Region left a +CDK bootstrap stack and a live Cognito user pool behind. Read `steering/deploy.md` +before previewing anything with `deploy`. + +## Pick a profile + +The profile decides which modules get walked and which feature flags are on. + +| Profile | Modules | For the person who… | Notes | +|---|---|---|---| +| `greenfield` | 3 4 5 6 9 | is building a first agent platform | everything opt-in stays off | +| `migration` | 3 4 6 7 9 | already has an agent and needs it governed | skips module 5, adds tools in 7 | +| `multi-agent` | 3 4 5 6 7 8 9 | needs agents that delegate to each other | `enable_a2a=true` | +| `platform-team` | 3 4 5 A 6 7 8 9 C E | runs the platform other teams build on | networking + security + A2A; **hourly cost** | +| `security-focused` | 3 4 5 6 9 E | has to answer to a security review | needs `ORG_ID`; **hourly cost** | + +`security-focused` also enables resource policies, the egress filter, Cedar, and +traceability. The memory resource policy renders `aws:PrincipalOrgID`, so export +an Organizations ID first or the run stops before the first module — including +under `--dry-run`: + +```bash +export ORG_ID=$(aws organizations describe-organization --query Organization.Id --output text) +``` + +Not in an Organization? Pick another profile, or turn `enable_resource_policies` +off. + +## The modules + +| # | Module | What lands | Proof | Time | +|---|---|---|---|---| +| **3** | Infrastructure Blueprint | Cognito pool, 3 OAuth clients, SSM discovery params | issuer URL in SSM | ~2 min | +| **4** | Identity Integration | the M2M credential provider agents use to reach the gateway | provider name in SSM | ~2 min | +| **5** | Gateway & Registry | AgentCore MCP Gateway, Lambda tool target, JWT auth | `test_gateway.py` | ~3 min | +| **A** | Memory | managed Memory + user-preference strategy | `test_memory.py` | ~2 min | +| **6** | Agent Deployment | orchestrator agent: CodeBuild → Runtime | `invoke.py` | **~7–8 min** | +| **7** | Gateway Integration | new tool targets, no agent redeploy | `test_gateway.py` | ~3 min | +| **8** | Agent-to-Agent | code + research sub-agents on their own runtimes | `invoke.py --a2a code-agent` | ~8 min | +| **9** | Observability | vended logs, X-Ray, Transaction Search | `check_observability.py` | ~3 min | +| **B** | Code Interpreter | redeploys the orchestrator; no verify, in no profile | — | — | +| **C** | Multi-Account Networking | VPC, private subnets, AgentCore VPC endpoints | `check_network.py` | ~5 min | +| **D** | CI/CD | nothing — a discussion over `.gitlab-ci.yml` | — | — | +| **E** | Security Automation | KMS CMK + CloudTrail | stack COMPLETE | ~3 min | + +Roughly 38–40 minutes of deploy time for a full walk. Modules run in the +profile's order, not numeric order. + +## Five things that waste the most time + +Lead with these. Each is real behaviour, not a caveat. + +1. **`--profile` does not scope the deploy — and it writes to disk.** It + materializes `presets/.yaml` as `platform.yaml`, the durable + deployment manifest (`scripts/deploy.sh:107-146`), and then still runs + `cdk deploy --all` — everything that manifest makes the app synthesize, not + just the profile's modules. Two consequences worth separating: the *intent* + now persists across runs, which is the fix; the *scope* still does not narrow, + which is the trap. Scope with `--module` or `--team`, or use + `workshop --profile

` for the module-by-module path. Interactively you get + a footprint prompt before `--all` (`confirm_footprint`, + `scripts/deploy.sh:658-679`) — so the trap is only silent under `--yes` or + `NON_INTERACTIVE=1`, which is exactly the CI path. Note that `--team platform` + and `--team security` both fail bare, because `TEAM_MAP` names flag-gated + stacks — turn networking/security on in the manifest first + (`facilitation.md`). +2. **`--dry-run` on `deploy` is accepted and ignored.** The `deploy` case never + checks it (`scripts/deploy.sh:1052-1081` — no `DRY_RUN` test), so + `deploy --module 3 --dry-run` bootstraps and deploys for real. `--dry-run` is + honoured only by `workshop`. Confirm from the output, not from what you typed: + a real dry run prints `DRY RUN — nothing will be deployed` and no + `═══ Deploying ═══` header. Misspellings, by contrast, are now safe — the + parser fails closed on an unknown option and on a bad `--profile`/`--team` + *value* for every action (`scripts/deploy.sh:955-990`), so `--dryrun`, + `--modul 6` and `--team platfrom` all exit 1 instead of escalating to `--all`. +3. **Module 6 going quiet for 7–8 minutes is not a hang.** CodeBuild is building + an arm64 container image remotely. +4. **The default `orchestrator` agent has no tools.** Asking it "what tools do + you have?" correctly returns nothing. Tools live on the gateway — query them + with `invoke.py --tools` or `test_gateway.py`, or deploy a tool-consuming + pattern (`strands-agent`, `langgraph-agent`, `claude-sdk-*`, `agui-*`). + It has **no memory either** — `agent-code/orchestrator/` never reads the + `MEMORY_ID` it is given, so a same-session recall demo fails even with Memory + `ACTIVE`. Use `strands-agent` or `langgraph-agent` for a memory demo. +5. **`CREATE_COMPLETE` proves nothing about behaviour.** The verify scripts exist + because stacks have completed while the thing they promise was broken. Run + them, and treat a failing verify as information rather than a dead end. + +## Runbooks — for doing, not explaining + +When the user wants to **run** something rather than understand it, read the +matching runbook and follow it. These are procedures with verification gates and +halt conditions, and they assume a real AWS account and real money. + +They are steering files, so read them the same way as any other file below — +`runbook-deploy-platform.md`, not a skill named `deploy-platform`. Kiro's power +installer copies only `POWER.md`, `mcp.json` and `steering/*.md`, so a runbook +that is not a steering file is not a runbook the agent can read. + +| Runbook | Use when | +|---|---| +| `runbook-deploy-platform.md` | deploying end to end — a profile walk, a workshop run, a first build | +| `runbook-deploy-module.md` | one module: adding a layer, redoing a failed one, swapping the agent pattern, turning on networking or security | +| `runbook-verify-platform.md` | "is my platform actually working" — the full per-layer pass/fail matrix | +| `runbook-recover-deploy.md` | a module failed, stalled, or was interrupted; a stack is stuck in a rollback state | +| `runbook-cost-audit.md` | "what is this costing" / "did we leave anything running" — read-only, no changes | +| `runbook-teardown-platform.md` | finishing up, or confirming an account is genuinely clean | + +All six follow the same discipline: **anything that creates, changes, deletes or +bills goes one command at a time** — state what it does and what it costs, then +stop and wait for approval, and never group it with anything else. Read-only +checks are cheap approvals; group them rather than teaching the participant to +click through five prompts without reading. Never run a billable command to learn +something a read-only call answers. + +Two things to say before the first `deploy` of any session, because both are +irreversible once they have happened: + +- Once `-auth` exists, **every** `deploy` prints the Cognito M2M client secret to + stdout in plaintext — including runs that never touch `-auth`. No `deploy` is + screen-safe. Say so before a shared screen, not after. +- Confirm the account id from `aws sts get-caller-identity` out loud. Deploying + into the wrong account is the one mistake here with no undo. + +## Detailed guidance — read on demand + +Read the steering file that matches the question. Do not read them all. The six +`runbook-*.md` files above live here too; these eight are the reference material. + +| File | Read this when | +|---|---| +| `deploy.md` | choosing/scoping a deploy, config precedence, `platform.yaml`, feature flags, teams, resume, teardown | +| `modules.md` | you need to know exactly what a module deploys, its stack names, and its verify command | +| `verify.md` | proving a layer works: `deploy.sh verify`, `invoke.py`, `test_gateway.py`, `test_memory.py`, `check_observability.py`, `check_network.py`, the dashboard | +| `troubleshooting.md` | something failed — symptom → cause → fix | +| `agent-patterns.md` | choosing or swapping an agent framework, A2A, AG-UI, protocols, adding a gateway tool | +| `security.md` | a security review: what each control really enforces, SCPs, Cedar, guardrails, VPC mode, identity | +| `facilitation.md` | running a guided team build: agenda, team split, pre-empts, what to say while a module builds | +| `patterns.md` | matching a customer situation to a concrete recipe | + +## MCP servers + +Two servers ship with this power. + +**`agentcore-mcp-server`** (`uvx awslabs.amazon-bedrock-agentcore-mcp-server@latest`) +— AgentCore documentation search plus the AgentCore control plane. This server +has write tools as well as read ones, and **nothing here is pre-approved**: Kiro +deletes `autoApprove` from a power's `mcp.json` on load, so a power cannot grant +itself auto-approval. Every call prompts until the user chooses to always-allow +it. That is the right default — treat the read-only tools below as the ones that +are safe to always-allow, and leave anything that creates, updates, deletes, or +invokes on manual confirmation. + +Most useful here: + +- `search_agentcore_docs`, `fetch_agentcore_doc` — authoritative service + behaviour when the accelerator's docs stop short +- `get_runtime_guide`, `get_gateway_guide`, `get_memory_guide`, + `get_identity_guide`, `get_policy_guide` — deep reference per service +- `list_agent_runtimes`, `get_agent_runtime` — confirm what a deploy produced, + including `protocolConfiguration` and the runtime version +- `gateway_list`, `gateway_get`, `gateway_target_list` — inspect the gateway and + its targets +- `memory_list`, `memory_get`, `memory_retrieve_records` — inspect Memory +- `policy_engine_list`, `policy_list` — inspect Cedar policies + +**`awsknowledge`** (`https://knowledge-mcp.global.api.aws`) — AWS documentation +for everything around AgentCore: Cognito, CloudFormation, Organizations/SCPs, +X-Ray and Transaction Search, VPC endpoints, Secrets Manager. + +Prefer the accelerator's own repo as the source of truth for *its* commands and +flags; use these servers for *service* behaviour. When they disagree about the +accelerator, the checked-out source wins. + +## Cost + +Real, billable AWS resources. Most of the platform is pay-per-use and nearly +free while idle, with three exceptions worth saying out loud before a session: + +- **Networking profiles create a NAT gateway and VPC endpoints that bill + hourly** — `platform-team` and `security-focused`. Precisely: 1 NAT gateway + plus 5 interface endpoints across 2 AZs, so 10 endpoint-AZ-hours. The other + three profiles leave **nothing** billing hourly. `deploy.md` has the full + standing-cost inventory, which is the answer to "what does this cost if we + leave it up overnight?" +- **Transaction Search changes span-ingestion pricing account-wide**, and it + defaults **on** (`enable_transaction_search`). It stays enabled after teardown + on purpose, because other workloads may come to rely on it. Deploy with + `-c enable_transaction_search=false` if a platform team owns tracing + elsewhere — but expect no traces, which is exactly the failure it prevents. +- CodeBuild runs per container build. + +Tear down the same day: `./scripts/deploy.sh destroy`. The networking stack can +refuse to delete for a few hours while AgentCore's network interfaces drain — +expected, and the expensive parts (NAT, endpoints) are already gone by then. + +## Never do these + +- Put a secret in `platform.yaml`, `workshop.env`, or a CDK context flag. The + accelerator passes Secrets Manager **names** only, resolved at deploy time. +- Assume the Cognito M2M client secret is only in Secrets Manager. The `-auth` + stack exports it to CloudFormation **in plaintext**, and once that stack exists + **every** later `deploy` prints it to stdout — including runs that never touch + `-auth`, because the closing summary dumps every prefix-matching stack's outputs. + No `deploy` is screen-safe. It is also the first thing a security reviewer can + confirm in one read-only call; `security.md` has the mechanism and the mitigations. +- Set `cedar_mode=ENFORCE` before reading the `LOG_ONLY` decision logs. The + shipped permit is unconstrained on principal and resource; narrow it first. +- Attach the Terraform SCPs anywhere but a sandbox OU on the first pass. They + are additive-deny and applied from the Organizations management account. +- Describe `enable_networking=true` as air-gapped. Private subnets keep a NAT + route; see `security.md`. + +--- + +This power ships inside the accelerator it drives, at +`kiro/agentcore-enterprise-platform/`. Every `file:line` citation, profile +sequence and `--flag` below is checked against the surrounding source tree by +`scripts/check-kiro-power.sh` on every pull request, so these files move when the +code moves instead of drifting from a pinned commit. MCP tool names validated +against `amazon-bedrock-agentcore-mcp-server` 1.29.0. Where a command here and +the source still disagree, the source wins — please open an issue. diff --git a/kiro/agentcore-enterprise-platform/mcp.json b/kiro/agentcore-enterprise-platform/mcp.json new file mode 100644 index 0000000..4577a3c --- /dev/null +++ b/kiro/agentcore-enterprise-platform/mcp.json @@ -0,0 +1,14 @@ +{ + "mcpServers": { + "agentcore-mcp-server": { + "command": "uvx", + "args": ["awslabs.amazon-bedrock-agentcore-mcp-server@latest"], + "disabled": false + }, + "awsknowledge": { + "url": "https://knowledge-mcp.global.api.aws", + "type": "http", + "disabled": false + } + } +} diff --git a/kiro/agentcore-enterprise-platform/steering/agent-patterns.md b/kiro/agentcore-enterprise-platform/steering/agent-patterns.md new file mode 100644 index 0000000..7a3b7de --- /dev/null +++ b/kiro/agentcore-enterprise-platform/steering/agent-patterns.md @@ -0,0 +1,482 @@ +# Agent patterns and protocols + +Read this when choosing or swapping an agent framework, wiring A2A or AG-UI, or +adding a tool to the gateway. + +**The platform is framework-agnostic: the agent pattern is a config value, not a +rewrite.** That is the claim the whole pitch rests on, and it is the single most +effective thing to demonstrate live. + +```bash +AGENT_PATTERN=langgraph-agent ./scripts/deploy.sh deploy --module 6 +``` + +One redeploy of the orchestrator runtime, no infrastructure change. A bad pattern +name is rejected **up front**, before any AWS call, rather than failing later inside +CodeBuild. Measured on a warm account: **172s** for `orchestrator` → `langgraph-agent`, +including the CodeBuild rebuild — quicker than module 6's first deploy because only +the image changes. Confirm the swap really happened rather than trusting the exit code: + +```bash +ARN=$(aws ssm get-parameter --name /$PROJECT_NAME/$ENVIRONMENT/runtimes/orchestrator/arn \ + --query Parameter.Value --output text) +aws bedrock-agentcore-control get-agent-runtime --agent-runtime-id "${ARN##*/}" \ + --query '[agentRuntimeVersion,protocolConfiguration,networkConfiguration.networkMode]' +aws ecr describe-images --repository-name $PREFIX-orchestrator \ + --query 'sort_by(imageDetails,&imagePushedAt)[-2:].imageTags' +``` + +A bumped `agentRuntimeVersion` **and** a new image tag is the proof. Either alone is not. + +### Rehearse the swap before demoing it: the output shape changes + +**`invoke.py` prints whatever the pattern emits, and the streaming patterns emit raw +SSE.** This is the single most likely way the framework-swap demo goes wrong, because +it looks like a crash and is not one: + +| Pattern | What `invoke.py` prints | +|---|---| +| `orchestrator` | one line — `{"status": "success", "response": "VPC OK"}` | +| `langgraph-agent` | **176 lines** of `data: {"content": [{"type":"text","text":" noted that your favour"}], …}` — one chunk per few tokens | + +The answer is in there, spread across the `content[].text` fields, but nobody reads it +off the screen. Two options, both fine — just pick one *before* the session: + +- Demo the swap with `--tools` or a stack/ECR diff, and keep the prose answer on the + `orchestrator` pattern. +- Or pipe it through a reassembler so the room sees a sentence: + +```bash +.venv/bin/python scripts/invoke.py "…" \ + | python3 -c 'import sys,json,re +print("".join(p["text"] for l in sys.stdin if l.startswith("data: ") + for p in (json.loads(l[6:]).get("content") or []) if isinstance(p,dict) and p.get("text")))' +``` + +Do not "fix" this by switching to `--agui`: that flag is for the `agui-*` patterns and +fails on protocol against an `HTTP` runtime. + +--- + +## The seven patterns + +Set with `AGENT_PATTERN=…` or `-c agent_pattern=…`, or `agents.pattern` in +`platform.yaml`. + +| Pattern | Protocol | Verifies caller JWT | Gateway tools | Uses Memory | Default model | Notes | +|---|---|---|---|---|---|---| +| `orchestrator` *(default)* | HTTP | no | **none** | **no** | `us.anthropic.claude-sonnet-4-6` | deliberately minimal | +| `strands-agent` | HTTP | yes | not wired in this pattern yet | yes | `us.anthropic.claude-sonnet-4-6` | | +| `langgraph-agent` | HTTP | yes | yes | yes | `us.anthropic.claude-sonnet-4-6` | needs Memory data-plane IAM (`ListEvents`) | +| `claude-sdk-agent` | HTTP | yes | yes | **no** | `us.anthropic.claude-opus-4-6-v1` | replies include `claude_session_id` | +| `claude-sdk-multi-agent` | HTTP | yes | yes | yes | `us.anthropic.claude-opus-4-6-v1` | delegates to a `code-analyst` subagent | +| `agui-strands-agent` | **AGUI** | yes | yes | yes | `us.anthropic.claude-sonnet-4-6` | typed SSE; invoke with `--agui` | +| `agui-langgraph-agent` | **AGUI** | yes | yes | yes | `us.anthropic.claude-sonnet-4-6` | slower first response (graph built per request) | + +**"Uses Memory" is not the same as "Memory exists."** A `-memory` stack and a live +Memory resource land at module 6 in every profile, and `MEMORY_ID` is injected into +every runtime's environment — but only the patterns marked yes above actually read +or write it. `agent-code/orchestrator/` contains no reference to memory at all. + +Verified consequence: on a clean `greenfield` deploy, with Memory `ACTIVE` and +`test_memory.py` passing 5/5, the default orchestrator still does not remember +across turns of the **same** session: + +```bash +.venv/bin/python scripts/invoke.py --session demo-1 "Remember my favourite colour is teal." +# "I'll remember that your favourite colour is teal!" +.venv/bin/python scripts/invoke.py --session demo-1 "What is my favourite colour?" +# "I don't have any information about your favourite colour!" +``` + +The first reply is the model being agreeable, not a write. **Never build a memory +demo on the default pattern** — the recall step fails in front of the room. Check it +against source before promising it: + +```bash +grep -rli memory agent-code// +``` + +### …and a memory-using pattern is still not enough for cross-session recall + +Measured on `langgraph-agent`, deployed and healthy, with Memory `ACTIVE`: + +| Question | Result | +|---|---| +| Recall within the **same** `--session` | **works** — "Your favourite colour is **teal**! You told me that just a moment ago in this conversation." | +| Recall in a **different** session | **`NO RECORD`** | + +Events really are being written — `list-sessions` showed all three session ids under +the actor. What is missing is the strategy that turns them into recallable facts. The +deployed memory has exactly one, and it is not a semantic one: + +```bash +aws bedrock-agentcore-control get-memory --memory-id "$MEMORY_ID" \ + --query 'memory.strategies[].{type:type,status:status}' +# → [{"type": "USER_PREFERENCE", "status": "ACTIVE"}] +``` + +Semantic fact extraction is added only when `use_long_term_memory=true`, which +defaults to **`false`** (`app.py:142-143`) because it costs more. So: + +```bash +USE_LONG_TERM_MEMORY=true AGENT_PATTERN=langgraph-agent \ + ./scripts/deploy.sh deploy --module A --module 6 +``` + +Be careful with the accelerator's own module-A narration, which says agents "keep +context **across sessions**." True with the flag on; not true of what the guided run +deploys by default. Demo within one session, or turn the flag on beforehand. + +### The Memory actor is the caller's `sub` — which collapses for M2M callers + +The verified `sub` becomes the AgentCore Memory `actor_id`. For a Cognito +`client_credentials` token there is no human subject, so `sub` is the **app client +id** — and `scripts/invoke.py` uses exactly that flow. Measured: + +```bash +aws bedrock-agentcore list-actors --memory-id "$MEMORY_ID" +# → actorId: 39p8o8b978f15rf1932c6lot0g ← identical to /auth/m2m-client-id +# → actorId: test-user-12345 ← written by test_memory.py +``` + +**Every `invoke.py` call in the room shares one actor.** That is fine for a demo and +wrong for any statement about per-user isolation: you cannot show tenant separation +through `invoke.py`, because there is only one tenant in the data. Demonstrating the +boundary needs two real user tokens (`3LO`/federated sign-in), not two `--session` +ids. See `patterns.md` → Tenant isolation. + +Plus the two A2A sub-agents, which are not orchestrator patterns: +`code-agent` and `research-agent` (both `A2A`, both default +`us.anthropic.claude-sonnet-4-6`). + +### Model override + +`MODEL_ID` unset means each pattern falls back to its own in-code +`DEFAULT_MODEL_ID` above. Those are dated ids, and dated ids eventually get +marked Legacy and rejected in fresh accounts — so in a new account, override with +a current cross-region inference profile: + +```bash +export MODEL_ID=us.anthropic.claude-sonnet-5 +``` + +`MODEL_ID` applies to **all** agents in the deployment, not per pattern. + +### The default has no tools, on purpose + +`orchestrator` ships toolless and extracts no caller identity. Asking it "what +tools do you have?" correctly returns nothing — that is not a broken deployment. +Route tool questions to the gateway (`invoke.py --tools`, `test_gateway.py`), or +deploy a tool-consuming pattern. + +**Every pattern shares `agent-code/` but builds its own Dockerfile, so a green +deploy of one proves nothing about the others.** Run the matrix before a release +(see the bottom of this file) — it has caught a stale container image, a missing +forwarded header, and two missing runtime-role permissions, none of which fail at +synth or deploy time. + +--- + +## Protocols, and how each is invoked + +`infra_utils/runtime_protocol.py` decides the protocol from the pattern, and the +protocol decides both the authorizer and the client call. + +| Protocol | Who | Inbound auth | Invoke with | +|---|---|---|---| +| `HTTP` | orchestrator patterns | Bearer JWT (CUSTOM_JWT authorizer) | `invoke.py "…"` | +| `AGUI` | `agui-*` patterns | Bearer JWT | `invoke.py --agui "…"` | +| `MCP` | an MCP-server runtime | Bearer JWT | — | +| `A2A` | `code-agent`, `research-agent` | **SigV4**, IAM `InvokeAgentRuntime` | `invoke.py --a2a "…"` | + +A JWT authorizer is attached **only** when the protocol is client-facing (`HTTP`, +`MCP`, `AGUI`) **and** a Cognito issuer was supplied. The `Authorization` header +allowlist is gated on exactly the same condition, because the control plane +enforces it: creating a runtime that allowlists `Authorization` without a +`customJWTAuthorizer` is rejected with a validation error. + +Consequence worth internalising: **an A2A runtime never sees an `Authorization` +header at all.** Its agent cannot require a caller JWT; IAM guards it instead. +Sending a bearer token to an A2A runtime is rejected exactly like sending SigV4 to +the orchestrator — `Authorization method mismatch` cuts both ways. + +Always prefer `scripts/invoke.py`: it picks the right mechanism per target. + +--- + +## A2A — agent to agent + +Enabled by `enable_a2a=true`. `--module 8` exports it automatically because the +sub-agent stacks do not exist in the CDK app otherwise. + +Sub-agents speak **JSON-RPC 2.0 on `0.0.0.0:9000`** with a `message/send` +envelope. The contract, served by `agent-code/shared/a2a_serve.py`: + +| Endpoint | Method | Returns | +|---|---|---| +| `/` | POST | JSON-RPC | +| `/.well-known/agent-card.json` | GET | the agent card | +| `/ping` | GET | `{"status": "Healthy"}` | + +**A sub-agent that returns HTTP 424 is serving the wrong protocol — it is not a +payload problem**, whatever the repo's `docs/TROUBLESHOOTING.md` says. Measured: +sending the plain `{"prompt": ...}` shape to a working `code-agent` returns **HTTP +200** with a JSON-RPC `-32600 "Request payload validation error"` body, not a 424. +424 means the image serves HTTP `/invocations` on 8080 while the runtime is +registered `A2A` — check all three endpoints on port 9000 and rebuild. + +```bash +.venv/bin/python scripts/invoke.py --a2a code-agent "Reply with exactly: A2A OK" +.venv/bin/python scripts/invoke.py --a2a research-agent "…" +``` + +Why this design rather than one big agent: specialized agents get **independent +auth, scaling, and lifecycle**. That is the architectural argument to make in a +session — the code-agent can be rate-limited, redeployed, or IAM-restricted +without touching the orchestrator. + +--- + +## AG-UI + +The `agui-*` patterns speak the AG-UI protocol — typed SSE events, intended for +building a UI on top of the agent rather than consuming a single response body. +They report protocol `AGUI` and **must** be invoked with `--agui`; calling them +without it fails on the protocol, not on auth. + +`agui-langgraph-agent` has a noticeably slower first response because it builds +its graph per request. Expected. + +--- + +## Caller identity + +`agent-code/shared/auth.py` (`extract_user_id_from_context`) verifies the token +signature against the issuer's published JWKS, pinned to `RS256`, then +`agent-code/shared/jwt_claims.py` checks `iss`, the client, and the presence of +`sub`. **Identity is the `sub` claim.** Every failure path raises — there is no +fallback to an unverified decode. + +| Condition | Result | +|---|---| +| No `Authorization` header | reject | +| `COGNITO_ISSUER_URL` unset | reject, *before* decoding | +| Bad signature, expired, unknown `kid`, JWKS unreachable | reject | +| `iss` mismatch | reject | +| Client not in `COGNITO_ALLOWED_CLIENTS` | reject | +| `COGNITO_ALLOWED_CLIENTS` **empty** | **accepted** — issuer-only pinning | +| No `sub` | reject | + +The asymmetry in the last two rows matters: a missing issuer is a hard reject, +but a missing client allowlist degrades quietly to "any client of the correct +issuer." **Set both.** `token_use` is deliberately not checked, so both Cognito +access tokens and ID tokens are accepted as identity. + +**Why check twice** — AgentCore Runtime's `CUSTOM_JWT` authorizer already +validated the token before the container saw it. The agent validates it again so +the check travels with your code rather than depending on how the runtime happens +to be deployed. Runtime and Gateway have **independent** authorizers; configuring +one says nothing about the other. + +**M2M callers:** Cognito `client_credentials` access tokens carry `client_id` +instead of `aud`. Checking only `aud` would reject machine callers, including +`scripts/invoke.py`. So `jwt_claims.py` prefers `aud` when present (first element +of a list) and falls back to `client_id`, and `auth.py` passes `verify_aud: False` +to validate the audience itself. + +**Where the identity goes:** the verified `sub` becomes the AgentCore Memory +`actor_id` — the tenant boundary for stored conversation history. An unverified or +defaulted identity would file different callers under one actor and mix their +history, which is why agents reject rather than substitute a placeholder. + +**Injection detail worth knowing when extending:** `COGNITO_ISSUER_URL` and +`COGNITO_ALLOWED_CLIENTS` are injected by **`app.py`**, not by `RuntimeStack`, and +today **only for the orchestrator runtime**. `RuntimeStack`'s own `env_vars` carry +only `PROJECT_NAME`, `ENVIRONMENT`, `COMPONENT_NAME`, `AWS_REGION_NAME`, +`SOURCE_HASH`. The A2A runtimes receive neither variable — harmless only because +their entrypoints take no `RequestContext` and never call the helper. **Any agent +you add there that does call it will fail closed until `app.py` passes the issuer +through.** + +JWKS is fetched on the first verified request and cached by `PyJWKClient`, which +refreshes on a 5-minute lifespan and re-fetches on an unknown key id — so Cognito +key rotation is handled. No explicit timeout is set, so a hung JWKS endpoint +stalls the request for PyJWT's 30-second default before failing closed. + +An agent that imports `shared/` must also `COPY shared/ shared/` in its +Dockerfile. + +--- + +## Outbound auth — agent to gateway + +Separate mechanism, same file. `get_gateway_access_token()` uses: + +```python +@requires_access_token(provider_name=os.environ["GATEWAY_CREDENTIAL_PROVIDER_NAME"], + auth_flow="M2M") +``` + +The decorator is evaluated **at import time** and `provider_name` defaults to +`""`, so a runtime without that variable binds an empty provider name at module +load. That is the mechanism behind "silently no tools." The A2A runtimes +intentionally do not receive it. + +Flow: runtime → AgentCore Identity `gateway-m2m` credential provider (Token +Vault) → `client_credentials` against Cognito → gateway JWT back to the runtime → +runtime calls the gateway over MCP with that JWT → gateway invokes its Lambda tool +targets. + +--- + +## Adding a tool to the gateway + +This is module 7's substance. **Agents pick up new tools on their next discovery, +with no agent redeploy** — worth demonstrating rather than asserting. + +Two kinds of target: + +| | Built-in connector | Lambda target | +|---|---|---| +| You write | nothing | a handler + tool schema | +| Good for | capabilities AWS operates (web search) | your APIs, data, business logic | +| Credentials | the gateway's IAM role | whatever your Lambda needs | +| Example | `web-search` in `stacks/gateway_stack.py` | `sample-tool` in `tools/sample_tool/` | + +### Built-in connector + +Registered via `CfnGatewayTarget` with +`credentialProviderConfigurations: [{"credentialProviderType": "GATEWAY_IAM_ROLE"}]` +and `target_configuration={"mcp": {}}`, then the connector itself set through +`add_property_override("TargetConfiguration.Mcp.Connector", {...})`. + +Three ways this goes wrong: + +1. **Passing the connector in `target_configuration` instead of + `add_property_override`.** The L1 construct's property mapping predates + connector targets and **silently drops the key** — the target deploys with no + connector and the tool never appears. A repo test guards this. +2. **Missing the connector's own IAM action on the gateway role.** Web search + needs `bedrock-agentcore:InvokeWebSearch` on + `arn:aws:bedrock-agentcore::aws:tool/web-search.v1` — note the literal + `aws` where an account id would normally be. Without it the target deploys and + every call fails at invoke time. +3. **Forgetting connectors are regional.** `app.py` gates web search on + `WEB_SEARCH_REGIONS` = `us-east-1`, `eu-west-1`, `ap-northeast-1` and turns it + off elsewhere rather than failing the deploy. Do the same for anything you add. + +### Lambda target + +The handler receives the tool name in the **context**, not the event: + +```python +def handler(event, context): + delimiter = "___" + tool_name = context.client_context.custom["bedrockAgentCoreToolName"] + tool_name = tool_name[tool_name.index(delimiter) + len(delimiter):] + + if tool_name == "my_tool": + return {"content": [{"type": "text", "text": do_the_thing(event.get("some_arg", ""))}]} + return {"error": f"Unsupported tool: {tool_name}"} +``` + +Arguments arrive as top-level keys in `event`. Return +`{"content": [{"type": "text", "text": ...}]}` on success, `{"error": "..."}` on +failure. One Lambda can serve several tools — dispatch on the suffix after `___`, +not the full name. + +Declare the schema in `app.py`'s `tool_configs` beside `sample-tool`, using +**PascalCase** keys — this is the CloudFormation shape, not MCP JSON: + +```python +"my-tool": { + "source_dir": "tools/my_tool", + "env_vars": {}, + "tool_schema": [ + { + "Name": "my_tool", + "Description": "What it does — the agent reads this to decide when to call it.", + "InputSchema": { + "Type": "object", + "Properties": {"some_arg": {"Type": "string", "Description": "…"}}, + "Required": ["some_arg"], + }, + }, + ], +}, +``` + +The gateway stack does the rest: creates the Lambda from `source_dir`, grants the +gateway permission to invoke it, registers the target. + +**Write the `Description` for a model, not for a human skimming a table.** It is +the only thing the agent has when deciding whether this tool answers the +question. "Agent never calls the tool" is usually a vague description — or the +`orchestrator` pattern, which has no tools at all. + +### Verify, then prove an agent uses it + +```bash +./scripts/deploy.sh deploy --module 7 +.venv/bin/python scripts/test_gateway.py # tools/list + one tools/call +.venv/bin/python scripts/invoke.py --tools # your tool should be listed + +AGENT_PATTERN=strands-agent ./scripts/deploy.sh deploy --module 6 +.venv/bin/python scripts/invoke.py "Use my_tool on '…' and report what it returns." +``` + +### Tool naming + +Tools are `___` — e.g. `sample-tool___text_analysis_tool`, +`web-search___WebSearch`. That full string is what `--tools` prints, what agent +prompts refer to, and what **Cedar policies must name**. A new tool is denied once +`cedar_mode=ENFORCE` unless a permit names it. + +--- + +## The pattern matrix + +Run before a release, or when a customer asks "does this really work with our +framework." One orchestrator redeploy per pattern, ~5–8 minutes each: + +```bash +export AWS_REGION=us-east-1 + +for p in orchestrator strands-agent langgraph-agent claude-sdk-agent claude-sdk-multi-agent; do + AGENT_PATTERN=$p NON_INTERACTIVE=1 ./scripts/deploy.sh deploy \ + --stack $PREFIX-runtime-orchestrator + .venv/bin/python scripts/invoke.py "Reply with exactly: $p LIVE" +done + +for p in agui-strands-agent agui-langgraph-agent; do + AGENT_PATTERN=$p NON_INTERACTIVE=1 ./scripts/deploy.sh deploy \ + --stack $PREFIX-runtime-orchestrator + .venv/bin/python scripts/invoke.py --agui "Reply with exactly: $p LIVE" +done +``` + +Then assert more than "it answered": + +```bash +# Right pattern, right protocol, fresh image +ARN=$(aws ssm get-parameter --name /$PROJECT_NAME/$ENVIRONMENT/runtimes/orchestrator/arn \ + --query Parameter.Value --output text) +aws bedrock-agentcore-control get-agent-runtime --agent-runtime-id "${ARN##*/}" \ + --query '[agentRuntimeVersion,protocolConfiguration,agentRuntimeArtifact]' + +# Each pattern pushed its own tag — identical tags mean no rebuild happened +aws ecr describe-images --repository-name $PREFIX-orchestrator \ + --query 'sort_by(imageDetails,&imagePushedAt)[-5:].{tags:imageTags,pushed:imagePushedAt}' + +# Tools actually loaded (exercises the gateway MCP client + token vault) +.venv/bin/python scripts/invoke.py "List the names of the tools you have available. Names only." +# Expect e.g. sample-tool___text_analysis_tool, execute_python_securely +``` + +Close the loop on observability afterwards — this is what makes module 9's +"end-to-end traces" claim true, reusing a span from the invokes above: + +```bash +.venv/bin/python scripts/check_observability.py --spans +``` diff --git a/kiro/agentcore-enterprise-platform/steering/deploy.md b/kiro/agentcore-enterprise-platform/steering/deploy.md new file mode 100644 index 0000000..d4f15be --- /dev/null +++ b/kiro/agentcore-enterprise-platform/steering/deploy.md @@ -0,0 +1,770 @@ +# Deploying + +Read this when choosing how to run the accelerator, scoping a deploy, resolving +where a config value came from, or tearing down. + +All commands run from the root of a `sample-agentcore-enterprise-platform` +checkout, with `.venv` created and `pip install -r requirements.txt` done. +`AWS_REGION` and credentials must be set for anything that touches AWS. + +--- + +## The four ways to run it + +```bash +./scripts/deploy.sh workshop # guided: explain → deploy → verify → pause, per module +./scripts/deploy.sh deploy --module 5 # one module's stacks +./scripts/deploy.sh deploy --team agent # one team's stacks +./scripts/deploy.sh deploy --profile greenfield # profile FLAGS + cdk deploy --all +``` + +`workshop` is the right default for a first pass or a facilitated session: it +narrates each module, deploys it, runs its verify, and waits for you before +moving on. + +### A profile is a preset file, and `--profile` writes it to disk + +`--profile X` copies `presets/X.yaml` over `platform.yaml` — the durable +deployment manifest — before the config is loaded, so it participates with the +normal precedence (env > `platform.yaml` > `workshop.env`). `materialize_preset` +at `scripts/deploy.sh:107-129` does the copy; the argv pre-scan at +`scripts/deploy.sh:134-146` is what makes it happen early. + +Three things follow, and all three surprise people: + +- **It runs on every action except `config`, and is suppressed only by + `--dry-run`.** So `ls --profile greenfield` is *not* a read-only measurement + any more — it rewrites `platform.yaml`. Verified. +- **The intent is now durable.** `deploy --profile greenfield` followed by a + plain `deploy` keeps greenfield's posture, because the manifest is still on + disk. Previously the flags died with the run and the plain re-run quietly + picked the app's own defaults back up — A2A among them, which is two extra + runtimes and their CodeBuild projects. +- **A hand-edited manifest is protected.** `platform.yaml` is regenerable only + while it carries the generated-from header. Remove the header (or write the + file yourself) and `--profile` refuses rather than overwriting: + `Refusing to overwrite a hand-edited manifest`. `--yes` overrides, with a + warning. Dropping `--profile` is usually the right answer — the file already + *is* the config. + +### The `--profile` scope trap + +Scope is a separate question from posture, and `--profile` still does not narrow +it: with no `--stack`/`--team`/`--module`, the stack list is **empty**, which +means `cdk deploy --all` (`scripts/deploy.sh:1067-1078`) — every stack the +manifest makes the app synthesize, not just the modules in that profile's +sequence. `PROFILE_MODULES` (`scripts/deploy.sh:267-276`) is read by the +`workshop` action and nothing else. + +```bash +./scripts/deploy.sh deploy --profile platform-team # deploys EVERYTHING +./scripts/deploy.sh workshop --profile platform-team # walks 3 4 5 A 6 7 8 9 C E +./scripts/deploy.sh deploy --profile platform-team --module 5 # manifest + just the gateway +``` + +Interactively this is no longer silent: before `--all`, `confirm_footprint` +(`scripts/deploy.sh:658-679`) prints the account, region, prefix, which config +source won, and the full `cdk ls` output, then blocks on `Proceed to deploy ALL +of the above? [y/N]`. **`--yes` and `NON_INTERACTIVE=1` both skip that prompt**, +so the trap is fully live on the CI path the README documents. + +**To measure the footprint without deploying and without writing the manifest, +ask the contract instead of the CLI:** + +```bash +.venv/bin/python -c " +from infra_utils.platform_config import load_platform_config +c = load_platform_config('presets/greenfield.yaml') +print(len(c.expected_stacks()), c.expected_stacks())" +``` + +`expected_stacks()` (`infra_utils/platform_config.py:295`) is the deployment +contract — deploy plans, verification, the dashboard and destroy all consume it, +and `scripts/check-contract.sh` fails CI if it drifts from what `app.py` actually +synthesizes. Measured across the five presets: + +| `--profile` | Stacks `deploy --all` would create | Overshoot vs the profile's modules | +|---|---|---| +| `greenfield` | 6 — auth, identity, **memory**, gateway, runtime-orchestrator, observability | `-memory` is module **A**, which greenfield never walks | +| `migration` | 6 — same six | `-memory` again; module 5 (gateway) is not in the sequence either | +| `multi-agent` | 8 — the six plus code-agent and research-agent | `-memory` | +| `platform-team` | 10 — networking, security, auth, identity, memory, gateway, 3 runtimes, observability | none; it is the whole app | +| `security-focused` | 8 — networking, security, auth, identity, memory, gateway, orchestrator, observability | `-memory` | + +Two nuances worth being precise about, because overstating this costs credibility: + +- **The manifest does gate the expensive stacks.** `greenfield` leaves + `security: {}`, and `security.networking` defaults to `false` + (`infra_utils/platform_config.py:179`), so `-networking` is not in the app at + all and `deploy --profile greenfield` cannot create a NAT gateway. The + overshoot is Memory, which is cheap. +- **What you actually lose is the walk**, not money: no per-module pause, no + verify step between layers, and one failure anywhere in `--all` leaves you + diagnosing a partially-deployed platform instead of a single module. + +--- + +## Actions and flags + +| Action | Does | +|---|---| +| `deploy` | deploy the selected stacks | +| `workshop` | guided module-by-module walk of a profile's sequence | +| `verify` | run every check this configuration promises; **exits non-zero on failure** (`verify.md`) | +| `destroy` | destroy selected stacks; no selection means `--all` | +| `synth` | `cdk synth` with the resolved context | +| `diff` | `cdk diff` with the resolved context | +| `export` | write `workshop-outputs-.json` (SSM params + stack outputs) | +| `ls` / `list` | `cdk ls` — the stacks the app currently defines | +| `config` | print saved answers | +| `config --reset` | delete saved answers | + +`verify` is missing from the script's own one-line `Usage:` string +(`scripts/deploy.sh:1177`) while being described in the `Actions:` block two lines +below it. The action works; do not conclude from `--help` that it does not exist. + +| Flag | Applies to | Does | +|---|---|---| +| `--stack NAME` | deploy, destroy, synth, diff | one named stack | +| `--profile P` | any action but `config` | **writes `presets/P.yaml` to `platform.yaml`**; also picks the sequence for `workshop` | +| `--team T` | deploy | `platform` \| `agent` \| `security` | +| `--module N` | deploy | `3 4 5 6 7 8 9 A B C D E` | +| `--from M` | workshop | skip forward to module M in the sequence | +| `--dry-run` | workshop **only** | print the plan; **zero AWS calls**, and the one thing that suppresses materialization. Accepted and silently ignored by `deploy` — see below | +| `--yes` | deploy, destroy | skip the full-footprint confirmation; overwrite a hand-edited `platform.yaml`; make the post-destroy sweep delete rather than ask | + +`NON_INTERACTIVE=1` skips every prompt — required for CI, and it turns some +prompts into hard failures (see ORG_ID below). It implies `--yes` for the +footprint confirmation but **not** for the post-destroy sweep, which reports and +leaves instead (`runbook-teardown-platform.md`). + +### The parser fails closed — spelling is no longer the risk + +This used to be the sharpest edge in the script: the argument loop ended in +`*) shift ;;`, so an unrecognised flag was dropped without a warning, which left +the stack list empty, and empty meant `cdk deploy --all`. A typo produced a +*larger* deployment than the one you asked for. + +Upstream closed it (`scripts/deploy.sh:955-990`). Verified, all five exit 1: + +| Typed | Now | +|---|---| +| `deploy --dryrun` | `Unknown option or argument: '--dryrun'` + the valid list | +| `deploy --modul 6` | same | +| `deploy --stack=identity` | same, plus `values are space-separated` | +| `deploy --team platfrom` | `Unknown team: 'platfrom'. Valid teams: security agent platform` | +| `deploy --profile greenfeld` | `Unknown profile: 'greenfeld'. Valid profiles: …` | + +`--profile` and `--team` values are now validated for **every** action, not just +`workshop`, and `require_flag_value` catches a flag given no value at all. The +asymmetry that used to matter — unknown *values* rejected, unknown *flag names* +ignored — is gone. + +Confirm you are in dry-run from the output, which says so explicitly, rather than +from what you typed: + +``` +[INFO] Dry run: skipping prerequisite and credential checks (no AWS calls) +[INFO] DRY RUN — nothing will be deployed +``` + +If those lines are missing, you are deploying. + +### `deploy --dry-run` is not a dry run — it deploys + +Now that spelling fails closed, this is the surviving trap in the flag parser — +and the nastier one, because you can type it perfectly and still be wrong. +`--dry-run` is parsed globally, but only the `workshop` action ever reads it. The +`deploy` case (`scripts/deploy.sh:1052-1081`) checks `NON_INTERACTIVE`, then calls +`cdk_bootstrap` and `deploy_stacks` with **no `DRY_RUN` test anywhere** — compare +`workshop`, where every one of those steps is guarded. + +Measured, and this is what it cost: `deploy --module 3 --dry-run` pointed at an +untouched Region ran to `Total time: 58.51s` and left two real stacks standing — +`CDKToolkit` (the bootstrap: staging bucket, ECR repo, five IAM roles) and +`-auth` (a live Cognito user pool, three app clients, a hosted domain). It printed +the deployment summary table, and with it the M2M client secret. Nothing warned. + +```bash +# preview a deploy without deploying: +./scripts/deploy.sh workshop --dry-run --profile greenfield # free, safe +./scripts/deploy.sh synth --module 3 # renders templates, no deploy +./scripts/deploy.sh diff --module 3 # needs an existing stack +``` + +`synth` and `diff` do run the credential check (they call CDK), but neither +creates anything. There is no dry run for `deploy`; use `workshop` when you want +one. + +### `workshop --dry-run` really is free + +`workshop --dry-run` skips the prerequisite check, the credential check, the +venv check, and CDK bootstrap entirely (`scripts/deploy.sh:819-826` — the skip is +conditional on `ACTION = workshop`, which is why `deploy` does none of it). It +prints, per module, the stacks it would deploy and the exact verify command it +would run. Use it to preview a profile, to confirm a config change landed, and as +the first thing in any session. + +**One exception, and it bites when you are preparing:** the `ORG_ID` gate runs +*before* the plan is printed and applies in dry-run too. So +`workshop --dry-run --profile security-focused` produces no plan at all unless +`ORG_ID` is set — with a terminal it stops and prompts, and with stdin closed it +warns and exits 1. To preview that profile's plan without an Organizations id, +any `o-…`-shaped value works, because dry-run never calls AWS: + +```bash +ORG_ID=o-preview0 ./scripts/deploy.sh workshop --dry-run --profile security-focused +``` + +Use a real id for anything that actually deploys. + +### Resuming + +```bash +./scripts/deploy.sh workshop --from 6 +``` + +`--from` is validated against the selected profile's sequence and fails with the +full sequence printed if the module is not in it. Modules before it are logged +as skipped, not silently dropped — you get one `Skipping module N (--from 6)` line +each. + +**A resumed run still prints the earlier stacks, and that is not a re-deploy.** +Module 6's `cdk deploy` names its dependencies, so `--from 6` walks `-auth`, +`-identity` and `-gateway` on the way through. Each comes back +`✅ (no changes)` with `Deployment time: 0s`; the whole detour cost ~18 +seconds in a real run. Read the `(no changes)` before concluding that `--from` was +ignored. + +If a deploy died mid-stack, just re-run the same command — CDK picks up from the +current stack state, and a stack in `UPDATE_ROLLBACK_COMPLETE` is safe to deploy +onto again. + +--- + +## Profiles + +Each profile is two things: a **flag set** and a **module sequence**. `deploy` +uses only the flags; `workshop` uses both. + +| Profile | Flags it sets | Sequence | +|---|---|---| +| `greenfield` | `enable_networking=false enable_security=false enable_a2a=false` | 3 4 5 6 9 | +| `migration` | `enable_networking=false enable_security=false enable_a2a=false` | 3 4 6 7 9 | +| `multi-agent` | `enable_networking=false enable_security=false enable_a2a=true` | 3 4 5 6 7 8 9 | +| `platform-team` | `enable_networking=true enable_security=true enable_a2a=true` | 3 4 5 A 6 7 8 9 C E | +| `security-focused` | `enable_networking=true enable_security=true enable_a2a=false enable_resource_policies=true enable_egress_filter=true enable_cedar=true enable_traceability=true` | 3 4 5 6 9 E | + +Two orderings that look wrong and are deliberate: + +- **`platform-team` runs A (Memory) before 6 (Agent Deployment).** The + orchestrator depends on memory (`app.py`: `runtime_orchestrator.add_dependency( + memory_stack)`). If 6 ran first, CDK would create memory implicitly and module A + would report "no changes" — teaching the room something false about what they + just built. + + The corollary matters for **every** profile: a `-memory` stack and a live Memory + resource land at module 6 whether or not A is in the sequence. A verified + `greenfield` run leaves **six** stacks for five modules — `-auth`, `-identity`, + `-gateway`, **`-memory`**, `-runtime-orchestrator`, `-observability` — and + `deploy.sh verify` duly runs `test_memory.py` against it. Module A is where Memory gets + *introduced and verified*, not where it comes from. Count it in teardown. +- **`security-focused` sets `enable_a2a=false`.** It is about the control plane, + not the agent fleet. + +`migration` skips module 5 and reaches the gateway through module 7 instead — +the point being that an existing agent gets governed by adding tool targets, not +by rebuilding. + +### `security-focused` needs an Organizations ID + +`enable_resource_policies=true` renders `aws:PrincipalOrgID` into the Memory +resource policy, so it cannot synthesize without one. + +```bash +export ORG_ID=$(aws organizations describe-organization --query Organization.Id --output text) +``` + +Behaviour when it is missing: + +- **Interactively** the script *prompts* for an `o-xxxx` value and only fails if + you leave it empty. +- **With `NON_INTERACTIVE=1`** it is a hard stop. + +Separately, `enable_networking=true` without `ORG_ID` does **not** fail — it +warns and creates the AgentCore VPC endpoint with **no policy at all**. That is +a quiet loss of a control, so treat the warning as an error in any environment +that matters. + +--- + +## Teams + +For a session split across workstreams: + +| Team | Stacks (`PREFIX` = `${PROJECT_NAME}-${ENVIRONMENT}`) | +|---|---| +| `platform` | `-networking -auth -identity -gateway -observability` | +| `agent` | `-runtime-orchestrator -runtime-code-agent -runtime-research-agent -memory` | +| `security` | `-security -observability` | + +Note `-observability` is in both `platform` and `security` — deliberate overlap, +and harmless because CDK is idempotent. + +--- + +## Where config values come from + +Precedence, highest first: + +1. **Explicit environment variables** — `AWS_REGION=…`, `AGENT_PATTERN=…` +2. **`platform.yaml`** — applied before saved answers are loaded +3. **`workshop.env`** — answers saved by a previous interactive run +4. **Interactive prompts**, then in-code defaults + +Ten keys are persisted to `workshop.env`: `AWS_REGION`, `IDP_TYPE`, +`IDP_TENANT_ID`, `IDP_CLIENT_ID`, `IDP_ISSUER_URL`, `MODEL_ID`, `ORG_ID`, +`PROJECT_NAME`, `ENVIRONMENT`, `AGENT_PATTERN`. **Secrets are never persisted.** + +Defaults: `PROJECT_NAME=agentcore-workshop`, `ENVIRONMENT=dev`, +`AWS_REGION=us-east-1`. So the default stack prefix is `agentcore-workshop-dev`. + +When answers seem stale — and they will, because the wizard remembers: + +```bash +./scripts/deploy.sh config # effective values, with their source +./scripts/deploy.sh config --reset # delete workshop.env +``` + +`config` needs no credentials and makes no AWS calls. It prints a +`platform.yaml → effective values` block if that file exists, then the raw +contents of `workshop.env` if that exists — and on a fresh checkout, where +neither does, it prints exactly `none`. That is the correct answer, not a failure. + +`config --reset` deletes **`workshop.env` only**. It does not touch +`platform.yaml`, so if a value survives a reset, that file is where it lives. + +### `platform.yaml` + +Declare the whole deployment in one file instead of environment variables: + +```yaml +project: acme-agents +environment: dev +region: us-east-1 +deployment: + strategy: centralized # centralized | distributed | federated +identity: + idp: cognito +agents: + pattern: langgraph-agent + a2a: true + memory: + long_term: true +gateway: + web_search: auto + tools: [sample-tool] +security: + networking: true + cloudtrail_alerting: true + traceability: true +observability: + transaction_search: true +``` + +`presets/` holds one starting file per profile (`greenfield.yaml`, +`migration.yaml`, `multi-agent.yaml`, `platform-team.yaml`, +`security-focused.yaml`). Validate offline before deploying: + +```bash +.venv/bin/python -m infra_utils.platform_config platform.yaml +``` + +Validation is deliberately strict and reports **every** problem at once, before +any AWS call. **Unknown keys are errors, not no-ops** — a typo would otherwise +silently do nothing. An invalid file hard-stops the deploy. + +Secrets never go in this file: it holds Secrets Manager *names*, never values. + +Two facts about the file itself, both easy to trip over: + +- **It is gitignored.** It is per-deployment state, not source. If you want to + commit a team's manifest, `git add -f`. +- **`--profile` regenerates it,** but only while the generated-from header is + intact. Delete the header to protect your edits; then `--profile` refuses + instead of overwriting, and `--yes` is the override. + +### Adding your own stack: the use-case extension point + +There is a first-class seam for this now — you do not need to patch `app.py`. +A use case is a directory under `use-cases//` with a `manifest.yaml`, and +the manifest is the only file the platform reads about it +(`infra_utils/platform_config.py:213-249`): + +```yaml +name: hello-platform # must equal the directory name +owner: # who reviews changes to this folder +summary: +requires: [gateway] # core stack suffixes that must be in the footprint +stacks: [uc-hello-platform] # suffixes this adds; the uc- prefix is enforced +entry: stack.py # module exposing build(app, platform_ctx, config) +``` + +Enable it by listing the name under `use_cases:` in the manifest. Four guardrails +worth knowing, because each one is a hard error rather than a surprise later: + +- `stacks:` suffixes **must** start with `uc-`, so a contribution cannot collide + with or masquerade as a core stack. +- `requires:` is checked against the footprint **per federation role**, so a + gateway-requiring use case cannot select itself into a federated workload + account that has no local gateway. +- `extra: forbid` on the manifest model — an unknown key fails to load. +- A broken manifest is a hard error naming the file; a contribution never + half-loads. + +`use-cases/hello-platform/` is the reference implementation and the thing to copy. +`CONTRIBUTING_USE_CASES.md` and `docs/PLATFORM_INTERFACE.md` are the contract. +This is the answer to "how do we add our own integration without forking" — +previously that meant reading platform values out of SSM and deploying a separate +app, which still works and is described under **Multi-account** below. + +--- + +## Feature flags + +Set as CDK context (`-c flag=value`) or as an environment variable +(`FLAG=value`, uppercased). + +| Flag | Env | Default | Effect | +|---|---|---|---| +| `agent_pattern` | `AGENT_PATTERN` | `orchestrator` | which agent framework builds | +| `enable_a2a` | `ENABLE_A2A` | true at app level; profiles override | A2A sub-agent stacks exist at all | +| `enable_networking` | `ENABLE_NETWORKING` | false | VPC + private subnets + AgentCore endpoints; runtimes get `network_mode: VPC` | +| `enable_security` | `ENABLE_SECURITY` | false | KMS CMK + CloudTrail; prerequisite for `enable_traceability` | +| `enable_resource_policies` | `ENABLE_RESOURCE_POLICIES` | false | Memory resource policy; **requires `org_id`** | +| `enable_egress_filter` | `ENABLE_EGRESS_FILTER` | false | Bedrock Guardrail + egress interceptor Lambda on the gateway | +| `enable_cedar` | `ENABLE_CEDAR` | false | Cedar policy engine on the gateway | +| `cedar_mode` | `CEDAR_MODE` | `LOG_ONLY` | `LOG_ONLY` or `ENFORCE` | +| `enable_traceability` | `ENABLE_TRACEABILITY` | false | SNS + EventBridge alerting on sensitive AgentCore API calls | +| `enable_transaction_search` | `ENABLE_TRANSACTION_SEARCH` | **true** | X-Ray trace segment destination → CloudWatch Logs. Account- and Region-scoped | +| `enable_web_search` | — | auto by Region | built-in web-search gateway connector | +| `use_long_term_memory` | — | false | long-term memory + semantic extraction (costs more) | +| `org_id` | `ORG_ID` | unset | `o-xxxx`, required by the resource policy and the VPCE policy | +| `model_id` | `MODEL_ID` | unset | Bedrock model override; unset means each pattern's in-code default | + +Two flags that trip people up: + +- **Flags are matched against the exact lowercase string `"true"`.** + `-c enable_cedar=True` silently does nothing. +- **`enable_transaction_search` defaults to `true`** because without it every + OTLP span batch is rejected with HTTP 400 while the deploy still reports + success. It is an account+Region setting and **survives teardown** by design. + +### The Region gate nobody expects + +`enable_web_search` is auto-enabled only in `us-east-1`, `eu-west-1`, and +`ap-northeast-1`, and off elsewhere. Creating the target in an unsupported +Region **fails the deploy**, so the gate is what keeps other Regions working. +Any connector you add yourself needs the same treatment. + +The interactive Region prompt offers: `us-east-1`, `us-west-2`, `eu-west-1`, +`eu-central-1`, `ap-northeast-1`, `ap-southeast-1`. + +**Nothing stops you from deploying into a Region where AgentCore does not exist.** +The only Region check is `sts get-caller-identity`, which succeeds anywhere the +Region is enabled. Measured in `eu-west-3`: every prerequisite passed, the +prerequisite banner printed `Region: eu-west-3`, and module 3 deployed a real +Cognito pool in 58s. The failure would not have surfaced until a module that calls +the AgentCore control plane — modules 5, A, 6 — by which point identity is already +standing in the wrong Region. With `NON_INTERACTIVE=1` there is no prompt to catch +it either, so the Region comes from `AWS_REGION` unvalidated. Confirm first: + +```bash +aws bedrock-agentcore-control list-gateways --region "$AWS_REGION" >/dev/null \ + && echo "AgentCore control plane responds in $AWS_REGION" +``` + +--- + +## Secrets + +- The IdP client secret is written to Secrets Manager as + `${PREFIX}-idp-client-secret` (override the name with + `IDP_CLIENT_SECRET_NAME`). Only the **name** is passed to CDK, resolved with + `{{resolve:secretsmanager:...}}`. +- The script strips whitespace from the pasted secret. This is not cosmetic — a + trailing newline from a copy-paste produces `invalid_client` at the IdP, which + is an hour of debugging that looks like a misconfiguration. +- API-key tool secrets are prompted for `tavily`, `google-search`, `google-maps` + and stored as `${PREFIX}--api-key`. +- Nothing secret is ever written to `workshop.env` or `platform.yaml`. + +--- + +## Prerequisites the script checks + +`node`, `npm`, `python3.13`, `aws` are required. `docker` is reported as +**optional** and the run continues without it — container images build in AWS +CodeBuild. + +The CDK CLI is probed with `npx --no-install cdk --version` and installed with +`npm install -g aws-cdk` if absent. The `--no-install` matters: a bare +`npx cdk --version` prompts "Ok to proceed?" and, with output suppressed and no +TTY, waits forever. + +**The check tests presence, not compatibility** — and that gap is a real failure. +`requirements.txt` pins `aws-cdk-lib>=2.265.0` with no ceiling, so pip installs the +newest library, which emits the newest cloud-assembly schema. The repo ships no +`package.json` and no `node_modules`, so `npx --no-install` resolves the **global** +CDK CLI, which may be months old. The prereq check prints a green `✓ cdk: `, the run proceeds, and bootstrap dies about ten seconds later on a schema +mismatch — under an error message that blames the account and IAM instead. Upgrade +the CLI before a session: + +```bash +npm install -g aws-cdk@latest && cdk --version +``` + +See `troubleshooting.md` → "CDK bootstrap failed". + +CDK bootstrap (`CDKToolkit`) runs automatically for the account/Region. + +### The wall of deprecation warnings is normal + +Every module prints a block of +`[WARNING] aws-cdk-lib.Stack#addDependency is deprecated` — nine at a time, twice +per module — plus, on current `aws-cdk-lib`, a `No cross-stack-reference strength +configured, defaulting to "strong"` annotation. Both come from the library, neither +affects the deploy, and both scroll past immediately before a successful stack. +Say so before it happens, or a room will read it as the deploy failing. + +--- + +## Multi-account + +Set by `deployment.strategy` in `platform.yaml`: + +| | `centralized` | `distributed` | `federated` | +|---|---|---|---| +| Accounts | one | one per team, full copy | one platform + N workload | +| Gateway & tools | local | per-account | **shared** (platform account) | +| Cognito / IdP | local | per-account | **shared** (platform account) | +| Runtimes | local | per-account | workload accounts only | +| Memory | local | per-account | **per-workload** | +| Choose when | workshops, pilots, one team | strong team autonomy | central tool governance, many agent teams | + +`centralized` is the default and what every module assumes. + +**Federated trust is pure OAuth — there is no cross-account IAM on the data +plane.** The workload account's own credential provider holds the platform +Cognito M2M client id + secret; the token vault exchanges them at the platform +Cognito token endpoint; the platform gateway validates the resulting JWT against +its own issuer and cannot tell which account called it. + +The **same `platform.yaml` deploys both sides** — the account you deploy into +decides the role. Platform side gets `auth`, `identity`, `gateway`, +`observability` (no runtimes); workload side gets `identity`, `memory`, +runtimes, `observability`. Deploying from an account in neither list fails at +synth with a message naming both. + +An incomplete `deployment.federation` block raises a `ValueError` naming the four +required keys: `gateway_url`, `issuer_url`, `m2m_client_id`, +`m2m_client_secret_name`. Memory stays per-workload on purpose — conversation +history is the tenant boundary, and account isolation is the strongest wall +available. + +--- + +## What is still billing after everyone goes home + +The usual question at the end of day one is "what does this cost if we leave it up +overnight?" Answer it by meter, not by guess — most of the platform is pay-per-use +and genuinely near-zero idle, and the standing cost is short enough to enumerate. + +**Standing hourly meters — these run whether or not anything is invoked.** Only +`enable_networking=true` creates any of them, which means `greenfield`, `migration` +and `multi-agent` leave **nothing** billing hourly, and `platform-team` and +`security-focused` do: + +| Resource | Count | Meter | +|---|---|---| +| NAT gateway | **1** (`nat_gateways=1`) | per hour, plus per GB processed | +| Interface VPC endpoints | **5** × 2 AZs = **10** endpoint-hours | per endpoint per AZ per hour, plus per GB | +| S3 gateway endpoint | 1 | **free** | + +The five interface endpoints are `bedrock-runtime`, `ecr.api`, `ecr.dkr`, `logs`, +and `bedrock-agentcore.gateway` (`stacks/networking_stack.py:70-105`), and the VPC +is `max_azs=2`, so each one is billed in two AZs. Ten endpoint-AZ-hours plus one +NAT-hour is the whole idle bill, and it is the reason the two networking profiles +carry a cost warning the other three do not. + +There is **no flag that turns the endpoints off.** `enable_vpc_endpoints=True` is +passed as a literal at the call site (`app.py:197`), not read from context, so +`enable_networking=true` always means all five. Nor is there a flag for one AZ — +`max_azs=2` is hardcoded in the stack. If someone needs a cheaper networking +demo, the only levers are a source edit or not enabling networking at all; there +is no `-c` you can hand them. Say that plainly rather than hunting for a flag +during a session. + +**Standing monthly meters — small but nonzero, and they are what makes "we forgot +about it" expensive over a quarter rather than a night:** + +- **KMS CMK** with rotation enabled (`enable_kms`, `stacks/security_stack.py:15-19`) — + per key per month. +- **Secrets Manager** entries created from IdP or API-key prompts — per secret per + month. These are the ones people forget, because nothing in the stack list names + them. +- **ECR image storage** — one arm64 image per agent pattern you built, so a + pattern matrix leaves several. Bounded, though: a lifecycle rule keeps the last + 10 (`stacks/runtime_stack.py:107-109`), and the repository is + `RemovalPolicy.DESTROY` with `empty_on_delete=True` (`:91-92`), so teardown does + remove it. Unlike the log groups below, this one is not an orphan risk. +- **CloudWatch Logs storage** for retained log groups, including the orphans in + "Tearing down" below. +- **S3** for the CloudTrail bucket. + +**Zero idle cost, despite looking expensive:** AgentCore Runtime (per invocation), +Gateway (per request), Code Interpreter (per session), Cognito at workshop user +counts, CodeBuild (per build minute — a spike when you swap patterns, not a +standing charge). + +Two account-scoped items behave differently from everything above and need naming +before you deploy, not in the wrap-up: + +- **Transaction Search** (`enable_transaction_search`, defaults **true**) changes + span-ingestion pricing **account-wide** and **survives teardown** by design. +- **CloudTrail.** The module E trail is management-events-only and single-region + (`is_multi_region_trail=False`, no event selectors), so it is free — **unless the + account already has a management-events trail**, in which case this is the second + copy and every event bills. In a shared account, check first: + + ```bash + aws cloudtrail describe-trails --query 'trailList[].[Name,IsMultiRegionTrail]' --output text + ``` + +So: on `greenfield`, leaving it up overnight costs effectively nothing and the +honest answer is "nothing standing — tear it down tomorrow." On `platform-team`, +say "one NAT gateway and ten endpoint-hours per hour" and price those two lines in +the calculator for the customer's Region. Do not quote dollar figures from memory; +these rates vary by Region and change. + +If the goal is a cheap platform left running between sessions, the lever is +`ENABLE_NETWORKING=false`, which removes every hourly meter at once. + +--- + +## Tearing down + +```bash +./scripts/deploy.sh destroy # everything, cascade intended +./scripts/deploy.sh destroy --stack # one stack — refuses if others depend on it +``` + +A targeted destroy passes `--exclusively`, so CloudFormation **refuses** rather +than cascading into stacks that depend on the target — which would take the +platform out from under other teams. The error names the consumer. That refusal +is the safety feature, not a bug. + +### A destroy stops at the first stack that fails — check what is left + +**This is the teardown fact that costs money, and it is the opposite of the +intuition.** A bare `destroy` runs `npx cdk destroy --all --force` +(`scripts/deploy.sh:590-595`), and CDK halts at the first stack it cannot delete. +Everything later in the sequence is never attempted. + +The sequence is reverse-dependency order, so it goes observability → the runtime +stacks → gateway, memory, identity, auth, security → **networking last**. Which +means a failure in a *runtime* stack — early, cheap, and unrelated — leaves the +**NAT gateway and all five interface endpoints standing and billing hourly.** + +Measured: a full-platform destroy ran 693s (11.6 min), failed on the fourth stack, +exited 1, and left six accelerator stacks up including `-networking`, with the NAT +gateway still `available`. + +**So never treat `destroy` as fire-and-forget. Always confirm:** + +```bash +./scripts/deploy.sh destroy; echo "destroy rc=$?" + +# rc != 0 means stacks are still standing. Find them: +aws cloudformation list-stacks \ + --query "StackSummaries[?StackStatus!='DELETE_COMPLETE'].[StackName,StackStatus]" \ + --output text | sort + +# And specifically: is anything still on an hourly meter? +aws ec2 describe-nat-gateways --filter Name=state,Values=available \ + --query 'NatGateways[].NatGatewayId' --output text +``` + +Fix the failed stack, then **re-run `destroy`** — it picks up the remainder. + +Two expected annoyances: + +- The **networking stack can fail to delete for up to ~8 hours** while + AgentCore's `agentic_ai` ENIs drain. If networking is genuinely the *only* thing + left, NAT and the endpoints are already gone with it and the wait costs nothing. + That reassurance only holds when the destroy got that far — see the abort + behaviour above. +- **Transaction Search stays enabled.** Account-scoped, and other workloads may + now depend on it. Revert deliberately: + `aws xray update-trace-segment-destination --destination XRay`. + +Then sweep, because `destroy` leaves things behind in every Region it touched. +All of it is cheap, all of it is confusing to find later, and none of it is a bug. + +A verified sweep on a full platform-team teardown found **everything clean at the +service level** — no runtimes, gateways, memories, ECR repositories, SSM +parameters, Secrets Manager entries, Cognito pools or CodeBuild projects — and +**13 orphaned log groups**, in three classes. Service-created log groups are not +CloudFormation resources, so a clean stack delete does not touch them: + +| Class | Prefix | Count in that run | +|---|---|---| +| Lambda | `/aws/lambda/-…` | 7 | +| CodeBuild | `/aws/codebuild/-build-` | 3 | +| AgentCore runtimes | `/aws/bedrock-agentcore/runtimes/--DEFAULT` | 3 | + +**Two traps in sweeping them, and both will bite a one-liner:** + +1. **The runtime log groups use underscores, not hyphens.** A runtime name cannot + contain a hyphen, so `agentcore-workshop-dev` becomes + `agentcore_workshop_dev_orchestrator`. A sweep keyed on `$PREFIX` finds the + Lambda and CodeBuild groups and silently misses the AgentCore ones. +2. **`/aws/bedrock-agentcore/` is a shared namespace.** Any other AgentCore work in + the account has log groups under the same prefix. Match on the project name, not + on `/aws/bedrock-agentcore/`, or you will delete someone else's logs. + +```bash +aws logs describe-log-groups --query "logGroups[?\ +starts_with(logGroupName,'/aws/lambda/${PREFIX}')||\ +starts_with(logGroupName,'/aws/codebuild/${PREFIX}')||\ +starts_with(logGroupName,'/aws/bedrock-agentcore/runtimes/${PREFIX//-/_}')\ +].logGroupName" --output text | tr '\t' '\n' > /tmp/orphans.txt + +# Read the file before running this. +while IFS= read -r g; do + [ -n "$g" ] && aws logs delete-log-group --log-group-name "$g" +done < /tmp/orphans.txt +``` + +Use the read loop rather than `for g in $(…)`: under zsh an unquoted variable does +not word-split, so the whole tab-separated list arrives as one name and the API +rejects it with `InvalidParameterException … must have length less than or equal +to 512`. + +One more, and it is deliberately *not* in the sweep above: + +```bash +# The CDK staging bucket, cdk-hnb659fds-assets--. It is +# RETAIN by design, versioned, and survives deleting CDKToolkit itself — +# so DeleteBucket fails with BucketNotEmpty until you purge every version. +``` + +Leave the bucket alone unless you are cleaning up a Region you never meant to +deploy into; it is shared by every CDK app in the account and Region. + +Capture the deployment before destroying it: + +```bash +./scripts/deploy.sh export # workshop-outputs-.json +``` diff --git a/kiro/agentcore-enterprise-platform/steering/facilitation.md b/kiro/agentcore-enterprise-platform/steering/facilitation.md new file mode 100644 index 0000000..8159dd7 --- /dev/null +++ b/kiro/agentcore-enterprise-platform/steering/facilitation.md @@ -0,0 +1,486 @@ +# Running a guided session + +Read this when you are facilitating a workshop, guided team build, or builder +session with this accelerator — planning the agenda, prepping the room, or +recovering in front of an audience. + +**The single highest-leverage thing you can do is run `--dry-run` for the chosen +profile before anyone arrives, and again on screen as step one.** It prints every +module, the stacks it will deploy, and the exact verify command, and it makes +**zero AWS calls**. Everything below is downstream of that. + +```bash +./scripts/deploy.sh workshop --dry-run --profile greenfield +``` + +Three things to know before you rely on it: + +- **`--dry-run` belongs to `workshop`, not `deploy`.** `deploy --dry-run` accepts + the flag and ignores it — it bootstraps and deploys for real. If you are + previewing, the word `workshop` has to be in the command. +- **A misspelled flag is silently ignored.** `--dryrun` is not `--dry-run`; the + parser drops what it does not recognise and the run is real. Confirm from the + output, not from what you typed — a real dry run prints + `DRY RUN — nothing will be deployed` and never reaches a + `═══ Deploying ═══` header. +- **`security-focused` needs `ORG_ID` even to dry-run.** That gate runs before the + plan is printed, so without it you get no plan: at a terminal it stops and + prompts. Export a real id, or use any `o-…`-shaped value just to preview + (`ORG_ID=o-preview0 …`), since dry-run makes no AWS calls. + +--- + +## The day before + +Do these yourself, in the actual account, on the actual laptop that will be +sharing a screen. + +```bash +# 1. Local toolchain — this is where the room loses its first 20 minutes +python3.13 --version # exactly this name, not python3 +bash --version # 4+; macOS /bin/bash is 3.2 +node --version && npm --version +aws --version + +# 2. Repo ready +python3.13 -m venv .venv && source .venv/bin/activate +pip install -r requirements.txt +npm install -g aws-cdk@latest # do this AFTER pip: the CLI must not be +cdk --version # older than the aws-cdk-lib pip just installed +make lint && make test-controls # no AWS needed + +# 2b. macOS only — the verify scripts use bare urllib, not boto3, so a python.org +# python3.13 fails every one of them with CERTIFICATE_VERIFY_FAILED +python3.13 -c "import ssl; print(ssl.get_default_verify_paths().cafile)" # None == broken +"/Applications/Python 3.13/Install Certificates.command" # once per machine + +# 3. Account ready — and it is the account you think it is +unset AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY AWS_SESSION_TOKEN # these beat AWS_PROFILE +aws sts get-caller-identity --query '[Account,Arn]' --output text +aws bedrock list-foundation-models --region $AWS_REGION \ + --query "modelSummaries[?contains(modelId,'anthropic')].modelId" --output text +aws bedrock-agentcore-control list-gateways --region $AWS_REGION >/dev/null \ + && echo "AgentCore responds in $AWS_REGION" # the script never checks this + +# 4. The plan you will actually walk +./scripts/deploy.sh workshop --dry-run --profile

+``` + +Then decide these four things in advance, because deciding them live costs the +room its momentum: + +| Decision | Where it goes | Note | +|---|---|---| +| Profile | `--profile

` | see the picker in `deploy.md` | +| Region | `AWS_REGION` | `us-east-1` unless there is a reason; web search is Region-gated | +| Agent pattern | `AGENT_PATTERN` | `orchestrator` for a first pass; a framework the customer uses if the pitch is "framework-agnostic" | +| Model | `MODEL_ID` | override with a current cross-region inference profile rather than trusting in-code defaults | + +**Do a full dress rehearsal in the same account and Region at least once, +including teardown.** The failures worth knowing about are account-specific: AZ +id mapping, Bedrock model enablement, Region-gated connectors, Organizations +membership. + +### Prerequisites to state to participants in writing + +| Tool | Required? | Note | +|---|---|---| +| `python3.13` | Yes | exactly this name on PATH | +| `node` + `npm` | Yes | the CDK CLI runs through npx | +| A **current** CDK CLI | Yes | `npm install -g aws-cdk@latest`. An old global CLI passes the prereq check and then fails bootstrap on a schema mismatch — the single most likely way to lose the first ten minutes | +| `aws` CLI | Yes | with working credentials | +| `bash` 4+ | Yes | macOS ships 3.2 — `brew install bash` | +| Docker / finch | **No** | images build in AWS CodeBuild | + +Plus: **Bedrock model access enabled in the session Region before module 6**, and +an account where they can create IAM roles, Cognito pools, ECR repositories, +CodeBuild projects, and AgentCore resources. Sandbox or dev. + +If the profile is `security-focused` — or anything with +`enable_resource_policies` — they also need an AWS Organizations id, and the +account must actually be in an Organization: + +```bash +export ORG_ID=$(aws organizations describe-organization --query Organization.Id --output text) +``` + +--- + +## What the guided run does per module + +`workshop` loops: **explain → deploy → verify → pause**. The narration is in the +script, so you are not improvising the "why" for each layer. + +```bash +./scripts/deploy.sh workshop --profile

+./scripts/deploy.sh workshop --from 6 # resume where you stopped +``` + +Two mechanics to know before you rely on them: + +- **The pause is a bare `read`** — it waits for ENTER after each module. Under + `NON_INTERACTIVE=1` it does not pause at all, which is right for CI and wrong + for a room. +- **A failed verify prompts `Continue anyway? [y/N]`** and defaults to aborting. + Under `NON_INTERACTIVE=1` a failed verify **exits 1** with no prompt. So do not + set `NON_INTERACTIVE=1` for a live walk. + +`--from` is validated against the selected profile's sequence and fails with the +full sequence printed if the module is not in it. Skipped modules are logged as +skipped, not silently dropped. + +--- + +## Timing an agenda + +Deploy times from real runs. These are the floor — add prompts, discussion, and +the room's own questions. + +| Profile | Sequence | Deploy time | +|---|---|---| +| `greenfield` | 3 4 5 6 9 | ~18 min | +| `migration` | 3 4 6 7 9 | ~18 min | +| `multi-agent` | 3 4 5 6 7 8 9 | ~29 min | +| `platform-team` | 3 4 5 A 6 7 8 9 C E | ~39 min | +| `security-focused` | 3 4 5 6 9 E | ~21 min | + +Per module: 3 ≈ 2 min · 4 ≈ 2 min · 5 ≈ 3 min · A ≈ 2 min · **6 ≈ 7–8 min** · +7 ≈ 3 min · **8 ≈ 8 min** · 9 ≈ 3 min · C ≈ 5 min · E ≈ 3 min · B ≈ a full +container rebuild · D ≈ discussion only, deploys nothing. + +Realistic shape for a full day on `greenfield` or `migration`: about a third +deploying, a third verifying and reading what landed, a third discussion — +extension points, the customer's own tools, what production would need. If you +only have a half day, run `greenfield` and cut module 9's discussion, not its +deploy. + +### Fill module 6's silent seven minutes deliberately + +It is the one long gap and it happens early. Have something ready: + +- Start the local dashboard before module 6 so the room has something to watch + (status only, no AWS resources): + + ```bash + .venv/bin/python dashboard/monitor.py & + python3 -m http.server 8888 -d dashboard/public # http://localhost:8888 + ``` + +- Or show the build actually moving: + + ```bash + aws codebuild list-builds-for-project --project-name $PREFIX-build-orchestrator \ + --query 'ids[0]' --output text + ``` + +- Or use the time for the architecture conversation the module is about: why the + image is arm64, why the build runs in CodeBuild rather than on laptops, what + the content-hash image tag means for CI. + +**Say "this takes about seven minutes with no output" before you press enter, not +after minute four.** Unannounced silence reads as a broken demo; announced +silence reads as a container build. + +--- + +## Splitting a room across workstreams + +For a group large enough that watching one screen wastes people: + +| Team | `--team` | Stacks | +|---|---|---| +| Platform | `platform` | `-networking -auth -identity -gateway -observability` | +| Agent | `agent` | `-runtime-orchestrator -runtime-code-agent -runtime-research-agent -memory` | +| Security | `security` | `-security -observability` | + +```bash +./scripts/deploy.sh deploy --team agent +``` + +`-observability` is deliberately in two teams; CDK is idempotent so the overlap +is harmless. + +**Two of the three teams fail out of the box, and this will happen live.** +`TEAM_MAP` names stacks that the app only creates behind a feature flag +(`scripts/deploy.sh:203-206`, `app.py:81-82`), and `cdk deploy` refuses the whole +batch when any one name does not exist — so nothing deploys, not even the four +stacks that were fine: + +Which teams survive a bare `deploy --team` depends entirely on the manifest in +force, because `TEAM_MAP` names stacks the manifest may not declare. Computed +from `expected_stacks()` for each preset — pick the row matching the profile you +materialized: + +| Manifest | `--team platform` | `--team agent` | `--team security` | +|---|---|---|---| +| `greenfield` | fails: `-networking` | fails: the two A2A runtimes | fails: `-security` | +| `migration` | fails: `-networking` | fails: the two A2A runtimes | fails: `-security` | +| `multi-agent` | fails: `-networking` | **works** | fails: `-security` | +| `platform-team` | **works** | **works** | **works** | +| `security-focused` | **works** | fails: the two A2A runtimes | **works** | + +`platform-team` is the only manifest under which all three splits work bare, and +that is not a coincidence — it is the profile written for this exact scenario. If +the room is splitting by team, materialize `platform-team` first: + +```bash +./scripts/deploy.sh deploy --profile platform-team --module 3 +``` + +It turns networking and security on, which starts two hourly meters. That is a +cost decision to make in the open rather than a flag to paste in a hurry. + +**A misspelled team no longer deploys everything.** This used to be the sharpest +edge in the script: `--team` values were never validated, `--profile` only on the +`workshop` action, and a bad value fell through the chain that sets `CDK_STACKS`, +leaving it empty — and an empty stack list means `cdk deploy --all`. Upstream +closed it. The parser now rejects unknown options, and `--profile`/`--team` +values are checked for every action (`scripts/deploy.sh:955-990`), so +`--team platfrom` and `deploy --profile greenfeild` exit 1 with the valid list +printed. Read the `Team … → Stacks:` line back anyway — it is still the fastest +way to see what you are about to approve. + +Two constraints that decide whether this works: + +- **Teams are not independent.** The agent team's runtimes need the platform + team's Cognito issuer and gateway. Run module 3 and 4 together, for everyone, + before splitting. +- **Give each team its own account, or its own `PROJECT_NAME`/`ENVIRONMENT`.** + Sharing a prefix in one account means two `cdk deploy` runs fighting over the + same stacks. Different prefixes in one account is fine and cheap: + + ```bash + PROJECT_NAME=team-a ENVIRONMENT=dev ./scripts/deploy.sh deploy --team agent + ``` + +Reconvene on a shared verify — `./scripts/deploy.sh verify` in each team's +`PROJECT_NAME`, then `invoke.py` — so the room sees one working platform rather +than three partial ones. `verify` derives each team's footprint from its own +config, which is what makes it usable across a split. + +--- + +## The five things to pre-empt + +Every one of these is a real question that has cost a session time. Say them +before they happen; they land as expertise and afterwards as excuses. + +1. **`--profile` on its own deploys everything the manifest declares**, not just + that profile's modules, because it materializes the preset and then runs + `cdk deploy --all`. Use `workshop --profile` for the guided walk, or combine + `--profile … --module …`. Measured at 6 stacks for `greenfield`, 10 for + `platform-team` (`deploy.md` has all five). Interactively you now get a + `Proceed to deploy ALL of the above? [y/N]` prompt with the full stack list — + read it out to the room; that is a better teaching moment than the warning. + Under `--yes` or `NON_INTERACTIVE=1` there is no prompt. Two related edges are + **fixed** upstream and no longer worth pre-empting — misspelled flags and + misspelled `--profile`/`--team` values now exit 1 instead of deploying the lot. + What survives is `deploy --dry-run`, spelled correctly: the `deploy` action + accepts it and ignores it. +2. **Module 6 is silent for ~7–8 minutes.** Remote arm64 container build. +3. **The default `orchestrator` agent has no tools and reads no caller + identity.** Both deliberate. Asking it "what tools do you have?" correctly + returns nothing — tools live on the gateway (`invoke.py --tools`). If someone + is going to ask a tool question, deploy `strands-agent` or `langgraph-agent`. +4. **`CREATE_COMPLETE` proves nothing about behaviour.** Say it once, early, and + then run every verify. It is also the honest framing for the security + conversation: module E's verify is a stack-status check because there is no + behavioural probe for "is this control enforcing." +5. **Module 4 deploys less than its title suggests.** The gateway M2M credential + provider is always created; enterprise IdP federation only happens if a + federated IdP was chosen, and the 3LO providers only when client ids are + supplied. Nothing silently half-configures — but say so, or someone concludes + it is broken. + +--- + +## Before you share your screen: a deploy prints a live secret + +**Once module 3 has run in this account, every later `deploy.sh deploy` prints the +Cognito M2M app client secret in plaintext** — under a CDK-generated key ending +`…UserPoolClientClientSecret…`. It scrolls past in a wall of ids and looks like +every other one of them. + +**Every run. Not "modules 3 and 4", and not even "runs that touch `-auth`."** The +script's end-of-run summary does its own account query for every stack whose name +starts with the project prefix and dumps all of their outputs, regardless of what +you deployed. Verified with `deploy --module C`: CDK's own `Outputs:` block listed +only the networking stack, and the summary table printed the `-auth` secret anyway. +Resumed runs (`--from 6`) walk it too. There is no flag that turns this off. +`security.md` has the mechanism and the mitigations. + +Three things that cost nothing: + +- **Do not assume a narrowly scoped `--module` is safe to screen-share.** No + `deploy` invocation is, once `-auth` exists. Scroll the summary table off screen + before you stop presenting, or run deploys off-screen and screen-share the + verifies. +- **If it does go on screen, say so and rotate afterwards** rather than hoping + nobody scrolled back. In a throwaway workshop account, teardown is the rotation. +- **Do not paste raw deploy output into a ticket or chat** when asking for help. + The collection commands in `verify.md` are safe; a full deploy log is not. + +Worth saying out loud when it happens, because a security-minded participant will +spot it and the honest version is a better moment than the caught-out one: this is +the finding to raise in the review, and it is one read-only API call to confirm. + +--- + +## When something breaks in front of people + +This is the part that decides how the session is remembered. The accelerator's +failure messages are good; use them rather than improvising. + +1. **Read the actual error out loud.** The verify prints why it failed. +2. **Container failures name their own cause in the logs.** Every runtime failure + so far has: + + ```bash + ARN=$(aws ssm get-parameter --name /$PROJECT_NAME/$ENVIRONMENT/runtimes/orchestrator/arn \ + --query Parameter.Value --output text) + aws logs filter-log-events \ + --log-group-name "/aws/bedrock-agentcore/runtimes/${ARN##*/}-DEFAULT" \ + --start-time $(( ($(date +%s) - 900) * 1000 )) \ + --query 'events[].message' --output text | grep -iE 'error|denied|traceback' + ``` + +3. **Answer `y` to "Continue anyway?" and write down which module failed.** + Later modules build on it, and the failure usually explains a stranger symptom + two modules later. Debugging live at minute 40 costs the rest of the agenda. +4. **A stack that died mid-deploy is safe to re-run.** CDK picks up from current + state, and `UPDATE_ROLLBACK_COMPLETE` can be deployed onto again. +5. **If you hit Ctrl-C — because the room is waiting and you panicked — nothing + in AWS stopped.** The script dies instantly and prints + `[ERROR] Failed to deploy `, which looks exactly like a real failure and + is not one. Measured: the stack went on to `UPDATE_COMPLETE` by itself, the + CodeBuild build succeeded, the runtime came back `READY`, and a re-run reported + `(no changes)` in 27.6s. Check `describe-stacks` before you tell the room + anything failed, and if it says `UPDATE_IN_PROGRESS`, wait rather than retry. +6. **Then go to `troubleshooting.md`**, which is organised by what the person + sees rather than by what the code does. + +Treat a real failure as content, not as a setback: "this is the failure mode you +will hit in your own account, and here is how the platform tells you" is more +valuable than a clean run. + +--- + +## Cost, and the three settings that are not pay-per-use + +Most of the platform is pay-per-use and nearly free idle. Three exceptions to say +out loud **before** deploying, not in the wrap-up: + +| Item | Why it matters | +|---|---| +| **NAT gateway + VPC interface endpoints** (`enable_networking`) | bill **hourly**, whether or not anything runs — 1 NAT plus 5 endpoints × 2 AZs. `platform-team` and `security-focused` both turn networking on; the other three profiles leave nothing hourly. | +| **Transaction Search** (`enable_transaction_search`, defaults **true**) | changes span-ingestion pricing **account-wide** and **survives teardown** by design, because other workloads may come to depend on it. | +| **CodeBuild** | per-build minutes; every pattern swap is another arm64 build. | + +**Someone will ask "what does this cost if we leave it up overnight?" — usually at +the end of day one, in front of whoever owns the account.** Have the answer ready +rather than promising to follow up. On `greenfield`, `migration` or `multi-agent` +the honest answer is "nothing standing"; on the two networking profiles it is "one +NAT gateway and ten endpoint-AZ-hours per hour, plus a KMS key and any Secrets +Manager entries per month." `deploy.md` has the itemised inventory. Give the meters +and offer to price them in the calculator for their Region — do not quote dollar +rates from memory. + +If the session is in an account where an account-wide setting is not yours to +change, decide before module 9: + +```bash +ENABLE_TRANSACTION_SEARCH=false ./scripts/deploy.sh deploy --module 9 +``` + +…and then say plainly that tracing will not work — every OTLP span batch is +rejected with HTTP 400 while the deploy still reports success. That tradeoff is +better than surprising the account owner. + +Revert it deliberately after a session if you need to: + +```bash +aws xray update-trace-segment-destination --destination XRay +``` + +--- + +## Capture before you tear down + +The export is the artifact participants take home — every SSM parameter and stack +output in one file, which is also the input to any follow-on work. + +```bash +./scripts/deploy.sh export # → workshop-outputs-.json +``` + +**This file contains account ids and resource ARNs.** Do not commit it or paste +it into a shared channel unaltered. + +--- + +## Teardown, same day + +```bash +./scripts/deploy.sh destroy; echo "destroy rc=$?" +``` + +**Check that exit code before you close the laptop.** A destroy stops at the first +stack it cannot delete, and because networking is torn down *last*, an unrelated +failure in a cheap runtime stack leaves the NAT gateway billing overnight. Measured +on a real run: exit 1 after 11.6 minutes, failed on the fourth of ten stacks, six +stacks still up. The cause was a runtime `DELETE_FAILED` / `NotStabilized`, which is +a stabilization timeout rather than a real failure — retrying that one stack cleared +it in 34 seconds, and a second `destroy` took out the rest. `troubleshooting.md` has +both entries. + +This is the one teardown step worth assigning to a named person with a calendar +reminder, not a "someone will check tomorrow." + +Then sweep, because two things linger by design: + +```bash +# Anything left standing +aws cloudformation describe-stacks \ + --query "Stacks[?starts_with(StackName,'$PREFIX')].{n:StackName,s:StackStatus}" --output table + +# Anything still on an hourly meter +aws ec2 describe-nat-gateways --filter Name=state,Values=available \ + --query 'NatGateways[].NatGatewayId' --output text + +# ENIs that block subnet/security-group deletion for up to ~8 hours +aws ec2 describe-network-interfaces \ + --filters Name=interface-type,Values=agentic_ai \ + --query 'NetworkInterfaces[].{id:NetworkInterfaceId,status:Status,subnet:SubnetId}' +``` + +- The **networking stack can fail to delete for up to ~8 hours** while AgentCore's + `agentic_ai` ENIs drain. NAT and endpoints are already gone by then, so the wait + costs nothing meaningful — but somebody has to come back and finish it. Own + that, or hand it to a named person. +- **Transaction Search stays enabled.** Account-scoped, on purpose. + +If you ran a pattern matrix or several profiles, also check the resources CDK does +not always take with it: ECR repositories, CloudWatch log groups, and any Secrets +Manager entries created from API-key prompts. + +--- + +## A ready-made run of show — `greenfield`, one day + +| Slot | What | Notes | +|---|---|---| +| Open | `workshop --dry-run` on screen | the whole plan, zero AWS calls; sets expectations for the day | +| Module 3 | Cognito, OAuth clients, SSM registry | frame it as the trust root: every AgentCore call here is authenticated | +| Module 4 | the M2M credential provider | say what it does *not* deploy by default | +| Module 5 | gateway + Lambda tool target | `test_gateway.py` is a real `tools/list` + `tools/call` | +| Module 6 | the agent | announce the 7–8 minutes; run the dashboard; talk architecture | +| Verify | `deploy.sh verify`, then `invoke.py --tools` | one command covers the footprint; make the "registered vs loaded" distinction explicit | +| Module 9 | observability | disclose the account-wide Transaction Search setting *before* deploying | +| Extend | add a tool target (module 7 material) | the moment it stops being a demo: agents pick up new tools with **no agent redeploy** | +| Swap | `AGENT_PATTERN=` on module 6 | the framework-agnostic claim, demonstrated rather than asserted | +| Close | `export`, then `destroy` | hand over the outputs file; name who checks the networking stack tomorrow | + +The two slots that produce the strongest reaction are **Extend** and **Swap** — +both are cheap, both are the actual differentiators, and both are the first things +cut when the morning runs long. Protect them by starting teardown on time, not by +shortening them. diff --git a/kiro/agentcore-enterprise-platform/steering/modules.md b/kiro/agentcore-enterprise-platform/steering/modules.md new file mode 100644 index 0000000..83570d3 --- /dev/null +++ b/kiro/agentcore-enterprise-platform/steering/modules.md @@ -0,0 +1,511 @@ +# The modules + +Read this when you need to know exactly what a module deploys, which stack +carries it, what it publishes for other modules to find, and the command that +proves it worked. + +`PREFIX` throughout is `${PROJECT_NAME}-${ENVIRONMENT}`, defaulting to +`agentcore-workshop-dev`. Verify commands run from the repo root. + +Every Python command here is spelled `.venv/bin/python`, which is what the guided +run itself uses (`MODULE_VERIFY` at `scripts/deploy.sh:190-199`). Prefer it over a +bare `python` even in an activated shell: `source .venv/bin/activate` does not +survive between an agent's tool calls, so a bare `python` fails with +`ModuleNotFoundError: boto3` for reasons that look nothing like the real cause. + +--- + +## Map at a glance + +| # | Module | Stacks | Verify | Time | +|---|---|---|---|---| +| 3 | Infrastructure Blueprint | `$PREFIX-auth` | SSM issuer URL | ~2 min | +| 4 | Identity Integration | `$PREFIX-auth` `$PREFIX-identity` | SSM credential provider name | ~2 min | +| 5 | Gateway & Registry | `$PREFIX-gateway` | `test_gateway.py` | ~3 min | +| 6 | Agent Deployment | `$PREFIX-runtime-orchestrator` | `invoke.py` | **~7–8 min** | +| 7 | Gateway Integration | `$PREFIX-gateway` | `test_gateway.py` | ~3 min | +| 8 | Agent-to-Agent | `$PREFIX-runtime-code-agent` `$PREFIX-runtime-research-agent` | `invoke.py --a2a code-agent` | ~8 min | +| 9 | Observability | `$PREFIX-observability` | `check_observability.py` | ~3 min | +| A | Memory | `$PREFIX-memory` | `test_memory.py` | ~2 min | +| B | Code Interpreter | `$PREFIX-runtime-orchestrator` | *(none)* | — | +| C | Multi-Account Networking | `$PREFIX-networking` | `check_network.py` | ~5 min | +| D | CI/CD | *(none — exits 0)* | *(none)* | — | +| E | Security Automation | `$PREFIX-security` | stack COMPLETE | ~3 min | + +Modules 5 and 7 are the **same stack**. So are 6 and B. That is intentional: 7 +grows the gateway by redeploying it with more targets, and B adds the code +interpreter by redeploying the orchestrator. + +`--module D` prints a message pointing at `.gitlab-ci.yml` and exits 0 — there +is nothing to deploy. An unknown module id exits 1 with the valid list. + +**Three modules live behind feature flags, and only one of them turns its own flag +on.** This bites on the standalone `deploy --module` path, which is what people use +to redo a single module: + +| Module | Flag | App default | Bare `deploy --module` | +|---|---|---|---| +| 8 | `enable_a2a` | **`true`** (`app.py:83`) | works | +| C | `enable_networking` | `false` | **fails**: `No stacks match the name(s) …-networking` | +| E | `enable_security` | `false` | **fails**: `No stacks match the name(s) …-security` | + +```bash +ENABLE_NETWORKING=true ./scripts/deploy.sh deploy --module C +ENABLE_SECURITY=true ./scripts/deploy.sh deploy --module E +``` + +The guided loop re-exports `ENABLE_A2A=true` for module 8 +(`scripts/deploy.sh:901`) so a profile that set it false does not break the walk; +there is no equivalent for C or E. Inside `workshop --profile platform-team` or +`security-focused` the flags are already set, so this only affects standalone runs. +The error's own advice — "Check CloudFormation console for details" — is a dead end: +the stack was never synthesized. + +--- + +## Module 3 — Infrastructure Blueprint + +**Stack:** `$PREFIX-auth` + +Cognito User Pool with email sign-in and three OAuth app clients — `app`, `web`, +`m2m` — plus the `agentcore/invoke` resource-server scope. This is the trust root +for everything else: every AgentCore call in this platform is authenticated. + +**Publishes to SSM** under `/{project}/{env}/auth/`: +`issuer-url`, `user-pool-id`, `app-client-id`, `web-client-id`, `m2m-client-id`. + +**Stack outputs** (verified on a real deploy — nine named): `UserPoolId`, +`UserPoolArn`, `IssuerUrl`, `DiscoveryUrl`, `DomainUrl`, `IdPType`, +`AppClientId`, `WebClientId`, `M2MClientId`. + +**Plus four CDK auto-generated `ExportsOutput…` entries, and one of them is the +M2M client secret in plaintext.** Not a typo and not a flag you can turn off — see +`security.md`. The practical consequence: **once this stack exists, every later +`deploy.sh deploy` prints a live secret to stdout**, because the script's closing +summary enumerates every stack matching the project prefix and dumps its outputs +regardless of what you deployed (`scripts/deploy.sh:659`). Verified with +`deploy --module C`, which does not touch `-auth` at all. Keep it off a shared +screen and out of pasted logs. + +```bash +aws ssm get-parameter --name "/$PROJECT_NAME/$ENVIRONMENT/auth/issuer-url" \ + --region "$AWS_REGION" --query Parameter.Value --output text + +# The outputs, without printing the secret value: +aws cloudformation describe-stacks --stack-name "$PREFIX-auth" \ + --region "$AWS_REGION" \ + --query 'Stacks[0].Outputs[].{Key:OutputKey,Len:length(OutputValue)}' --output table +``` + +--- + +## Module 4 — Identity Integration + +**Stacks:** `$PREFIX-auth`, `$PREFIX-identity` + +The one thing this always creates is the **`gateway-m2m` OAuth2 credential +provider** in the AgentCore Identity Token Vault. That is what lets a runtime +fetch its own gateway token instead of being handed one. + +**It deploys less than its title suggests, by default.** Say this before someone +concludes it is broken: + +- Enterprise IdP federation happens only if you chose one + (`IDP_TYPE=entra_id|okta|ping`); the prompt defaults to plain Cognito. +- The Google / GitHub / Notion 3LO providers appear only when you supply their + client ids — **and a secret *name* alongside each one.** A client id with no + `_client_secret_name` is a hard failure at synth + (`stacks/identity_stack.py:51-62`), and the plaintext + `_client_secret` / `_CLIENT_SECRET` keys are rejected outright + (`app.py:166-174`). The error prints the `create-secret` command and the `-c` + key to pass, and `scripts/deploy.sh` does both for you when the secret is in + the environment. So the shape is: + + ```bash + aws secretsmanager create-secret --name agentcore-workshop-dev-google-oauth-secret \ + --secret-string '' + # then: -c google_client_secret_name=agentcore-workshop-dev-google-oauth-secret + ``` + + If you are looking at an older deployment: before this was fixed, every 3LO + provider synthesized with an empty `Oauth2ProviderConfigInput` because the + config dict's top-level key did not match the CloudFormation model, and the L1 + mapping dropped the whole block silently. They could not have worked. A + redeploy on current `main` is the fix, not a configuration change. + +Nothing silently half-configures. Full Entra ID walkthrough is in the repo at +`docs/ENTERPRISE_IDP.md`, including how to verify federation without opening a +browser. + +**Publishes to SSM:** `/{project}/{env}/identity/gateway-credential-provider-name`, +and `/{project}/{env}/identity/{provider}-provider-arn` per 3LO provider. + +```bash +aws ssm get-parameter --name "/$PROJECT_NAME/$ENVIRONMENT/identity/gateway-credential-provider-name" \ + --region "$AWS_REGION" --query Parameter.Value --output text +``` + +--- + +## Module 5 — Gateway & Registry + +**Stack:** `$PREFIX-gateway` + +The AgentCore MCP Gateway with `CUSTOM_JWT` auth against the Cognito issuer, and +a Lambda tool target. Agents discover tools through the gateway instead of +hardcoding endpoints, and every call is auditable. + +Ships `sample-tool` (Lambda, `tools/sample_tool/`) exposing +`text_analysis_tool`. In Regions where it is supported, also the built-in +`web-search` connector. Tools are named `___` — e.g. +`sample-tool___text_analysis_tool`. That full name is what Cedar policies and +agent prompts refer to. + +Opt-in on this stack: the Cedar policy engine (`enable_cedar`) and the Bedrock +Guardrail + egress interceptor Lambda (`enable_egress_filter`). + +**Publishes to SSM:** `/{project}/{env}/gateway/url`. + +```bash +.venv/bin/python scripts/test_gateway.py # tools/list + one real tools/call +``` + +--- + +## Module A — Memory + +**Stack:** `$PREFIX-memory` + +AgentCore managed Memory with a **user-preference strategy**. Semantic fact +extraction is added only when `use_long_term_memory=true` — it costs more, so it +is off by default. `ltm_top_k` defaults to 10 and `ltm_relevance_score` to 0.3. + +Opt-in: KMS CMK encryption, and the in-account-only resource policy +(`enable_resource_policies`, needs `org_id`). + +**Publishes to SSM:** `/{project}/{env}/memory/memory-id`, +`/{project}/{env}/memory/memory-arn`. + +```bash +.venv/bin/python scripts/test_memory.py +``` + +Note what this does *not* prove: `test_memory.py` uses **your local +credentials**, so it passes regardless of whether the runtime role can reach +Memory. See `troubleshooting.md` for how to check the role itself. + +**Ordering:** in `platform-team`, A runs before 6 on purpose — the orchestrator +depends on memory, so running 6 first would make CDK create memory implicitly +and module A would then report "no changes." + +--- + +## Module 6 — Agent Deployment + +**Stack:** `$PREFIX-runtime-orchestrator` + +The orchestrator agent on AgentCore Runtime. CodeBuild builds an **arm64** +container image remotely — no local Docker — and `CfnRuntime` runs it. Image +tags are a content hash of the source plus the selected pattern, so CodeBuild +only reruns when something actually changed. + +**This is the slow one: ~7–8 minutes with almost no output on a first build.** +That is the remote container build, not a hang. + +The runtime receives `MODEL_ID`, `GATEWAY_URL`, +`GATEWAY_CREDENTIAL_PROVIDER_NAME`, `MEMORY_ID`, and — for client-facing +protocols — `COGNITO_ISSUER_URL` and `COGNITO_ALLOWED_CLIENTS` (injected by +`app.py`, not by the runtime stack). + +**Publishes to SSM:** `/{project}/{env}/runtimes/{component}/arn` and +`/{project}/{env}/runtimes/{component}/id`. + +```bash +.venv/bin/python scripts/invoke.py "Reply with exactly: WORKSHOP OK" +``` + +**Bedrock model access must be enabled in this Region before this module**, or +the first invoke fails with an access error. If `MODEL_ID` is unset, each +pattern falls back to its own in-code default; a dated model id that has aged +out into Legacy status is rejected in fresh accounts, so override it: + +```bash +export MODEL_ID=us.anthropic.claude-sonnet-5 +``` + +The default `orchestrator` pattern has **no tools** and extracts no caller +identity. Both are deliberate. + +--- + +## Module 7 — Gateway Integration + +**Stack:** `$PREFIX-gateway` (redeployed with more targets) + +This is the module where the platform stops being something you deployed and +becomes something you extend. **Agents pick up new tools on their next +discovery, with no agent redeploy.** That is the claim worth demonstrating live. + +Two kinds of target: + +| | Built-in connector | Lambda target | +|---|---|---| +| You write | nothing | a handler + tool schema | +| Good for | capabilities AWS operates (web search) | your APIs, data, business logic | +| Credentials | the gateway's IAM role | whatever your Lambda needs | +| Example | `web-search` in `stacks/gateway_stack.py` | `sample-tool` in `tools/sample_tool/` | + +Three things that are easy to get wrong — all in `docs/GATEWAY_TARGETS.md`: + +1. **Connector config must go through `add_property_override`**, not + `target_configuration`. The L1 construct predates connector targets and + silently drops the key; the target then deploys with no connector and the tool + never appears. +2. **The gateway role needs the connector's own action.** Web search needs + `bedrock-agentcore:InvokeWebSearch` on + `arn:aws:bedrock-agentcore::aws:tool/web-search.v1` — note the literal + `aws` where an account id would normally be. Without it the target deploys and + every call fails at invoke time. +3. **Connectors are regional.** Gate any connector you add the way `app.py` + gates web search. + +For a Lambda tool, the tool name arrives in the **context**, not the event: +`context.client_context.custom["bedrockAgentCoreToolName"]`, formatted +`___`. Split on `___` and dispatch on the suffix. Return +`{"content": [{"type": "text", "text": ...}]}` or `{"error": "..."}`. Declare +the schema in `app.py`'s `tool_configs` using **PascalCase** keys (`Name`, +`Description`, `InputSchema`, `Type`, `Properties`, `Required`) — that is the +CloudFormation shape, not MCP JSON. + +Write the tool `Description` for a model, not for a human skimming a table. It is +the only thing the agent has when deciding whether the tool answers the question. + +```bash +./scripts/deploy.sh deploy --module 7 +.venv/bin/python scripts/test_gateway.py +.venv/bin/python scripts/invoke.py --tools # your tool should be listed +``` + +--- + +## Module 8 — Agent-to-Agent (A2A) + +**Stacks:** `$PREFIX-runtime-code-agent`, `$PREFIX-runtime-research-agent` + +Two specialized sub-agents on their own runtimes, each with independent auth, +scaling, and lifecycle. The orchestrator delegates to them. + +The A2A stacks only exist in the CDK app when `enable_a2a=true`, so +**`--module 8` exports `ENABLE_A2A=true` automatically.** + +The sub-agents speak **JSON-RPC 2.0 on port 9000**, not the HTTP payload shape. +Their contract, served by `agent-code/shared/a2a_serve.py`: + +- `POST /` — JSON-RPC +- `GET /.well-known/agent-card.json` +- `GET /ping` returning `{"status": "Healthy"}` + +all on `0.0.0.0:9000`. + +**Two different failures get conflated here, including by the accelerator's own +docs.** `docs/TROUBLESHOOTING.md` ("Invoke returns HTTP 424") and the docstring at +`scripts/invoke.py:112` both say that sending `{"prompt": ...}` to an A2A runtime +"gets you a 424." Measured against a working `code-agent`, it does not: + +| What is wrong | What you actually see | +|---|---| +| **Client** sends the wrong envelope (`{"prompt": …}` instead of JSON-RPC `message/send`) | **HTTP 200** with a JSON-RPC error body: `"code":-32600`, `"message":"Request payload validation error"`, and a pydantic `missing field: method` detail | +| **Container** serves the wrong protocol (built on `BedrockAgentCoreApp`/HTTP 8080 instead of JSON-RPC 9000) | **HTTP 424**, with clean container logs | + +So a 200 is not success here — read the body. And 424 means the *image* is wrong, +which is a redeploy, not a payload fix. Searching the repo docs for "424" after an +envelope mistake sends you down the wrong path entirely; that mismatch is real +enough that `tests/test_a2a_contract.py` exists to guard the container side of it +statically. + +They are also guarded differently: **A2A runtimes use SigV4, not a JWT.** A2A is +not a client-facing protocol, so those runtimes deliberately get no JWT +authorizer and rely on IAM `InvokeAgentRuntime`. `scripts/invoke.py` picks the +right mechanism per target, which is why it is the recommended path. + +```bash +.venv/bin/python scripts/invoke.py --a2a code-agent "Reply with exactly: A2A OK" +.venv/bin/python scripts/invoke.py --a2a research-agent "…" +``` + +--- + +## Module 9 — Observability + +**Stack:** `$PREFIX-observability` + +Vended log delivery plus X-Ray tracing for the gateway, memory, and runtimes — +and the two settings that make tracing work at all: a CloudWatch Logs resource +policy allowing X-Ray to write span log groups, and a trace segment destination +of `CloudWatchLogs`. + +**Without that destination change, every OTLP span batch is rejected with HTTP +400 and the deployment still reports success.** This is why +`enable_transaction_search` defaults to `true`. It is **account- and +Region-scoped**, it changes span-ingestion pricing account-wide, and destroying +this stack does **not** revert it — other workloads may have come to depend on +it. + +Opt-in: `enable_traceability` adds SNS + EventBridge alerting on sensitive +AgentCore API calls. It needs CloudTrail management events, so enable it together +with `enable_security` (module E). + +```bash +.venv/bin/python scripts/check_observability.py # destination + policy + deliveries +.venv/bin/python scripts/check_observability.py --spans # …and spans are searchable +``` + +`--spans` is deliberately **not** part of module 9's verify: span delivery lags +an invocation by a minute or two. Run it later, after some invokes. + +**Expect the verify to fail the first time in a new account.** Turning Transaction +Search on is asynchronous and outlasts the stack that requests it, and the verify +runs about a second after `CREATE_COMPLETE` with no retry: + +``` +FAIL: trace segment destination is CloudWatchLogs but status is PENDING +``` + +Measured on a fresh account: **8m22s** from `CREATE_COMPLETE` to `ACTIVE`. Poll +`aws xray get-trace-segment-destination`, then re-run the verify — it passes +unchanged. Only the first enablement per account is slow. Because module 9 is last +in `greenfield`, this lands on the final step of a guided session; say it in advance. + +--- + +## Module B — Code Interpreter + +**Stack:** `$PREFIX-runtime-orchestrator` (redeploy) + +Adds the sandboxed code interpreter to the orchestrator. It has **no guided +narration and no verify command, and appears in no profile's sequence** — it is +available via `--module B` but is not part of any standard walk. Budget a full +container rebuild for it, same as module 6. + +--- + +## Module C — Multi-Account Networking + +**Stack:** `$PREFIX-networking` + +VPC, private subnets, and AgentCore VPC endpoints. When `enable_networking=true`, +runtimes get `network_mode: VPC` with the private subnets and a runtime security +group — passing nothing is what would otherwise make "enterprise network +isolation" untrue. + +**Publishes to SSM:** `/{project}/{env}/networking/vpc-id`, +`/{project}/{env}/networking/private-subnet-ids`, +`/{project}/{env}/networking/runtime-security-group-id`. + +**38 resources, ~3m30s.** Six endpoints get created: `bedrock-agentcore.gateway`, +`bedrock-runtime`, `ecr.api`, `ecr.dkr`, `logs` (interface) and S3 (gateway). + +The endpoint policy needs `org_id`. Without it the endpoint is created with **no +policy at all** — a warning, not an error. And with it, only the AgentCore endpoint +is restricted; the other five keep the wide-open AWS default. See `security.md` for +what that policy does and does not cover. + +**This module does not put your agents in the VPC.** Existing runtimes stay +`networkMode: PUBLIC` until redeployed — measured, with `check_network.py +--expect-public` passing while the networking stack was `CREATE_COMPLETE`. Finish +the job: + +```bash +ENABLE_NETWORKING=true ORG_ID=o-xxxx ./scripts/deploy.sh deploy --module 6 # 345s +ENABLE_NETWORKING=true ORG_ID=o-xxxx ./scripts/deploy.sh deploy --module 8 # 172s +``` + +**The AZ trap, which is real and costs ~15 minutes:** AgentCore supports a limited +set of AZ **ids** per Region, `max_azs=2` takes the first two AZ *names* +alphabetically, and name → id mapping differs per account. On a fresh `us-east-1` +account this landed a subnet in `use1-az6`, which AgentCore does not support; the +stack still reached `CREATE_COMPLETE`, and the runtime redeploy is what failed. The +fix is a **source edit** (there is no flag) plus a **destroy and recreate** (an +in-place AZ change collides on subnet CIDRs). Full recipe and measurements in +`security.md`. + +```bash +aws cloudformation describe-stacks --stack-name "$PREFIX-networking" \ + --region "$AWS_REGION" --query "Stacks[0].StackStatus" --output text | grep -q COMPLETE \ + && .venv/bin/python scripts/check_network.py +``` + +`check_network.py` stops at the first failure, so expect to see the AZ problem and +the placement problem one after the other, not together. +`check_network.py --expect-public` asserts the opposite posture, for confirming a +non-VPC deployment is genuinely public rather than accidentally so. + +**Teardown warning:** AgentCore leaves `agentic_ai` ENIs behind for up to ~8 +hours after runtimes stop using the VPC, and they block subnet/security-group +deletion. NAT and endpoints delete normally, so waiting costs nothing meaningful. +If no runtime ever successfully entered the VPC there are no ENIs and the stack +deletes cleanly first time — measured at 4m32s. + +--- + +## Module D — CI/CD + +No stacks. `--module D` logs a pointer to `.gitlab-ci.yml` as the reference +implementation and exits 0. This is a discussion module: read the pipeline, map +it onto the customer's own CI, talk about what gates a promotion. + +--- + +## Module E — Security Automation + +**Stack:** `$PREFIX-security` + +KMS customer-managed key encryption and CloudTrail audit logging. It is the +prerequisite for `enable_traceability` (module 9's alerting), because that rule +only fires if CloudTrail management events are being recorded. + +Verify is a stack-status check — there is no behavioural probe here: + +```bash +aws cloudformation describe-stacks --stack-name "$PREFIX-security" \ + --region "$AWS_REGION" --query "Stacks[0].StackStatus" --output text | grep -q COMPLETE +``` + +Which is exactly why `security.md` exists: reaching `COMPLETE` tells you the +resources were created, not that any control is enforcing anything. + +--- + +## The SSM registry is the extension seam + +Every stack publishes its outputs under `/{project}/{env}/*`. That registry is +how the stacks find each other, and it is the clean place for customer-specific +extensions to read platform values without importing CDK constructs or +hardcoding ARNs: + +``` +/{project}/{env}/auth/issuer-url | user-pool-id | app-client-id | web-client-id | m2m-client-id +/{project}/{env}/identity/gateway-credential-provider-name +/{project}/{env}/identity/{provider}-provider-arn +/{project}/{env}/gateway/url +/{project}/{env}/memory/memory-id | memory-arn +/{project}/{env}/runtimes/{component}/arn | id +/{project}/{env}/networking/vpc-id | private-subnet-ids | runtime-security-group-id +``` + +Capture the whole set at once: + +```bash +./scripts/deploy.sh export # → workshop-outputs-.json +``` + +**Use the `ssm_parameters` half and ignore `stack_outputs`.** The SSM collection is +a single API call and is correct — 18 parameters on a full platform-team deploy, and +none of them is a secret. `stack_outputs` is always garbage: the script merges each +stack's JSON by splitting on whitespace and re-parsing the fragments, so the +documents are shredded into single characters (`stack_outputs` came back as a list +of 769 one-character strings, with every `OutputKey` lost). It fails silently under +`except: pass`. If you need stack outputs, query CloudFormation directly. + +The file is written to the repo root and is **not** in `.gitignore`. Harmless today +only because the broken merge drops the `-auth` secret — see `security.md`. diff --git a/kiro/agentcore-enterprise-platform/steering/patterns.md b/kiro/agentcore-enterprise-platform/steering/patterns.md new file mode 100644 index 0000000..502d8c2 --- /dev/null +++ b/kiro/agentcore-enterprise-platform/steering/patterns.md @@ -0,0 +1,682 @@ +# Situation → recipe + +Read this when someone describes their situation and you need to turn it into a +concrete sequence of commands rather than a tour of the options. + +Each recipe assumes: repo root, `.venv` active, `AWS_REGION` and credentials set. +`PREFIX` is `${PROJECT_NAME}-${ENVIRONMENT}`. + +Jump table: + +| They said | Recipe | +|---|---| +| "We're building our first agent platform" | [First platform](#first-platform) | +| "We already have an agent and need it governed" | [Govern an existing agent](#govern-an-existing-agent) | +| "We need agents that call other agents" | [Delegating agents](#delegating-agents) | +| "We're the platform team; other teams build on us" | [Platform for other teams](#platform-for-other-teams) | +| "We have to get through a security review" | [Security review](#security-review) | +| "We use LangGraph / CrewAI / our own framework" | [Framework swap](#framework-swap) | +| "We log in with Entra ID / Okta / Ping" | [Corporate IdP](#corporate-idp) | +| "Many agent teams, one governed tool catalogue" | [Central tool governance](#central-tool-governance) | +| "Nothing may reach the internet" | [No internet egress](#no-internet-egress) | +| "Prove our tenants are isolated" | [Tenant isolation](#tenant-isolation) | +| "We need end-to-end traces" | [End-to-end traces](#end-to-end-traces) | +| "Add our internal API as a tool" | [Add an internal API as a tool](#add-an-internal-api-as-a-tool) | +| "Agents must only call approved tools" | [Deny tools by policy](#deny-tools-by-policy) | +| "Half a day, minimal spend" | [Cheapest useful demo](#cheapest-useful-demo) | +| "What would production actually need?" | [The production gap](#the-production-gap) | + +--- + +## First platform + +**They have no agent in production and want the governed shape from day one.** + +Profile `greenfield` (3 4 5 6 9), ~18 minutes of deploy. Networking, security and +A2A all off — deliberately, so the first pass is about the trust root, the tool +gateway, and one working agent. + +```bash +export AWS_REGION=us-east-1 +export MODEL_ID=us.anthropic.claude-sonnet-5 +./scripts/deploy.sh workshop --dry-run --profile greenfield +./scripts/deploy.sh workshop --profile greenfield +``` + +Then prove it. One command covers the whole footprint and exits non-zero if any +part of it is broken: + +```bash +./scripts/deploy.sh verify +``` + +Run the individual tools when you need to show the room *which* layer works, or +when `verify` fails and you want to isolate it — each answers a different question: + +```bash +.venv/bin/python scripts/test_gateway.py # gateway serves tools +.venv/bin/python scripts/invoke.py "Reply with exactly: PLATFORM OK" +.venv/bin/python scripts/invoke.py --tools # registered on the GATEWAY, not the agent +.venv/bin/python scripts/check_observability.py +``` + +**Where the conversation should go next:** module 7 (add their own tool) and a +pattern swap. Those two are what make it their platform rather than a sample. + +**Do not** reach for `platform-team` because it sounds more complete. It turns on +networking, which bills hourly, and adds 20 minutes before anyone sees an agent +answer. + +--- + +## Govern an existing agent + +**They have a working agent on EC2, ECS, Lambda, or a laptop, and the ask is +governance — auth, an audited tool path, observability — not a rewrite.** + +Profile `migration` (3 4 6 7 9). Note it **skips module 5** and reaches the +gateway through module 7 instead. That ordering is the argument: an existing agent +gets governed by adding **tool targets**, not by rebuilding the agent. + +```bash +./scripts/deploy.sh workshop --profile migration +``` + +The sequence to walk with them: + +1. **3 + 4** — their agent now has an issuer to validate tokens against and a + credential provider to fetch its own gateway token from. Nothing about their + agent code changed yet. +2. **6** — lift the agent onto Runtime. Their framework stays; the pattern is a + config value (see [Framework swap](#framework-swap)). +3. **7** — move each capability the agent calls directly into a gateway target. + This is the step that produces the audit trail. +4. **9** — traces across the whole path. + +The honest version of the migration cost: their agent code has to (a) read +identity from the verified caller token rather than trusting a header, and +(b) call tools through the gateway's MCP client instead of direct SDK calls. +`agent-code/langgraph-agent/` and `agent-code/strands-agent/` are the two +worked examples — point at the code, not at a diagram. + +`workshop-simulation/` in the repo carries an end-to-end version of exactly this +story, including a pre-existing EC2-style agent to migrate. + +--- + +## Delegating agents + +**One agent cannot reasonably own everything — they want specialists.** + +Profile `multi-agent` (3 4 5 6 7 8 9), ~29 minutes. + +```bash +./scripts/deploy.sh workshop --profile multi-agent +.venv/bin/python scripts/invoke.py --a2a code-agent "Reply with exactly: A2A OK" +.venv/bin/python scripts/invoke.py --a2a research-agent "Summarise what you can do." +``` + +The architectural point to make, and it is a real one: the sub-agents get +**independent auth, scaling, and lifecycle**. The code-agent can be +rate-limited, redeployed, or IAM-restricted without touching the orchestrator. +That is not true of a single agent with many tools. + +Two facts that shape their design: + +- Sub-agents are guarded by **IAM/SigV4**, not a caller JWT — A2A is not a + client-facing protocol, so those runtimes get no JWT authorizer at all. The + boundary between "our agents" and "our users" is therefore a real boundary. +- The contract is **JSON-RPC 2.0 on port 9000** plus an agent card and `/ping`. + Anything they write must serve all three or it returns HTTP 424. + +Details in `agent-patterns.md`. + +**When to argue against it:** if the sub-agents would share one execution role and +one deployment cadence, they are getting the complexity of A2A and none of the +isolation. Tools on the gateway are the simpler answer. + +--- + +## Platform for other teams + +**They are a central team and their customer is other engineering teams.** + +Profile `platform-team` (3 4 5 A 6 7 8 9 C E), ~39 minutes. Networking and +security on — check the cost conversation first. + +```bash +export ORG_ID=$(aws organizations describe-organization --query Organization.Id --output text) +./scripts/deploy.sh workshop --profile platform-team +``` + +The three things this profile is actually demonstrating: + +1. **The SSM registry is the product interface.** Their teams read platform values + from `/{project}/{env}/*` instead of importing CDK constructs or hardcoding + ARNs. That is the seam that lets the platform change underneath consumers: + + ``` + /{project}/{env}/auth/issuer-url | user-pool-id | app-client-id | web-client-id | m2m-client-id + /{project}/{env}/identity/gateway-credential-provider-name + /{project}/{env}/gateway/url + /{project}/{env}/memory/memory-id | memory-arn + /{project}/{env}/runtimes/{component}/arn | id + /{project}/{env}/networking/vpc-id | private-subnet-ids | runtime-security-group-id + ``` + +2. **`--team` splits ownership.** `platform` owns networking/auth/identity/gateway/ + observability; `agent` owns the runtimes and memory; `security` owns the + security and observability stacks. + +3. **A targeted `destroy` refuses rather than cascading.** That refusal is the + multi-tenant safety property — one team cannot take the platform out from under + another. + +Note the ordering: **A (Memory) runs before 6.** The orchestrator depends on +memory, so running 6 first makes CDK create memory implicitly and module A then +reports "no changes," teaching the room something false. + +If they have many agent teams and want one governed tool catalogue, go to +[Central tool governance](#central-tool-governance). + +--- + +## Security review + +**There is a security or risk function that has to sign off.** + +Profile `security-focused` (3 4 5 6 9 E). It needs an Organizations id and +enables resource policies, the egress filter, Cedar, and traceability. Note it +sets `enable_a2a=false` on purpose — this profile is about the control plane, not +the agent fleet. + +```bash +export ORG_ID=$(aws organizations describe-organization --query Organization.Id --output text) +./scripts/deploy.sh workshop --profile security-focused +``` + +**Lead with the disclosure, not the feature list.** The credibility of this +accelerator in a review comes from saying these first: + +- Every security control is **off by default**. +- **`CREATE_COMPLETE` proves nothing is being enforced.** Module E's verify is a + stack-status check because there is no behavioural probe. +- Cedar ships in **`LOG_ONLY`** with one permit that is unconstrained on principal + and resource. It is not a default-deny gateway until three separate things + change. +- The egress filter **masks; it rarely blocks**, does no authorization, passes + unrecognised payload shapes through unchanged, and has no defined + fail-open/fail-closed behaviour. +- The VPC endpoint policy's org restriction covers **SigV4 callers only** — JWT + callers carry no IAM principal. +- Two `iam.*` library files are **reference policies nothing deploys**. +- The runtime role still holds `bedrock-agentcore:*` on `Resource: "*"`, + knowingly. + +Then show the mechanism, which is genuinely strong: + +```bash +make validate-controls && make test-controls + +# Prove a control is real without deploying: flag off → absent, flag on → present +cdk synth $PREFIX-gateway | grep -c PolicyEngine +cdk synth $PREFIX-gateway -c enable_cedar=true | grep -c PolicyEngine +``` + +And the org layer, **attached to a sandbox OU first**: + +```bash +cd terraform/org-guardrails +terraform init +terraform plan -var 'target_ids=["ou-sandbox-xxxx"]' +terraform apply -var 'target_ids=["ou-sandbox-xxxx"]' +``` + +Full control inventory, what each really enforces, and the SCP quota mechanics are +in `security.md`. The single sharpest control to show a reviewer is +`scp.identity.deny-workload-token-for-userid` — it closes an API that takes the +user identifier as an unverified string. + +--- + +## Framework swap + +**"Does this work with the framework we already use?"** + +This is the claim the whole pitch rests on, and it is one command: + +```bash +AGENT_PATTERN=langgraph-agent ./scripts/deploy.sh deploy --module 6 +.venv/bin/python scripts/invoke.py "Reply with exactly: LANGGRAPH LIVE" +``` + +One orchestrator redeploy, ~5–8 minutes of CodeBuild, no infrastructure change. A +bad pattern name is rejected before any AWS call. + +Seven patterns ship: `orchestrator`, `strands-agent`, `langgraph-agent`, +`claude-sdk-agent`, `claude-sdk-multi-agent`, `agui-strands-agent`, +`agui-langgraph-agent`. The `agui-*` ones need `invoke.py --agui`. + +**If their framework is not in the list**, the extension is small and worth +showing rather than promising: + +1. Add `agent-code//` with a `Dockerfile` — the build runs + `docker build --platform linux/arm64 -f /Dockerfile .` from + `agent-code/`, so `shared/` is in the build context. Anything importing + `shared/` must `COPY shared/ shared/`. +2. Add the name to `AGENT_PATTERNS` in `scripts/deploy.sh` (the up-front + validation list). + +That is it — no stack changes. A directory named `agui-*` is automatically given +the `AGUI` protocol; everything else gets `HTTP`. + +Prove a swap actually happened rather than trusting a green deploy — the image tag +is a content hash, so identical tags mean CodeBuild never rebuilt: + +```bash +aws ecr describe-images --repository-name $PREFIX-orchestrator \ + --query 'sort_by(imageDetails,&imagePushedAt)[-5:].{tags:imageTags,pushed:imagePushedAt}' +``` + +If they want the full matrix before committing, `agent-patterns.md` has it. + +--- + +## Corporate IdP + +**"Our people log in with Entra ID / Okta / Ping."** + +Module 4 handles it, but **only if a federated IdP was chosen** — the prompt +defaults to plain Cognito, and nothing silently half-configures. + +```bash +export IDP_TYPE=entra_id # or okta | ping +export IDP_TENANT_ID= # Entra +export IDP_CLIENT_ID= +./scripts/deploy.sh deploy --module 4 +``` + +The client secret is prompted for interactively (never echoed, never persisted to +`workshop.env`), stored in Secrets Manager as `$PREFIX-idp-client-secret`, and only +the **name** is passed to CDK. Override the name with `IDP_CLIENT_SECRET_NAME` to +use a secret they already own. + +One detail that costs an hour when it goes wrong: **the script strips whitespace +from the pasted secret.** A trailing newline from a copy-paste produces +`invalid_client` at the IdP, which looks like a misconfiguration rather than a +paste error. + +Cognito stays in the picture as the token issuer the platform validates against — +federation means their IdP authenticates the human and Cognito issues the token +the platform checks. The full Entra ID walkthrough, including verifying federation +without opening a browser, is `docs/ENTERPRISE_IDP.md` in the repo. + +Separately, the Google / GitHub / Notion **3LO** providers appear only when their +client ids are supplied. Those are for agents acting on a user's behalf against a +third-party API — a different thing from workforce login. Do not conflate them in +the same slide. + +--- + +## Central tool governance + +**Many agent teams, and the tool catalogue must be governed centrally.** + +`deployment.strategy: federated` in `platform.yaml`. One platform account owns the +gateway, tools, and Cognito; workload accounts own their runtimes and their own +memory. + +```yaml +project: acme-agents +environment: prod +region: us-east-1 +deployment: + strategy: federated + federation: + gateway_url: + issuer_url: + m2m_client_id: + m2m_client_secret_name: +``` + +```bash +.venv/bin/python -m infra_utils.platform_config platform.yaml # validate offline first +``` + +Three properties worth naming, all confirmed in the code: + +- **The same `platform.yaml` deploys both sides.** The account you deploy from + decides the role. Platform side gets auth/identity/gateway/observability and + **no runtimes**; workload side gets identity/memory/runtimes/observability. + Deploying from an account in neither list fails at synth with a message naming + both. +- **Federated trust is pure OAuth — no cross-account IAM on the data plane.** The + workload account's own credential provider holds the platform Cognito M2M client + id and secret; the token vault exchanges them at the platform token endpoint; the + platform gateway validates the resulting JWT against its own issuer and + **cannot tell which account called it**. That last part is either the feature or + the objection, depending on the customer — surface it rather than waiting. +- **Memory stays per workload account**, on purpose. Conversation history is the + tenant boundary and account isolation is the strongest wall available. + +An incomplete `federation` block raises a `ValueError` naming all four required +keys, so a partial config fails before any deploy. + +If instead each team wants a full independent copy, that is +`strategy: distributed`. Compare all three in `deploy.md`. + +--- + +## No internet egress + +**"Nothing in this VPC may reach the internet."** + +Give the honest answer first: **`enable_networking=true` is not that.** It creates +private subnets with a **NAT gateway** and a route to the internet — deliberately, +because AgentCore ENIs in a public subnet get no internet route at all and the +runtimes would lose Bedrock access. There is no flag that turns this into an +air-gapped VPC. + +What the platform does give them, and it is not nothing: + +```bash +export ORG_ID=$(aws organizations describe-organization --query Organization.Id --output text) +ENABLE_NETWORKING=true ./scripts/deploy.sh deploy --module C # ~3m30s, 38 resources +ENABLE_NETWORKING=true ./scripts/deploy.sh deploy --module 6 # ~6m — REQUIRED +ENABLE_NETWORKING=true ./scripts/deploy.sh deploy --module 8 # ~3m, if A2A is deployed +.venv/bin/python scripts/check_network.py +``` + +- runtime security group with **no inbound rules** and egress restricted to + **TCP 443 only** +- interface endpoints for Bedrock Runtime, ECR API, ECR Docker, CloudWatch Logs, + and the AgentCore Gateway, plus a free S3 **gateway** endpoint — which also keeps + image-layer pulls off NAT data processing +- an org-scoped endpoint policy on the AgentCore endpoint — **but only for SigV4 + callers**; JWT callers carry no `aws:PrincipalOrgID` + +**Three traps, all measured, all of which have bitten a real run:** + +1. **Module C alone proves nothing.** Runtimes stay `networkMode: PUBLIC` until + redeployed — which is why modules 6 and 8 are in the sequence above and not + optional. `check_network.py --expect-public` *passes* in the half-done state. +2. **`enable_networking=true` without `ORG_ID` does not fail.** It warns and creates + the endpoint with **no policy at all**. Treat that warning as an error. +3. **`ORG_ID` restricts one endpoint out of six.** `bedrock-runtime`, `ecr.*`, + `logs` and S3 keep the wide-open AWS default policy. Do not let "we set `ORG_ID`" + stand as an answer about the model-invocation path. + +**Budget for the AZ trap before you promise a timeline.** AgentCore only supports +some AZ *ids*, `max_azs=2` picks AZ *names*, and the mapping is per-account — so +whether this works is an accident of the account. On a fresh `us-east-1` account it +failed: `CREATE_COMPLETE` networking stack, then the runtime redeploy rejected with +`subnets are in unsupported availability zones`. Fixing it needs a **source edit** +(no flag exists) and a **destroy + recreate** (an in-place AZ change collides on +subnet CIDRs), which is about **15 minutes**. Run `check_network.py` in the target +account *before* the session — it is a two-minute pre-flight that turns a live +detour into a footnote. + +The real no-egress design is a follow-on engagement: an endpoint for every +dependency, no NAT gateway, and a validated list of what the chosen agent framework +calls at runtime. Say that plainly — it is a credible next step, not a gap in the +sample. + +If the requirement is really "tools must not reach public endpoints," the +control they want is the org SCP `scp.gateway.targets-require-private-endpoint`, +which is a different and much cheaper answer. See `security.md`. + +--- + +## Tenant isolation + +**"Show us that one customer's data cannot reach another's."** + +Three layers, and it is worth walking all three because each fails differently: + +1. **Identity.** The verified `sub` claim from the caller's JWT becomes the + AgentCore Memory `actor_id`. Every failure path in + `agent-code/shared/auth.py` **raises** — there is no fallback to an unverified + decode and no placeholder identity, precisely because a defaulted identity would + file two callers under one actor and mix their history. + + The asymmetry to disclose: a missing issuer is a hard reject, but an **empty** + `COGNITO_ALLOWED_CLIENTS` degrades quietly to "any client of the correct + issuer." Set both. + +2. **Resource policy.** `enable_resource_policies=true` puts an + `AWS::BedrockAgentCore::ResourcePolicy` on memory: allow this account, deny + anything outside the org. Requires `org_id`; the stack raises rather than + deploying a policy with a hole in it. + +3. **Account.** For hard multi-tenancy, `strategy: federated` keeps **memory per + workload account**. Account isolation is the strongest wall available and the + reason memory is not centralised. + +```bash +export ORG_ID=$(aws organizations describe-organization --query Organization.Id --output text) +./scripts/deploy.sh deploy --module A -c enable_resource_policies=true +.venv/bin/python scripts/test_memory.py +``` + +**Be careful what you claim from that last command.** `test_memory.py` uses **your +local credentials**, so it passes regardless of whether the *runtime role* can +reach memory. It is not an isolation proof. To check the role: + +```bash +aws iam list-role-policies --role-name -orchestrator-role +``` + +What is genuinely missing today, and worth saying: the platform gives per-actor +memory scoping and account-level separation, but the orchestrator's role holds +`bedrock-agentcore:*` on `Resource: "*"` — so isolation between tenants **inside +one account** rests on the agent code passing the right `actor_id`, not on IAM. +For a strict tenancy requirement, one account per tenant is the defensible answer. + +--- + +## End-to-end traces + +**"We need to see a request across the agent, the gateway, and the model."** + +```bash +./scripts/deploy.sh deploy --module 9 +.venv/bin/python scripts/check_observability.py +.venv/bin/python scripts/invoke.py "hi" +sleep 120 +.venv/bin/python scripts/check_observability.py --spans +``` + +Disclose the cost mechanic **before** deploying: `enable_transaction_search` +defaults to **true**, is **account- and Region-scoped**, changes span-ingestion +pricing account-wide, and **survives teardown** by design because other workloads +may come to depend on it. If that setting is not theirs to change: + +```bash +ENABLE_TRANSACTION_SEARCH=false ./scripts/deploy.sh deploy --module 9 +``` + +…and say plainly that tracing then does not work: every OTLP span batch is +rejected with HTTP 400 while the deploy still reports success. That failure mode is +the best possible argument for why the flag defaults on — a stack that says +`CREATE_COMPLETE` while silently discarding all telemetry. + +**Do not let anyone "fix" empty `aws xray batch-get-traces` results.** With +Transaction Search all spans are searchable in `aws/spans`, while the classic X-Ray +APIs only serve the indexed sample (default rule: 1%). An empty trace-API result is +expected. + +--- + +## Add an internal API as a tool + +**"How do we expose our own service to the agents?"** + +The strongest demonstrable claim in the whole accelerator: **agents pick up new +tools on their next discovery, with no agent redeploy.** Show it in that order. + +1. Write the handler in `tools//handler.py`. The tool name arrives in the + **context**, not the event: + + ```python + def handler(event, context): + delimiter = "___" + name = context.client_context.custom["bedrockAgentCoreToolName"] + name = name[name.index(delimiter) + len(delimiter):] + if name == "my_tool": + return {"content": [{"type": "text", "text": do_the_thing(event["some_arg"])}]} + return {"error": f"Unsupported tool: {name}"} + ``` + +2. Declare it in `app.py`'s `tool_configs` beside `sample-tool`, using + **PascalCase** keys (`Name`, `Description`, `InputSchema`, `Type`, + `Properties`, `Required`) — that is the CloudFormation shape, not MCP JSON. + +3. Deploy and verify: + + ```bash + ./scripts/deploy.sh deploy --module 7 + .venv/bin/python scripts/test_gateway.py + .venv/bin/python scripts/invoke.py --tools # the new tool is listed + ``` + +4. **Now the point** — an agent that was deployed *before* the tool existed uses + it: + + ```bash + .venv/bin/python scripts/invoke.py "Use my_tool on '…' and report what it returns." + ``` + +Two things to tell them before they write it: + +- **Write the tool `Description` for a model, not for a human skimming a table.** + It is the only thing the agent has when deciding whether this tool answers the + question. "The agent never calls our tool" is usually a vague description — or + the `orchestrator` pattern, which has no tools at all. +- **Their tool needs a Cedar permit** the moment `cedar_mode=ENFORCE`. The action + name is the full `___`. + +If what they want is a capability AWS operates rather than their own code, the +built-in connector path is different and shorter — but connectors are **regional** +and need their own IAM action on the gateway role. Both traps are in +`agent-patterns.md`. + +--- + +## Deny tools by policy + +**"Agents must only be able to call approved tools, and we need the evidence."** + +Cedar, rolled out in the only safe order: + +```bash +# 1. Attach in LOG_ONLY (the default) and generate real traffic +./scripts/deploy.sh deploy --module 5 -c enable_cedar=true +.venv/bin/python scripts/test_gateway.py +.venv/bin/python scripts/invoke.py "Use the text analysis tool on 'hello world'." + +# 2. Read the decision logs. Confirm what WOULD have been denied. +# 3. Narrow the shipped permit's principal and resource: +# control-library/cedar/gateway-default/10-permit-read-tools.cedar +# 4. Only then enforce: +./scripts/deploy.sh deploy --module 5 -c enable_cedar=true -c cedar_mode=ENFORCE +``` + +What to tell them up front: + +- **Cedar is implicit default-deny** — no permit means denied. That is the + property they want, and it is real. +- The shipped permit is **unconstrained on principal and resource**, so out of the + box any authenticated caller may invoke the sample tool on any gateway. Narrow it + before enforcing. +- **Never add a blanket `forbid`.** In Cedar a matching forbid overrides every + permit unconditionally — it would make the permits dead code and deny everything. + Keep any forbid narrow. +- **Every new tool is denied under `ENFORCE` unless a permit names it.** Correct + behaviour, and the most common "we broke the demo" moment. Build it into their + tool-onboarding checklist. +- Flags match the exact lowercase string `"true"`. `-c enable_cedar=True` silently + does nothing. + +At org scope, the matching control is +`scp.gateway.require-policy-engine`, which denies creating a gateway *without* a +Cedar engine in `ENFORCE`. That is how a central team makes this +non-optional — see `security.md`, including the fact that it deploys merged with +seven other gateway SCPs into one policy. + +--- + +## Cheapest useful demo + +**Half a day, someone else's account, and no appetite for surprise spend.** + +```bash +export AWS_REGION=us-east-1 +export MODEL_ID=us.anthropic.claude-sonnet-5 +./scripts/deploy.sh workshop --dry-run --profile greenfield # free +./scripts/deploy.sh workshop --profile greenfield +``` + +`greenfield` sets `enable_networking=false`, which is the whole cost story: **no +NAT gateway, no interface endpoints, nothing billing hourly.** What remains is +pay-per-use and near-free idle, plus CodeBuild minutes per container build. + +Three decisions to make explicitly: + +- **`--profile` alone deploys every stack the manifest declares**, not just the + profile's modules. Use `workshop --profile`, or scope with `--module`. For + `greenfield` the overshoot is cheap — 6 stacks, the extra one being Memory — + and the manifest does keep networking out of the app. What you lose is the + module-by-module walk and the verify step between layers. Note `--profile` + writes `platform.yaml`, so it is not a read-only way to look: use `--dry-run`, + or read `expected_stacks()` off the preset (`deploy.md`). +- **In someone else's account, `deploy --dry-run` is the trap that ruins the + "no surprise spend" promise.** Only `workshop` honours `--dry-run`; `deploy` + accepts it and deploys. Measured: `deploy --module 3 --dry-run` bootstrapped a + fresh Region and left a Cognito user pool standing. If the command you are about + to run to "just show them the plan" says `deploy`, it is not a preview. +- **`enable_transaction_search` defaults true** and is account-scoped. In someone + else's account, either get agreement or run module 9 with + `ENABLE_TRANSACTION_SEARCH=false` and say what that costs you. + +Capture, then destroy the same day: + +```bash +./scripts/deploy.sh export # workshop-outputs-.json — contains account ids +./scripts/deploy.sh destroy +``` + +If you never enabled networking there is no ENI drain to wait out, which is the +other reason `greenfield` is the right demo profile. + +--- + +## The production gap + +**"What else would we need before this is production?"** Worth having ready, +because answering it well is more persuasive than the demo. + +The accelerator is deliberately a starting point. The gaps that are real, in +roughly the order they bite: + +| Gap | Why it matters | Where to start | +|---|---|---| +| Controls are off by default | the shipped defaults are a working platform, not a hardened one | `security.md` | +| Cedar permits are unconstrained | narrow principal + resource, then `ENFORCE` | [Deny tools by policy](#deny-tools-by-policy) | +| Guardrail resolves to `DRAFT` | pin a published version | `security.md` | +| Runtime role holds `bedrock-agentcore:*` on `*` | scope it once the gateway/memory/A2A ARNs exist | `security.md` | +| Egress interceptor has no fail-open/fail-closed decision | a Bedrock throttle surfaces as a Lambda failure | `security.md` | +| CloudTrail is single-Region, bucket auto-deletes | do not present it as replacing an org trail | `security.md` | +| SNS alert topic has no subscriber and no CMK | alerts go nowhere by default | `security.md` | +| `iam.*` control-library files are reference-only | nothing deploys them | `security.md` | +| CI/CD is a reference pipeline, not a deployment | module D deploys nothing on purpose | `--module D` | +| No multi-Region story | the accelerator is single-Region | — | + +Framing that holds up: **every one of these is disclosed in the repo rather than +hidden**, which is the strongest signal about the accelerator's quality that you +can give a security-minded customer. Lead with the list, not with the demo. + +--- + +*Recipes here are grounded in the accelerator's source. Where a recipe states a +timing or a failure signature, it came from a real run — see the pinned commit in +`POWER.md`. Anything not confirmed either way is left out rather than hedged.* diff --git a/kiro/agentcore-enterprise-platform/steering/runbook-cost-audit.md b/kiro/agentcore-enterprise-platform/steering/runbook-cost-audit.md new file mode 100644 index 0000000..f8dea4f --- /dev/null +++ b/kiro/agentcore-enterprise-platform/steering/runbook-cost-audit.md @@ -0,0 +1,215 @@ +# What is this costing right now + +Read this when someone asks what this is costing, whether anything was left +running, whether the account is clean, what is still deployed from a previous +session, or why a bill went up after a workshop. + +**This is a runbook, not reference material.** Follow it in order. Anything that +creates, changes, deletes or bills goes one command at a time: state what it +does and what it costs, then stop and wait for approval. Never group one of +those with anything else. The halt conditions at the end are not advisory. + +Every command here is **read-only**. Nothing in this runbook changes, creates or +deletes anything, so approvals are cheap — ask for them freely rather than +guessing at an answer. That is the whole point: the alternative to running these +is someone estimating from memory, and the standing costs here are the ones people +estimate wrong. + +Answer three questions in order, because they have very different urgency: + +1. **What is billing per hour?** This is what makes "we left it up over the + weekend" expensive. It is also the shortest list. +2. **What is billing per month?** Small, and the reason a forgotten account + costs real money over a quarter. +3. **What survived a teardown?** Not everything the accelerator creates is owned + by a stack. + +If the answer to any of them is "more than they expected", hand off to +`teardown-platform` rather than deleting things here. + +## Step 0 — which account, and what is deployed + +```bash +aws sts get-caller-identity --query Account --output text +``` + +Say the account id back before reporting numbers. A cost report against the wrong +account is worse than no report. + +```bash +aws cloudformation describe-stacks \ + --query "Stacks[?starts_with(StackName,'agentcore-workshop-dev')].[StackName,StackStatus,CreationTime]" \ + --output table +``` + +`$PREFIX` defaults to `agentcore-workshop-dev` (`${PROJECT_NAME}-${ENVIRONMENT}`). +Substitute if either was overridden — and if you are auditing an account that hosted +a multi-team session, there may be several prefixes (`team-a-dev`, `team-b-dev`). +Drop the filter entirely if you are not sure what to look for: + +```bash +aws cloudformation describe-stacks \ + --query 'Stacks[?StackStatus!=`DELETE_COMPLETE`].[StackName,StackStatus]' --output table +``` + +`CreationTime` is the useful column. It turns "is this costing us anything" into +"this has been up for eleven days", which is the number that actually gets a +decision made. + +## Step 1 — the hourly meters + +Only `enable_networking=true` creates any of these, which means `greenfield`, +`migration` and `multi-agent` leave **nothing** billing hourly. If no +`-networking` stack appeared in Step 0, say so plainly and skip to Step 2 — +that is a genuinely good answer, not an incomplete one. + +```bash +aws ec2 describe-nat-gateways --filter Name=state,Values=available \ + --query 'NatGateways[].[NatGatewayId,VpcId,CreateTime]' --output table +``` + +One NAT gateway is expected per deployed VPC (`nat_gateways=1`). It bills per hour +plus per GB processed, and it is the single largest idle line item in the whole +platform. + +```bash +aws ec2 describe-vpc-endpoints \ + --query "VpcEndpoints[?VpcEndpointType=='Interface'].[VpcEndpointId,ServiceName,length(SubnetIds)]" \ + --output table +``` + +Expect **5** interface endpoints, each in **2** subnets, so **10 endpoint-AZ-hours** +— `bedrock-runtime`, `ecr.api`, `ecr.dkr`, `logs`, `bedrock-agentcore.gateway`. The +third column is the multiplier and is the part people miss. A sixth S3 endpoint of +type `Gateway` is free and will not appear in this filter. + +**There is no flag to reduce either number.** `enable_vpc_endpoints=True` is a +literal at the call site (`app.py:197`) and `max_azs=2` is hardcoded. If the answer +someone wants is "make the networking demo cheaper", the honest reply is: turn +networking off, or edit the stack. Do not go looking for a `-c` flag. + +## Step 2 — the monthly trickle + +Small individually. Worth listing because none of them appear in a stack list by a +name anyone recognises. + +```bash +aws kms list-aliases \ + --query "Aliases[?starts_with(AliasName,'alias/agentcore-workshop-dev')].[AliasName,TargetKeyId]" \ + --output table +``` + +Module E creates one customer-managed key with rotation on, aliased +`alias/$PREFIX-agentcore`. Per key per month. + +```bash +aws secretsmanager list-secrets --query 'SecretList[].[Name,CreatedDate]' --output table +``` + +Deliberately unfiltered: these are created from IdP or API-key prompts, so the +**names are whatever the operator typed** and no prefix filter is reliable. This is +the line item people forget, precisely because nothing in the stack list names it. + +```bash +aws logs describe-log-groups --log-group-name-prefix /aws/lambda/agentcore-workshop-dev \ + --query 'logGroups[].[logGroupName,storedBytes,retentionInDays]' --output table +``` + +A `retentionInDays` of `None` means never expires, and none of the accelerator's +groups set retention, so they accumulate for as long as the account does. + +**Keep this prefix-scoped.** The tempting version — +`logGroups[?storedBytes>\`0\`]` with no prefix — returned **187 groups** in a +lightly-used test account, almost all of them unrelated. An audit that buries its +finding in 187 rows has not answered the question. Step 3 has the full set of +prefixes worth sweeping. + +Not worth auditing, and say so rather than running commands to prove it: **ECR +image storage** is capped at the last 10 images by a lifecycle rule and the +repository is `RemovalPolicy.DESTROY` with `empty_on_delete=True` +(`stacks/runtime_stack.py:91-109`), so it goes away with the stack. **AgentCore +Runtime, Gateway, Code Interpreter and Cognito** are per-invocation or +per-request; idle they cost nothing, however alarming the stack list looks. + +## Step 3 — what survived, or is about to + +Two categories outlive the stacks that made them. + +```bash +aws logs describe-log-groups \ + --query "logGroups[?starts_with(logGroupName,'/aws/bedrock-agentcore/') || starts_with(logGroupName,'/aws/lambda/agentcore-workshop-dev') || starts_with(logGroupName,'/aws/codebuild/agentcore-workshop-dev')].[logGroupName,storedBytes]" \ + --output table +``` + +Log groups are created by the *service*, not the stack, so a clean delete leaves +them. A measured teardown left 13 — 7 Lambda, 3 CodeBuild, 3 AgentCore runtime. +Note that the runtime groups use **underscores** +(`/aws/bedrock-agentcore/runtimes/agentcore_workshop_dev_orchestrator`), so a +hyphenated prefix sweep silently finds none of them. Also note +`/aws/bedrock-agentcore/` is a **shared namespace** — measured in a test account +with no accelerator deployed at all, this query still returned runtime log groups, +from unrelated AgentCore work by other tools. **Match the project prefix, not the +service prefix**, and never report a `/aws/bedrock-agentcore/` hit as an orphan of +this platform without checking the name actually carries the project prefix. + +```bash +aws bedrock-agentcore-control list-agent-runtimes --query 'agentRuntimes[].agentRuntimeName' --output text +aws bedrock-agentcore-control list-gateways --query 'items[].name' --output text +aws bedrock-agentcore-control list-memories --query 'memories[].id' --output text +``` + +Control-plane resources that a failed teardown can strand. If a stack is gone but +its runtime still lists here, that is a real orphan and worth naming in the report. +Empty output from all three is the clean state. + +## Step 4 — the account-wide one + +```bash +aws xray get-trace-segment-destination +``` + +`Destination: CloudWatchLogs` with `Status: ACTIVE` means **Transaction Search is +on**, which changes span-ingestion pricing for the **entire account** — not just +this platform. It defaults on and **stays on after teardown by design**, because +other workloads may have come to depend on it. + +This is the one item on this list that a teardown will not fix and that you should +not silently switch off, because it is shared. Report it as a standing account-level +decision and let its owner decide. + +```bash +aws cloudtrail describe-trails --query 'trailList[].[Name,IsMultiRegionTrail]' --output text +``` + +Module E's trail (`$PREFIX-agentcore-trail`) is management-events-only and +single-region, so it is free — **unless the account already had a management-events +trail**, in which case this is the second copy and every event bills. Two rows here +is the finding. + +## Step 5 — report it as a decision, not a dump + +Give them, in this order: + +1. **Per hour, right now** — the NAT count and the endpoint-AZ count, or the + sentence "nothing in this account is billing hourly", which is often the true + answer and the one they most want. +2. **How long it has been up**, from `CreationTime`. This is what makes the number + mean something. +3. **Per month** — keys, secrets, retained logs. +4. **Orphans** — anything in Step 3 whose stack is gone. +5. **Account-wide** — Transaction Search state, and a second CloudTrail if present. + +Then one recommendation, and be direct about it: keep it up, tear it down today +(`teardown-platform`), or tear down only networking to stop the hourly meter while +keeping the platform. Give meters and counts, never invented dollar amounts — +rates vary by Region and change, and a wrong number here gets repeated to a +customer. + +## Halt conditions + +- The account id is not the one they named. Stop and confirm before reporting. +- You are about to quote a dollar figure. You do not have current pricing; give + the meters and let them price it, or point at Cost Explorer for actuals. +- An audit turns into a cleanup. Deleting things is `teardown-platform`, with its + own confirmations — do not start deleting from inside a read-only audit because + the answer looked bad. diff --git a/kiro/agentcore-enterprise-platform/steering/runbook-deploy-module.md b/kiro/agentcore-enterprise-platform/steering/runbook-deploy-module.md new file mode 100644 index 0000000..b9040fa --- /dev/null +++ b/kiro/agentcore-enterprise-platform/steering/runbook-deploy-module.md @@ -0,0 +1,179 @@ +# Deploy one module + +Read this when someone wants a single module rather than a whole profile walk - +adding memory, redoing a failed layer, swapping the agent pattern, or turning on +networking or security. + +**This is a runbook, not reference material.** Follow it in order. Anything that +creates, changes, deletes or bills goes one command at a time: state what it +does and what it costs, then stop and wait for approval. Never group one of +those with anything else. The halt conditions at the end are not advisory. + +The standalone `deploy --module` path is what people use to add a layer or redo a +broken one. It has three sharp edges the guided walk hides, and all three fail in +ways that look like something else. + +## The rule + +**Propose one command, state what it creates, its cost posture and its expected +duration, then stop and wait.** Read-only checks are cheap approvals — use them +freely instead of guessing. + +**Say this before the first `deploy`:** once the `-auth` stack exists, **every** +`deploy` prints the Cognito M2M client secret to stdout in plaintext, including +runs that never touch `-auth`. No `deploy` is screen-safe. + +**Never propose `deploy --dry-run`.** The flag is parsed and ignored on this +action; it bootstraps and deploys for real. To preview, use +`./scripts/deploy.sh workshop --dry-run` or `synth`. + +## Step 1 — check the dependencies exist (read-only) + +A module deployed onto a missing layer fails deep in CloudFormation with an error +about the wrong thing. Confirm the prerequisites first: + +| Want | Needs already deployed | Why | +|---|---|---| +| 3 | — | trust root | +| 4 | 3 | needs the Cognito M2M client | +| 5 | 3, 4 | gateway needs JWT auth + the credential provider | +| A | 3 | memory is independent of the gateway | +| 6 | 3, 4, 5 (+ A if using memory) | the runtime resolves gateway + identity from SSM | +| 7 | 5 | grows the same `-gateway` stack | +| 8 | 3, 4, 5, 6 | sub-agents reach the gateway the same way | +| 9 | 6 | nothing to observe without a runtime | +| B | 6 | redeploys the orchestrator | +| C | — | standalone VPC, but see the ordering trap below | +| E | — | standalone KMS + CloudTrail | + +```bash +aws cloudformation describe-stacks \ + --query "Stacks[?starts_with(StackName,'agentcore-workshop-dev')].[StackName,StackStatus]" \ + --output table +``` + +Read the statuses, not just the names. A stack in `UPDATE_ROLLBACK_COMPLETE` is +safe to deploy onto again. A stack in `ROLLBACK_COMPLETE` was never successfully +created and must be deleted before it can be recreated. + +## Step 2 — export the flag if this module lives behind one + +Only module 8 turns its own flag on. This is the single most common failure on +this path: + +| Module | Flag | App default | Bare `deploy --module` | +|---|---|---|---| +| 8 | `enable_a2a` | **`true`** | works | +| C | `enable_networking` | `false` | **fails**: `No stacks match the name(s) …-networking` | +| E | `enable_security` | `false` | **fails**: `No stacks match the name(s) …-security` | + +```bash +ENABLE_NETWORKING=true ./scripts/deploy.sh deploy --module C +ENABLE_SECURITY=true ./scripts/deploy.sh deploy --module E +``` + +The error's advice to "check CloudFormation console for details" is a dead end — +the stack was never synthesized, so there is nothing in the console to look at. + +## Step 3 — deploy + +```bash +./scripts/deploy.sh deploy --module +``` + +Never substitute `--profile` for `--module`. `--profile` materializes that +profile's preset as `platform.yaml` and then runs `cdk deploy --all` — the whole +manifest, not the profile's modules. To see the blast radius first without +writing anything, read `expected_stacks()` off the preset (`deploy.md`); +`ls --profile

` also rewrites `platform.yaml`. + +Typos used to be the sharpest edge here — a misspelled flag or value was +discarded, which left the stack list empty, and empty means `cdk deploy --all`. +Upstream made the parser fail closed (`scripts/deploy.sh:955-990`): unknown +options exit 1 with the valid list, and `--profile`/`--team` values are validated +for every action, not just `workshop`. `--stack` with no value is caught too. +Do not spend the room's attention pre-empting this any more. + +What still needs care, because neither is a spelling mistake: + +- **`--dry-run` on `deploy`** is accepted and ignored — only `workshop` honours + it. `deploy --module 3 --dry-run` bootstraps and deploys for real. +- **`--profile` with no other scope** is a legitimate command that means the whole + manifest. Interactively `confirm_footprint` will show you the list and ask; with + `--yes` or `NON_INTERACTIVE=1` it will not. + +The tell is the echoed scope line. `Workshop Module 6 → Stacks: …` or +`Team agent → Stacks: …` means the flag took. No such line means the run is +unscoped, and the next thing to happen is a full deploy. + +Modules with no stacks: `--module D` prints a pointer to `.gitlab-ci.yml` and +exits 0. `--module B` redeploys the orchestrator with the code interpreter and has +no verify command. + +## Step 4 — verify + +| Module | Verify | +|---|---| +| 3 | `aws ssm get-parameter --name /agentcore-workshop/dev/auth/issuer-url` | +| 4 | `aws ssm get-parameter --name /agentcore-workshop/dev/identity/gateway-credential-provider-name` | +| 5, 7 | `.venv/bin/python scripts/test_gateway.py` — then `invoke.py --tools` for 7 | +| A | `.venv/bin/python scripts/test_memory.py` | +| 6 | `.venv/bin/python scripts/invoke.py "Reply with exactly: WORKSHOP OK"` | +| 8 | `.venv/bin/python scripts/invoke.py --a2a code-agent "Reply with exactly: A2A OK"` | +| 9 | `.venv/bin/python scripts/check_observability.py` | +| C | `.venv/bin/python scripts/check_network.py` | +| E | stack `CREATE_COMPLETE` only — there is no behavioural verify | + +Two verifies mean less than they appear to, and saying so is the honest move: + +- **`test_memory.py` uses your local credentials**, so it passes regardless of + what the runtime role can actually do. It is not an isolation proof. +- **A `-security` stack reaching `CREATE_COMPLETE`** means the KMS key and trail + exist, not that anything is enforced. Cedar ships `LOG_ONLY` with an + unconstrained permit; the egress filter masks rather than blocks. + +## The two ordering traps + +**Module C after module 6 leaves the agent outside the VPC.** A runtime deployed +before networking keeps `networkMode: PUBLIC` until it is redeployed. If someone +adds networking to a running platform, module C alone is not enough — redeploy the +runtimes afterward: + +```bash +ENABLE_NETWORKING=true ./scripts/deploy.sh deploy --module 6 +ENABLE_NETWORKING=true ./scripts/deploy.sh deploy --module 8 +``` + +Then `check_network.py`. Without the redeploy it reports the agent is not in the +VPC and it is correct. + +**Module A after module 6 reports "no changes".** The orchestrator runtime +depends on the memory stack, so deploying 6 first creates memory implicitly. +That is why `platform-team` runs A before 6. Not a failure — just say what +happened rather than letting it read as a broken deploy. + +## Swapping the agent pattern + +Framework choice is decoupled from infrastructure — this is a module 6 redeploy, +no infra change: + +```bash +AGENT_PATTERN=langgraph-agent ./scripts/deploy.sh deploy --module 6 +``` + +Valid values are rejected up front with the list printed, so a typo fails fast +rather than deploying the default. Each swap is another arm64 CodeBuild build, so +budget ~7–8 minutes and mention the build cost. + +Use `strands-agent` or `langgraph-agent` when the demo needs tools or memory — +the default `orchestrator` has neither. For recall *across* sessions also set +`USE_LONG_TERM_MEMORY=true`, which defaults to `false`; without it a new +`--session` answers `NO RECORD`. + +## Halt conditions + +- The dependency check shows a required stack missing or in `ROLLBACK_COMPLETE`. +- The module needs a flag and the participant has not agreed to enable it — + `ENABLE_NETWORKING=true` starts an hourly meter, so that is a cost decision. +- The verify fails twice for the same reason. Route to + `steering/troubleshooting.md` by symptom rather than retrying. diff --git a/kiro/agentcore-enterprise-platform/steering/runbook-deploy-platform.md b/kiro/agentcore-enterprise-platform/steering/runbook-deploy-platform.md new file mode 100644 index 0000000..ee2d3b2 --- /dev/null +++ b/kiro/agentcore-enterprise-platform/steering/runbook-deploy-platform.md @@ -0,0 +1,290 @@ +# Deploy the platform, end to end + +Read this when someone wants to actually deploy the platform (a profile walk, a +workshop run, or a first build) rather than ask how it works. + +**This is a runbook, not reference material.** Follow it in order. Anything that +creates, changes, deletes or bills goes one command at a time: state what it +does and what it costs, then stop and wait for approval. Never group one of +those with anything else. The halt conditions at the end are not advisory. + +You are driving a real deployment that spends real money in someone's AWS +account. The participant approves every command. Your job is to make each +approval an informed one and to never let a broken layer be built on. + +## The rule that governs every step + +**Anything that creates, changes, deletes or bills goes one command at a time.** +For each of those state: + +1. The exact command, copy-pasteable, with the working directory if it is not + the repo root. +2. What it creates or changes — resources, not adjectives. +3. Its cost posture: `read-only` / `pay-per-use` / **`starts an hourly meter`**. +4. How long it should take, so silence is not mistaken for a hang. + +Local read-only checks — `pwd`, a `--version`, an SSM read — can go in one group. +Splitting five harmless reads across five approvals teaches the participant to +click without reading, which is the opposite of what the approvals are for. Two +things stand alone even though they are read-only: `aws sts get-caller-identity`, +because its output is a go/no-go you have to confirm out loud, and any command +whose result decides what you propose next. + +Never put a resource-creating command in a group, and never pair one with a read +— that is how something billable gets approved on the strength of the harmless +command next to it. Never run a billable command to learn something a read-only +call answers. If the participant says "just run everything", say once that you +will still stop before each command that creates resources, and continue — the +approvals are the point of this runbook. + +## If you cannot run commands yourself + +Some sessions have no shell — the tool is unavailable, or there is nobody present +to approve anything. Say so plainly, say that nothing has run and nothing was +created, and hand over the sequence for the participant to run themselves. + +When you do, **keep every `deploy` on its own line and never chain it to +anything with `&&`.** A handed-over block gets pasted whole, and a chain of +`deploy && verify` pairs then walks the entire profile unattended — which is +precisely the go/no-go this runbook exists to enforce, deleted. One command per +line, the verify under it, and a blank line and a comment at each module +boundary so the person can stop there. + +Say what you could not establish, too. An unverified prerequisite is not a +passed one: if you never ran `aws sts get-caller-identity`, the account is +unknown, and that is the fact to report rather than an assumption to build on. + +## Before the first command: two warnings that must be said out loud + +**Every `deploy` prints the Cognito M2M client secret to stdout in plaintext, +once the `-auth` stack exists — including runs that never touch `-auth`.** The +closing summary dumps the outputs of every prefix-matching stack, and that +secret is a CloudFormation export. **No `deploy` is screen-safe.** If this is a +workshop, a demo, or any shared screen, say this before the first deploy and let +them decide whether to stop sharing. + +**`--dry-run` only works on `workshop`.** The `deploy` action parses the flag and +never reads it, so `deploy --module 3 --dry-run` bootstraps the Region and +deploys for real. Never propose `deploy` with `--dry-run`; it reads as safe and +is not. + +## Phase 0 — establish where you are (all read-only) + +Do not assume the checkout, the Region, or the account. Where you are is one +group: + +```bash +pwd && git -C . rev-parse --short HEAD # confirm the accelerator checkout +echo "${AWS_REGION:-}" # unset means the CLI default, not us-east-1 +``` + +Which account is its own proposal, because its answer is a go/no-go rather than +a fact to note: + +```bash +aws sts get-caller-identity +``` + +Stop and confirm the account id and Region with the participant before going +further — deploying into the wrong account is the one mistake here with no undo. +If `AWS_REGION` is unset, propose setting it explicitly rather than relying on a +profile default: + +```bash +export AWS_REGION=us-east-1 +``` + +## Phase 1 — preflight (read-only) + +Each of these has produced a failed session. Check them before deploying, not +after a failure. + +```bash +python3.13 --version # must exist under exactly this name +bash --version # must be 4+; macOS /bin/bash is 3.2 and dies on declare -A +node --version && npm --version +npx cdk --version # must be current; see below +aws bedrock list-foundation-models --query "modelSummaries[?contains(modelId,'claude')].modelId" --output text +``` + +Two of these need interpretation rather than a pass/fail glance: + +- **The CDK CLI.** `requirements.txt` pins no upper bound on `aws-cdk-lib` and + the repo has no `package.json`, so pip installs the newest library while `npx` + uses whatever global CLI exists. A stale CLI fails at bootstrap with a + cloud-assembly schema mismatch. The accelerator's own prereq check prints + `✓ cdk` for any version — it only tests presence. If in doubt, propose + `npm install -g aws-cdk@latest`. +- **Bedrock model access.** An empty list here means module 6's first invoke will + fail with an access error ~8 minutes into the build. Fix it now, in the Bedrock + console, not then. + +Then the virtual environment: + +```bash +python3.13 -m venv .venv && source .venv/bin/activate && pip install -r requirements.txt +``` + +`source` does not persist between your tool calls. Prefix later Python commands +with the venv interpreter (`.venv/bin/python scripts/…`) rather than assuming an +activated shell. + +## Phase 2 — preview the whole plan for free + +This makes **zero AWS calls** — no credential check, no bootstrap. Always do it +before committing to a profile. + +```bash +./scripts/deploy.sh workshop --dry-run --profile +``` + +Confirm from the output, not from what you typed. A real dry run prints: + +``` +[INFO] Dry run: skipping prerequisite and credential checks (no AWS calls) +[INFO] DRY RUN — nothing will be deployed +``` + +and never reaches a `═══ Deploying ═══` header. If you see that header, it is +deploying — stop it. + +One exception: `--profile security-focused` hits the `ORG_ID` gate *before* the +plan prints, even in dry-run. Export any `o-…`-shaped value just to preview, or +the real one: + +```bash +export ORG_ID=$(aws organizations describe-organization --query Organization.Id --output text) +``` + +If the account is not in an Organization, say so and steer to another profile or +`enable_resource_policies=false` — do not let them discover it mid-walk. + +Read the printed sequence back against the profile they chose. If it does not +match `steering/deploy.md`, trust the output and say the docs are stale. + +## Phase 3 — the cost conversation, before anything bills + +Say this before the first billable command, in meters rather than dollars +(rates vary by Region and change): + +- `greenfield`, `migration`, `multi-agent` leave **nothing** billing hourly. +- `platform-team` and `security-focused` enable networking, which creates **1 NAT + gateway plus 5 interface endpoints across 2 AZs = 10 endpoint-AZ-hours**, + billing whether or not anything runs. There is **no flag** to reduce this — + `enable_vpc_endpoints=True` is a literal at the call site (`app.py:197`) and + `max_azs=2` is hardcoded. +- **Transaction Search defaults on** and changes span-ingestion pricing + **account-wide**, and it **stays on after teardown** by design. Add + `ENABLE_TRANSACTION_SEARCH=false` if a platform team owns tracing elsewhere — + but then module 9's tracing genuinely does not work, so make it a decision, not + a default. +- CodeBuild bills per container build, and every agent-pattern swap is another one. + +Then ask for an explicit go, and ask when they intend to tear down. A platform +that nobody agreed to destroy is the one still running next month. + +## Phase 4 — bootstrap the Region + +```bash +npx cdk bootstrap +``` + +Creates the `CDKToolkit` stack and its staging bucket. One-time per +account/Region, ~1–2 min, pay-per-use and negligible. Note that this is shared +infrastructure — do **not** delete it at teardown if anything else in the account +uses CDK. + +## Phase 5 — the module loop + +Walk the profile's sequence **in the profile's order, not numeric order**. +`platform-team` deliberately runs A before 6, because the orchestrator runtime +depends on the memory stack. + +| Profile | Sequence | +|---|---| +| `greenfield` | 3 4 5 6 9 | +| `migration` | 3 4 6 7 9 | +| `multi-agent` | 3 4 5 6 7 8 9 | +| `platform-team` | 3 4 5 A 6 7 8 9 C E | +| `security-focused` | 3 4 5 6 9 E | + +For each module, three proposals in order — deploy, verify, then a go/no-go. + +**Step 1, deploy.** Scope it with `--module`, never with `--profile` alone: +`--profile` materializes that profile's preset as `platform.yaml` and then runs +`cdk deploy --all`, which deploys everything the manifest declares. + +```bash +./scripts/deploy.sh deploy --module +``` + +Two modules need their flag exported on the standalone `deploy --module` path, +because only module 8 turns its own flag on: + +```bash +ENABLE_NETWORKING=true ./scripts/deploy.sh deploy --module C +ENABLE_SECURITY=true ./scripts/deploy.sh deploy --module E +``` + +Without them you get `No stacks match the name(s) …-networking`, and the error's +own advice to check the CloudFormation console is a dead end — the stack was +never synthesized. + +**Step 2, verify.** `CREATE_COMPLETE` proves resources exist, not that anything +works. Stacks have completed while the thing they promise was broken; that is why +these scripts exist. + +| Module | Stack(s) | Verify | Expect | +|---|---|---|---| +| 3 | `$PREFIX-auth` | `aws ssm get-parameter --name /agentcore-workshop/dev/auth/issuer-url` | ~2 min | +| 4 | `$PREFIX-auth` `$PREFIX-identity` | `aws ssm get-parameter --name /agentcore-workshop/dev/identity/gateway-credential-provider-name` | ~2 min | +| 5 | `$PREFIX-gateway` | `.venv/bin/python scripts/test_gateway.py` | ~3 min | +| A | `$PREFIX-memory` | `.venv/bin/python scripts/test_memory.py` | ~2 min | +| 6 | `$PREFIX-runtime-orchestrator` | `.venv/bin/python scripts/invoke.py "Reply with exactly: WORKSHOP OK"` | **~7–8 min** | +| 7 | `$PREFIX-gateway` | `test_gateway.py`, then `invoke.py --tools` | ~3 min | +| 8 | `$PREFIX-runtime-code-agent` `-research-agent` | `.venv/bin/python scripts/invoke.py --a2a code-agent "Reply with exactly: A2A OK"` | ~8 min | +| 9 | `$PREFIX-observability` | `.venv/bin/python scripts/check_observability.py` | ~3 min | +| C | `$PREFIX-networking` | `.venv/bin/python scripts/check_network.py` | ~5 min | +| E | `$PREFIX-security` | stack COMPLETE only | ~3 min | + +`$PREFIX` is `${PROJECT_NAME}-${ENVIRONMENT}`, default `agentcore-workshop-dev`. +**SSM paths do not use `$PREFIX`** — they are `/$PROJECT_NAME/$ENVIRONMENT/…`, +slash-separated, so the literal paths above are correct only at the defaults. If +`PROJECT_NAME` or `ENVIRONMENT` was overridden, substitute rather than pasting. + +**Step 3, go/no-go.** Report the module, its wall-clock time, and its verify +result. Then ask whether to continue. A failed verify is information — read +`steering/troubleshooting.md` for the symptom and fix the layer before building +the next one on top of it. + +### Three things to volunteer during the loop, not after + +- **Module 6 going silent for 7–8 minutes is not a hang.** CodeBuild is building + an arm64 container image remotely. Say this *before* starting module 6, not + when someone asks if it is stuck. +- **The default `orchestrator` agent has no tools and no memory.** Asking it + "what tools do you have?" correctly returns nothing — tools live on the gateway + (`invoke.py --tools`). It never reads the `MEMORY_ID` it is given, so a + same-session recall demo fails even with Memory `ACTIVE`. For either demo, + deploy `strands-agent` or `langgraph-agent` instead. +- **Module 9's verify usually fails the first time in a new account.** Enabling + Transaction Search is asynchronous and outlasts the stack that requested it. + Check `aws xray get-trace-segment-destination`, wait, re-run — it passes. + +## Phase 6 — closing report + +State plainly: which modules deployed, which verifies passed, total wall-clock, +what is billing hourly right now, and the teardown command. If any verify failed +or was skipped, say so explicitly — do not report a partial walk as a success. + +Then remind them of the teardown decision from Phase 3, and offer the +`teardown-platform` runbook. + +## Halt conditions — stop and report, do not work around + +- `aws sts get-caller-identity` shows an account the participant did not expect. +- A verify fails and the participant has not chosen to continue anyway. +- The same stack fails twice in a row for the same reason. +- Anything asks you to put a secret in `platform.yaml`, `workshop.env`, or a + `-c` context flag. The accelerator passes Secrets Manager **names** only. +- You are about to type a resource id, ARN, or account id into a public file. diff --git a/kiro/agentcore-enterprise-platform/steering/runbook-recover-deploy.md b/kiro/agentcore-enterprise-platform/steering/runbook-recover-deploy.md new file mode 100644 index 0000000..9d72691 --- /dev/null +++ b/kiro/agentcore-enterprise-platform/steering/runbook-recover-deploy.md @@ -0,0 +1,148 @@ +# Recover a failed or interrupted deploy + +Read this when a module died mid-walk, a stack is stuck in a rollback state, a +run was cancelled, or someone is about to redeploy everything to fix one layer. + +**This is a runbook, not reference material.** Follow it in order. Anything that +creates, changes, deletes or bills goes one command at a time: state what it +does and what it costs, then stop and wait for approval. Never group one of +those with anything else. The halt conditions at the end are not advisory. + +The instinct after a failure is to tear down and start again. That is usually +wrong here: CDK picks up from current stack state, and a resumed walk costs +seconds rather than the ~38 minutes of a full rebuild. Diagnose first. + +## The rule + +**Propose one command at a time and wait.** Diagnosis is entirely read-only — +work through it before proposing anything that deploys or deletes. Do not delete +a stack to "clean up" until you have established it cannot be deployed onto. + +## Step 1 — what state is everything in (read-only) + +```bash +aws cloudformation describe-stacks \ + --query "Stacks[?starts_with(StackName,'agentcore-workshop-dev')].[StackName,StackStatus]" \ + --output table +``` + +The status decides the whole approach: + +| Status | Meaning | Action | +|---|---|---| +| `CREATE_COMPLETE` / `UPDATE_COMPLETE` | fine | move on | +| `UPDATE_ROLLBACK_COMPLETE` | update failed, previous version intact | **safe to deploy onto again** — just re-run | +| `ROLLBACK_COMPLETE` | initial create failed; never existed successfully | must be **deleted** before it can be recreated | +| `*_IN_PROGRESS` | still working | **wait** — do not start a second deploy | +| `DELETE_FAILED` | see the teardown-specific causes below | usually a stabilization timeout, not real | +| `UPDATE_ROLLBACK_FAILED` | rollback itself failed | needs `continue-update-rollback` | + +Then get the actual cause — the console link in the CLI output is rarely the +fastest path: + +```bash +aws cloudformation describe-stack-events --stack-name \ + --query "StackEvents[?ResourceStatus=='CREATE_FAILED'||ResourceStatus=='UPDATE_FAILED'].[LogicalResourceId,ResourceStatusReason]" \ + --output table +``` + +Read the **first** failure chronologically, not the last. CloudFormation reports +the cascade too, and the last event is usually a consequence. + +## Step 2 — the failures that are not what they say + +Check these before treating a message at face value: + +**`No stacks match the name(s) …-networking` / `…-security`.** The stack was never +synthesized because the feature flag was off — nothing failed, and there is +nothing in the console to look at. Re-run with the flag: + +```bash +ENABLE_NETWORKING=true ./scripts/deploy.sh deploy --module C +ENABLE_SECURITY=true ./scripts/deploy.sh deploy --module E +``` + +**A cloud-assembly schema mismatch at bootstrap.** The global CDK CLI is older +than the `aws-cdk-lib` pip installed. The prereq check passes it anyway because it +only tests that `cdk` exists. Fix: `npm install -g aws-cdk@latest`. + +**A model access error ~8 minutes into module 6.** No Claude model is enabled in +Bedrock in this Region. The build succeeded; the first invoke did not. + +**`ResourceNotFoundException` mentioning a Legacy model.** The hardcoded model id +aged out. The message says "Access denied" but access is not the problem — set +`MODEL_ID` to a current model. + +**`declare -A: invalid option`.** Someone ran it under macOS `/bin/bash` 3.2. +`brew install bash`, then invoke as `bash scripts/deploy.sh`. + +**A "dry run" that deployed.** `--dry-run` is only read by `workshop`. On `deploy` +it is parsed and ignored. If a `deploy --dry-run` created resources, that is the +cause, and the resources are real — treat this as an unplanned deploy and check +what landed, including a `CDKToolkit` stack in an unintended Region. + +**A misspelled flag that deployed everything.** The parser ends in `*) shift ;;`, +so `--modul 6` drops the flag and deploys the whole app. Check the blast radius +against what they intended before deciding what to remove. + +For anything not listed, route by symptom in `steering/troubleshooting.md` — it is +indexed by what the participant sees. + +## Step 3 — clear the state that blocks a retry + +Only for `ROLLBACK_COMPLETE`, which cannot be deployed onto: + +```bash +aws cloudformation delete-stack --stack-name +aws cloudformation wait stack-delete-complete --stack-name +``` + +For `UPDATE_ROLLBACK_FAILED`: + +```bash +aws cloudformation continue-update-rollback --stack-name +``` + +State plainly what deleting the stack destroys before proposing it. Deleting +`-auth` invalidates every downstream reference to the Cognito pool and forces a +rebuild of the layers above it — that is a much larger action than it looks. + +## Step 4 — resume, do not restart + +If the walk died partway through a profile sequence, resume at the failed module: + +```bash +./scripts/deploy.sh workshop --from 6 +``` + +`--from` is validated against the selected profile's sequence and fails with the +full sequence printed if the module is not in it. Skipped modules are logged, one +`Skipping module N (--from 6)` line each — not silently dropped. + +**A resumed run reprinting earlier stacks is not a re-deploy.** Module 6's +`cdk deploy` names its dependencies, so `--from 6` walks `-auth`, `-identity` and +`-gateway` on the way through. Each returns `✅ (no changes)` with +`Deployment time: 0s` — about 18 seconds of detour in a measured run. Read the +`(no changes)` before concluding `--from` was ignored, and say this in advance so +nobody panics at seeing module 3 scroll past again. + +If a single module failed rather than the walk, re-run just that module — +`deploy --module ` — after fixing the cause. + +## Step 5 — confirm the recovery + +Re-run the failed module's verify script, not just the deploy. A stack that +reaches `CREATE_COMPLETE` on the retry can still be broken in the way that caused +the original failure. The `verify-platform` runbook has the per-layer matrix. + +## What not to do + +- **Do not tear down and rebuild to fix one layer.** It costs ~38 minutes, and if + the cause was a prerequisite (CDK version, model access, bash version) the + rebuild fails the same way at the same place. +- **Do not run a second deploy while one is `*_IN_PROGRESS`.** You get a + confusing failure on top of a working deploy. +- **Do not delete `CDKToolkit`.** It is shared with anything else in the account + using CDK. +- **Do not retry more than twice for the same reason.** Two identical failures + means the cause is upstream of the command. Stop and report. diff --git a/kiro/agentcore-enterprise-platform/steering/runbook-teardown-platform.md b/kiro/agentcore-enterprise-platform/steering/runbook-teardown-platform.md new file mode 100644 index 0000000..08c0ef1 --- /dev/null +++ b/kiro/agentcore-enterprise-platform/steering/runbook-teardown-platform.md @@ -0,0 +1,220 @@ +# Tear down the platform + +Read this when someone is finished with a deployment, a session is ending, or +they need to confirm an account is actually clean. + +**This is a runbook, not reference material.** Follow it in order. Anything that +creates, changes, deletes or bills goes one command at a time: state what it +does and what it costs, then stop and wait for approval. Never group one of +those with anything else. The halt conditions at the end are not advisory. + +A destroy that "finished" is not the same as an account that is clean. This +runbook is built from a measured teardown: the destroy exited non-zero after 11.6 +minutes having deleted 4 of 10 stacks, and the NAT gateway was still `available`. +Check, do not assume. + +## The rule + +**Propose one command at a time and wait.** Deletions are irreversible, so state +what each one destroys before proposing it. Every check here is read-only — run +them freely. + +## Step 1 — say what is about to be destroyed (read-only) + +```bash +aws cloudformation describe-stacks \ + --query "Stacks[?starts_with(StackName,'agentcore-workshop-dev')].[StackName,StackStatus]" \ + --output table +``` + +Confirm the account and Region out loud before deleting anything: + +```bash +aws sts get-caller-identity && echo "${AWS_REGION:-}" +``` + +Ask explicitly whether anything in this platform is still needed. Also ask +whether anything **else** in the account depends on `CDKToolkit` — the answer is +usually yes, and it is not part of this teardown. + +## Step 2 — destroy, and check the exit code + +The exit code is the whole point of this step. `destroy` runs +`npx cdk destroy --all --force`, and **CDK stops at the first stack it cannot +delete.** Deletion order is reverse-dependency, which puts `-networking` **last** — +so an unrelated failure early in the order leaves the NAT gateway running. + +```bash +./scripts/deploy.sh destroy; echo "destroy rc=$?" +``` + +~12 minutes for a full platform. Do not report success without reading that `rc`. + +**A full destroy now sweeps behind itself, and you must know which prompts are +which.** After `--all`, `sweep_leftovers` (`scripts/deploy.sh:694-751`) asks +CloudFormation what still exists under the prefix — the stacks the *current* +config cannot see, which is the profile-switch case where a manifest with +networking off leaves a live NAT — and separately lists the secrets this script +creates outside CloudFormation (`-idp-client-secret`, the three 3LO OAuth +secrets, the three API keys). It only runs on an untargeted destroy, so +`destroy --stack ` never sweeps. + +Three behaviours, and the middle one is the one that bites: + +| Invocation | What the sweep does | +|---|---| +| interactive | asks per category: stacks, then secrets | +| `--yes` | deletes both, secrets with **no recovery window** | +| `NON_INTERACTIVE=1` without `--yes` | **reports and leaves** — "They may bill hourly" | + +That last row is deliberate: CI must not delete resources the config does not +declare. It also means a scripted teardown can exit 0 having told you, in a +warning you did not read, that the NAT is still running. If you are tearing down +from CI, either pass `--yes` or treat the sweep's output as a task list. + +The sweep covers stacks and secrets. It does **not** cover log groups or the +`agentic_ai` ENIs, which is why Steps 3 and 5 still exist. + +## Step 3 — confirm the expensive things are actually gone + +Do this whatever the exit code said. This is the check that catches an aborted +destroy: + +```bash +aws ec2 describe-nat-gateways --filter Name=state,Values=available \ + --query 'NatGateways[].NatGatewayId' --output text +``` + +Empty is the only acceptable answer. If it returns an id, the hourly meter is +still running regardless of what the destroy printed. + +```bash +aws cloudformation describe-stacks \ + --query "Stacks[?starts_with(StackName,'agentcore-workshop-dev')].[StackName,StackStatus]" \ + --output table +``` + +## Step 4 — the two failures that are expected + +Neither is real breakage. Recognising them is the difference between a five-minute +finish and an afternoon of debugging. + +### `DELETE_FAILED` with `NotStabilized` on an AgentCore runtime + +The message is "Request timed out while deleting +`AWS::BedrockAgentCore::Runtime`". That is a **stabilization timeout, not a +failed deletion** — the runtime is already gone. Confirm it, then just retry: + +```bash +aws bedrock-agentcore-control list-agent-runtimes --query 'agentRuntimes[].agentRuntimeName' --output text +``` + +Empty means the runtime is deleted and only CloudFormation's bookkeeping is +behind. Retry the stack delete — it took **34 seconds** in the measured case: + +```bash +aws cloudformation delete-stack --stack-name +aws cloudformation wait stack-delete-complete --stack-name +``` + +Ignore the handler's advice to delete it from the AWS console. There is nothing +there to delete. + +### The networking stack failing on subnets or security groups + +AgentCore's `agentic_ai` ENIs outlive their runtimes and hold the subnets. **Two +different clocks, and conflating them causes the wrong decision:** the delete +fails **fast**, in about 17 minutes, and then you wait roughly **8 hours** before a +retry succeeds. It is not an 8-hour hang. + +```bash +aws ec2 describe-network-interfaces \ + --filters Name=interface-type,Values=agentic_ai \ + --query 'NetworkInterfaces[].[NetworkInterfaceId,Status]' --output table +``` + +These **cannot be force-deleted.** `RequesterManaged` is `false` and the +description is blank, so they look account-owned, but the attachment is +`ela-attach-…` with `InstanceOwnerId: amazon-aws`. Manual +`delete-network-interface` is refused. There is no force. Wait and retry the stack +delete later. + +**The NAT and the endpoints delete before the failure**, so once the destroy has +actually reached `-networking`, the wait costs nothing meaningful. That is only +true if it got there — which is why Step 3 exists. + +### If the destroy aborted before reaching networking + +CDK's serial deletion means one stuck stack holds the rest hostage, including +`-auth` — the stack whose CloudFormation export carries the M2M client secret in +plaintext. Delete the blockers individually so the queue drains: + +```bash +aws cloudformation delete-stack --stack-name agentcore-workshop-dev-gateway +aws cloudformation delete-stack --stack-name agentcore-workshop-dev-identity +aws cloudformation delete-stack --stack-name agentcore-workshop-dev-security +aws cloudformation delete-stack --stack-name agentcore-workshop-dev-auth +``` + +Then re-run `./scripts/deploy.sh destroy` for whatever is left. + +## Step 5 — sweep the orphans + +Service-created log groups are **not** owned by the stacks, so they survive a +clean delete. A real teardown left **13** across three classes: 7 Lambda, 3 +CodeBuild, 3 AgentCore runtime. + +Two traps make a naive sweep miss or over-reach: + +- **Runtime log groups use underscores.** `$PREFIX` is `agentcore-workshop-dev` + but the runtime group is `/aws/bedrock-agentcore/runtimes/agentcore_workshop_dev_orchestrator`. + A sweep keyed on the hyphenated prefix finds none of them. +- **`/aws/bedrock-agentcore/` is a shared namespace.** Deleting the prefix + wholesale takes out unrelated groups — 15 of them in the measured account. Match + the project prefix, not the service prefix. + +List first, delete second — and note that **zsh does not word-split unquoted +variables**, so a `for g in $LIST` loop passes the whole list as one name and +fails with a length error: + +```bash +PREFIX=agentcore-workshop-dev +aws logs describe-log-groups --query "logGroups[?\ +starts_with(logGroupName,'/aws/lambda/${PREFIX}')||\ +starts_with(logGroupName,'/aws/codebuild/${PREFIX}')||\ +starts_with(logGroupName,'/aws/bedrock-agentcore/runtimes/${PREFIX//-/_}')\ +].logGroupName" --output text | tr '\t' '\n' > /tmp/orphans.txt + +cat /tmp/orphans.txt # review before deleting + +while IFS= read -r g; do + [ -n "$g" ] && aws logs delete-log-group --log-group-name "$g" +done < /tmp/orphans.txt +``` + +Show the participant the list before the delete loop. Log storage is cheap, so +leaving them is a valid choice — silently deleting the wrong ones is not. + +## Step 6 — what deliberately stays, and say so + +Do not delete these quietly, and do not report the account as clean without +naming them: + +- **`CDKToolkit` and its staging bucket** — shared with anything else in the + account using CDK. +- **Transaction Search stays enabled account-wide, by design.** It changes + span-ingestion pricing and other workloads may have come to depend on it. It is + the one thing a teardown does not undo, and it is an **account-scoped** change + someone should know about. +- **Secrets Manager entries** may be in a recovery window rather than deleted. +- **ECR images** persist if the repository was not part of a deleted stack. + +## Step 7 — report + +State: exit code, stacks deleted, stacks remaining and why, whether any hourly +meter is still running, orphans found and whether they were removed, and what was +deliberately left. If anything is still standing, give the retry command and the +time to retry it — do not leave a partial teardown described as done. + +If a stack is waiting on ENI drain, say plainly: nothing is billing hourly, retry +the delete in about 8 hours, and here is the command. diff --git a/kiro/agentcore-enterprise-platform/steering/runbook-verify-platform.md b/kiro/agentcore-enterprise-platform/steering/runbook-verify-platform.md new file mode 100644 index 0000000..cd39727 --- /dev/null +++ b/kiro/agentcore-enterprise-platform/steering/runbook-verify-platform.md @@ -0,0 +1,146 @@ +# Verify the platform + +Read this when someone asks whether their platform is healthy, wants to confirm +a deploy really worked, or needs evidence for a review. + +**This is a runbook, not reference material.** Follow it in order. Anything that +creates, changes, deletes or bills goes one command at a time: state what it +does and what it costs, then stop and wait for approval. Never group one of +those with anything else. The halt conditions at the end are not advisory. + +`CREATE_COMPLETE` proves resources exist. It does not prove they work — stacks +have completed while the thing they promise was broken, which is why these +scripts exist. This runbook produces evidence, and is honest about the limits of +each piece of it. + +## The rule + +**Propose one command at a time and wait.** Almost everything here is read-only, +so these are cheap approvals — but `invoke.py` calls a model and bills per token, +so label it pay-per-use rather than free. + +## Step 0 — what is actually deployed (read-only) + +Do not verify from an assumption about which modules were run. + +```bash +aws cloudformation describe-stacks \ + --query "Stacks[?starts_with(StackName,'agentcore-workshop-dev')].[StackName,StackStatus]" \ + --output table +``` + +Build the checklist from what this returns. `$PREFIX` defaults to +`agentcore-workshop-dev` (`${PROJECT_NAME}-${ENVIRONMENT}`). The SSM paths below +are **not** `$PREFIX` — they are `/$PROJECT_NAME/$ENVIRONMENT/…`, slash-separated, +so substitute if either was overridden rather than pasting the literal. + +## Step 1 — run the built-in verify first + +```bash +./scripts/deploy.sh verify; echo "verify rc=$?" +``` + +It derives the footprint from this configuration and runs the matching check for +each promised stack — gateway, memory, observability, networking, and a live +invoke per runtime — exiting non-zero if any of them fails. Read the `Footprint:` +line it prints, not just the verdict: it tells you which claims were actually +tested. + +That gets you most of the matrix below in one read-only pass. Go on to the matrix +anyway, for the two reasons it still exists: + +- **`verify` has no check for `-auth`, `-identity`, `-security`, or a `uc-*` + use-case stack**, and it omits `--spans`. Those rows are yours. +- **When `verify` fails, it tells you *which* check failed, not why.** The matrix + is how you isolate it — and `troubleshooting.md` is how you fix it. + +Do not report a platform as verified on `rc=0` alone. Say which footprint it +checked. + +## Step 2 — the matrix + +Run only the rows whose stack exists. Report each as PASS / FAIL / SKIPPED with +the reason, and never mark a row PASS because the stack was complete. + +| Layer | Stack | Command | Cost | +|---|---|---|---| +| Identity | `-auth` | `aws ssm get-parameter --name /agentcore-workshop/dev/auth/issuer-url` | read-only | +| Credential provider | `-identity` | `aws ssm get-parameter --name /agentcore-workshop/dev/identity/gateway-credential-provider-name` | read-only | +| Gateway | `-gateway` | `.venv/bin/python scripts/test_gateway.py` | read-only | +| Tools visible to the agent | `-gateway` | `.venv/bin/python scripts/invoke.py --tools` | pay-per-use | +| Memory | `-memory` | `.venv/bin/python scripts/test_memory.py` | read-only | +| Agent | `-runtime-orchestrator` | `.venv/bin/python scripts/invoke.py "Reply with exactly: WORKSHOP OK"` | pay-per-use | +| A2A | `-runtime-code-agent` | `.venv/bin/python scripts/invoke.py --a2a code-agent "Reply with exactly: A2A OK"` | pay-per-use | +| Observability | `-observability` | `.venv/bin/python scripts/check_observability.py` | read-only | +| Traces searchable | `-observability` | `.venv/bin/python scripts/check_observability.py --spans` | read-only | +| Network isolation | `-networking` | `.venv/bin/python scripts/check_network.py` | read-only | +| Security controls | `-security` | see "what a green security stack does not prove" | read-only | + +## Step 3 — read the results honestly + +Four of these mean less than their names suggest. If you report them without the +caveat, you have handed someone false assurance — which is worse than a failure +they can act on. + +**`test_memory.py` is not an isolation proof.** It uses **your local +credentials**, so it passes regardless of what the runtime role is permitted to +do. It proves the memory resource works, not that the agent can reach it. + +**`invoke.py --tools` answers a narrower question than it looks like.** It lists +what the *gateway* exposes. The default `orchestrator` agent consumes none of +them — it ships toolless. So a populated `--tools` list plus an agent that says +it has no tools is two correct answers, not a contradiction. + +**A green `-security` stack proves existence, not enforcement.** The KMS CMK and +CloudTrail exist. Cedar ships in `LOG_ONLY` with a permit that is unconstrained on +principal and resource; the egress filter masks rather than blocks; +`IsMultiRegionTrail` is `false` and hardcoded. Enumerate what is actually +enforced from `steering/security.md` rather than reporting the stack status. + +**`enable_networking=true` is not air-gapped.** Private subnets keep a NAT route, +by design. `check_network.py` proves the ENIs are in the private subnets, not that +egress is closed. Say which claim you are making. + +## Step 4 — the two failures that are usually not failures + +**`check_observability.py` fails the first time in a new account.** Enabling +Transaction Search is asynchronous and outlasts the stack that requested it. Check +the destination, wait, re-run: + +```bash +aws xray get-trace-segment-destination +``` + +`--spans` is deliberately not part of module 9's verify because span delivery lags +behind the deploy. A fresh platform with no traffic has no spans to find; generate +one with `invoke.py` first, then wait before asserting. + +**`check_network.py` reporting the agent is not in the VPC** is correct if the +runtimes were deployed before networking. A runtime keeps `networkMode: PUBLIC` +until it is redeployed. The fix is a redeploy, not a re-verify: + +```bash +ENABLE_NETWORKING=true ./scripts/deploy.sh deploy --module 6 +``` + +`check_network.py` stops at the first failure, so fix and re-run rather than +reading one failure as the complete picture. `--expect-public` asserts the +opposite posture, for confirming a deliberately public deployment. + +## Step 5 — report + +Give the matrix, then three sentences: what is proven, what is unproven and why, +and what to do about any FAIL. If a row was skipped because its stack is not +deployed, say "not deployed" rather than leaving it blank — an empty cell reads +as a pass. + +For a security review, `steering/security.md` has the control-by-control +inventory, including which controls are advisory in their shipped configuration. +That inventory is the deliverable, not this matrix. + +## If something fails and you cannot place it + +Route by **symptom** in `steering/troubleshooting.md` — it is indexed by what the +participant sees, not by which script emitted it. Collect the evidence listed at +the end of `steering/verify.md` before escalating: the command, its full output, +the stack status, and the Region and account. diff --git a/kiro/agentcore-enterprise-platform/steering/security.md b/kiro/agentcore-enterprise-platform/steering/security.md new file mode 100644 index 0000000..d991f16 --- /dev/null +++ b/kiro/agentcore-enterprise-platform/steering/security.md @@ -0,0 +1,723 @@ +# Security controls + +Read this when a security, network, or compliance reviewer is in the room — or +before anyone claims this deployment is hardened. + +**Two sentences to lead with, because they set every expectation correctly:** + +1. **Every security control is off by default.** The defaults are a working + platform, not a hardened one. Two flags elsewhere do default on — + `enable_transaction_search` (tracing does not work without it) and + `enable_a2a`. +2. **`$PREFIX-security` reaching `CREATE_COMPLETE` means resources exist, not + that anything is being enforced.** Module E's verify is a stack-status check + precisely because there is no behavioural probe for "is this control working." + +The credibility of this accelerator in a security review comes from the second +point. Say the honest version before a reviewer finds it. + +One finding does not wait for a flag and is not hypothetical: the `-auth` stack +publishes the **Cognito M2M client secret in plaintext** as a CloudFormation +export. Read "The Cognito M2M client secret is a plaintext CloudFormation export" +below before any review, and before putting a deploy on a shared screen. + +--- + +## The scope-split model + +One control library, two engines. Every control is authored **once** as valid +JSON or Cedar in `control-library/`, indexed by `catalog.yaml`, with +`<>` parameters injected at deploy time. + +| | Owns | Reads from | +|---|---|---| +| **Terraform** `terraform/org-guardrails/` | org scope — SCPs | `control-library/scp/` | +| **CDK (Python)** | account/workload scope — resource policies, VPCE policy, Cedar, Guardrails, interceptor | `control-library/` via `infra_utils/policy_loader.py` | + +Two consequences worth naming: + +- A policy fix lands in both engines at once — no duplicated policy bodies + drifting across languages. +- Because the artifacts are **valid JSON, not templated `.tftpl`**, checkov, + IAM Access Analyzer and cfn-guard can scan every file in CI. Sentinels are + `<>` specifically so they never collide with IAM policy variables + (`${aws:...}`) or Terraform `templatefile()` syntax. + +That portability is the answer to "we don't use Terraform": the same JSON goes +into Control Tower custom controls, AFT customizations, CloudFormation StackSets +(`AWS::Organizations::Policy`), or a console paste. Terraform is the path this +repo ships and tests; the JSON is what the customer keeps. + +```bash +make validate-controls # control-library ↔ catalog.yaml consistency +make test-controls # pytest tests/ -q +``` + +--- + +## What the library actually contains + +15 controls in `catalog.yaml`. Read this table as the inventory to walk a +reviewer through: + +| Control id | Type | Scope | Applied by | +|---|---|---|---| +| `scp.memory.enforce-cmk` | SCP | org | Terraform | +| `scp.identity.deny-workload-token-for-userid` | SCP | org | Terraform | +| `scp.gateway.require-cmk` | SCP | org | Terraform *(merged)* | +| `scp.gateway.deny-no-auth` | SCP | org | Terraform *(merged)* | +| `scp.gateway.require-policy-engine` | SCP | org | Terraform *(merged)* | +| `scp.gateway.enforce-approved-idp` | SCP | org | Terraform *(merged)* | +| `scp.gateway.restrict-protocol` | SCP | org | Terraform *(merged)* | +| `scp.gateway.targets-require-private-endpoint` | SCP | org | Terraform *(merged)* | +| `scp.gateway.targets-restrict-credential-provider` | SCP | org | Terraform *(merged)* | +| `scp.gateway.targets-restrict-type` | SCP | org | Terraform *(merged)* | +| `resource-policy.memory.in-account-only` | resource policy | workload | CDK, `enable_resource_policies` | +| `vpce.agentcore-in-org` | VPCE policy | account | CDK, `enable_networking` + `org_id` | +| `guardrail.egress-default` | Guardrail | workload | CDK, `enable_egress_filter` | +| `cedar.gateway-default.permit-read` | Cedar | workload | CDK, `enable_cedar` | +| `iam.runtime-execution-least-privilege` | IAM | account | **nothing — reference only** | +| `iam.identity-credential-provider-scoped` | IAM | account | **nothing — reference only** | + +The last two are templates a team adopts, not controls the deploy applies. Their +`engine: [cdk]` field describes intent, not behaviour. `infra_utils/agentcore_role.py` +points at the first one in a comment and does not read it. Do not present them as +deployed. + +Deploy the whole workload/account set at once: + +```bash +export ORG_ID=$(aws organizations describe-organization --query Organization.Id --output text) +./scripts/deploy.sh deploy --profile security-focused --module E +``` + +Or one control at a time (note `NON_INTERACTIVE` is read by `deploy.sh`, not by +`cdk`): + +```bash +cdk deploy agentcore-workshop-dev-gateway -c enable_cedar=true +``` + +--- + +## Cedar — say what it does, not what "policy engine" implies + +`enable_cedar=true` attaches a `CfnPolicyEngine` to the gateway and loads one +Cedar policy from the library. **Three things must all be true before "default-deny +authorization on tool calls" is a fair description, and by default none of them +are:** + +1. `enable_cedar` defaults **off** — no engine, no evaluation at all. +2. `cedar_mode` defaults to **`LOG_ONLY`** — decisions are logged, nothing is + denied. +3. The single shipped permit is **unconstrained on principal and resource**: + + ``` + permit(principal, action in [AgentCore::Action::"sample-tool___text_analysis_tool"], resource); + ``` + + Any authenticated caller may invoke the sample tool on any gateway. Narrow the + principal and resource before enforcing. + +What *is* genuinely strong: **Cedar is implicit default-deny.** No permit means +denied, so the library ships permits only — deliberately no blanket `forbid`, +because in Cedar a matching forbid overrides every permit unconditionally and +would make the permit dead code and deny everything. Keep any forbid you add +narrow. + +The permitted action name is the full gateway tool name, +`___` (three underscores). The default comes from +`catalog.yaml` (`read_action`, default `sample-tool___text_analysis_tool`). +**Every tool you add is denied once `cedar_mode=ENFORCE` unless a permit names +it** — that is the correct behaviour and also the most common "we broke the demo" +moment. + +Safe rollout, in this order: + +```bash +# 1. Attach in LOG_ONLY and generate traffic +./scripts/deploy.sh deploy --module 5 -c enable_cedar=true +.venv/bin/python scripts/test_gateway.py +.venv/bin/python scripts/invoke.py "Use the text analysis tool on 'hello world'." + +# 2. Read the decision logs. Confirm what would have been denied. +# 3. Narrow the permit's principal and resource in +# control-library/cedar/gateway-default/10-permit-read-tools.cedar +# 4. Only then: +./scripts/deploy.sh deploy --module 5 -c enable_cedar=true -c cedar_mode=ENFORCE +``` + +Never enable `ENFORCE` before step 2. And remember flags are matched against the +exact lowercase string `"true"` — `-c enable_cedar=True` silently does nothing. + +--- + +## Egress filter — masking, not blocking + +`enable_egress_filter=true` creates a Bedrock Guardrail from +`control-library/guardrails/egress-default.json` and a Lambda +(`$PREFIX-egress-interceptor`, from `tools/egress_interceptor/`) registered as a +gateway interceptor on both `REQUEST` and `RESPONSE`, with +`pass_request_headers=False` — the interceptor never sees caller tokens. + +The shipped guardrail config: + +| Policy | Setting | +|---|---| +| Prompt attack filter | input `HIGH`, output `NONE` | +| `EMAIL`, `PHONE`, `NAME`, `ADDRESS` | `ANONYMIZE` | +| `US_SOCIAL_SECURITY_NUMBER`, `CREDIT_DEBIT_CARD_NUMBER` | `BLOCK` | + +Four caveats to state plainly, because a reviewer will ask: + +- **It masks; it rarely blocks.** The handler raises only when an assessment comes + back `action == "BLOCKED"`. Otherwise it substitutes anonymized text and the + request proceeds. +- **It performs no authorization.** That is Cedar, behind a separate flag. + `enable_egress_filter` alone gives you none of it. +- **Unrecognised payload shapes pass through unchanged.** The + `gatewayRequest`/`gatewayResponse` shape is unvalidated, so the handler scans + string leaves generically and forwards anything it does not recognise. Validate + against live gateway traces before relying on it. +- **No fail-open/fail-closed decision exists.** There is no try/except around + `ApplyGuardrail`, so a Bedrock throttle surfaces as a Lambda failure rather than + a defined behaviour. Decide which you want before production. + +Also: the guardrail resolves to the **`DRAFT`** version (`guardrail.attr_version`). +Pin a published version for production. + +--- + +## Memory resource policy + +`enable_resource_policies=true` attaches an +`AWS::BedrockAgentCore::ResourcePolicy` to the memory resource: allow this +account's root principal, deny everything whose `aws:PrincipalOrgID` is not your +org, with `aws:ViaAWSService: false` so service-mediated access still works. + +It **requires `org_id`** — the stack raises rather than deploying a policy with a +hole in it. `deploy.sh` prompts for `o-xxxx` interactively and hard-stops under +`NON_INTERACTIVE=1`. + +Memory is where conversation history lives, keyed by `actor_id` = the verified +`sub` claim. That makes it the tenant boundary, which is why the federated +multi-account strategy keeps memory **per workload account**: account isolation is +the strongest wall available. + +--- + +## Networking — read this before saying "isolated" + +`enable_networking=true` creates a VPC (`10.0.0.0/16`, 2 AZs, **1 NAT gateway**), +public + `PRIVATE_WITH_EGRESS` subnets, a runtime security group, and AgentCore +VPC endpoints. Runtimes get `network_mode: VPC` with the private subnets and that +security group. + +**This is not an air-gapped VPC.** Private subnets keep a NAT route to the +internet — deliberately, because AgentCore ENIs in a *public* subnet get no +internet route at all and the runtimes would lose Bedrock access. If a customer +needs no-internet-egress, that is a different design (endpoints for every +dependency, no NAT), not a flag. + +What genuinely constrains the agent: + +- The runtime security group has **no inbound rules** and egress limited to + **TCP 443 only** (`allow_all_outbound=False`). Callers reach the agent through + the AgentCore data plane, not through the VPC. +- With `enable_vpc_endpoints` (set true whenever networking is on): interface + endpoints for Bedrock Runtime, ECR API, ECR Docker, CloudWatch Logs, the + AgentCore Gateway endpoint, and a **gateway** endpoint for S3. The ECR + S3 ones + matter for cost as well as isolation — AgentCore pulls and refreshes the image + from ECR whose layers live in S3, so without them the traffic bills as NAT data + processing. The S3 gateway endpoint is free. + +### The VPC endpoint policy restricts less than it looks like + +`org_id` renders `vpce.agentcore-in-org` onto the AgentCore endpoint: allow the +four data-plane actions, then deny anything whose `aws:PrincipalOrgID` is outside +your org — **but only where that key exists.** OAuth/JWT callers carry no IAM +principal, so they arrive under `Principal: "*"` with no `aws:PrincipalOrgID`, and +the deny's `Null` condition (`"aws:PrincipalOrgID": "false"`) makes it +inapplicable to them. **The org restriction covers SigV4 callers only.** JWT +callers are governed by the gateway/runtime authorizers instead. + +The repo is honest about this in the policy itself — the two statement ids read +`AllowDataPlaneIncludingOAuth` and `DenySigV4CallersOutsideOrg`. Read them out loud +if someone in the room is about to call this an org boundary for all traffic. + +**And it lands on one endpoint out of six.** Measured on a real +`ENABLE_NETWORKING=true ORG_ID=o-… ` deploy: only +`com.amazonaws..bedrock-agentcore.gateway` carries the org policy. The +other five — `bedrock-runtime`, `ecr.api`, `ecr.dkr`, `logs`, and the S3 gateway +endpoint — are created with the AWS **default** endpoint policy, which is +`Action: "*"`, `Principal: "*"`, `Resource: "*"`. `bedrock-runtime` is the +model-invocation path, so "we set `ORG_ID`, our endpoints are org-restricted" is +wrong about the endpoint a reviewer cares most about. Check before claiming: + +```bash +VPC=$(aws ssm get-parameter --name "/$PROJECT_NAME/$ENVIRONMENT/networking/vpc-id" \ + --query Parameter.Value --output text) +aws ec2 describe-vpc-endpoints --filters Name=vpc-id,Values=$VPC \ + --query 'VpcEndpoints[].[ServiceName,length(PolicyDocument)]' --output text +``` + +A ~128-character policy is the wide-open default; the org-restricted one is ~500. + +And the failure mode that costs a control silently: **`enable_networking=true` +without `ORG_ID` does not fail.** It warns and creates the endpoint with **no +policy at all**. Treat that warning as an error. Provable for free, before any +deploy — this is the cheapest instance of the pattern taught above: + +```bash +cdk synth $PREFIX-networking -c enable_networking=true | grep -c PolicyDocument +# → 0 +cdk synth $PREFIX-networking -c enable_networking=true -c org_id=o-xxxx | grep -c PolicyDocument +# → 1 +``` + +### The AZ trap — it is not hypothetical, and it is not a quick fix + +AgentCore supports a limited set of AZ **ids** per Region +(`infra_utils/runtime_network.py:19`, `SUPPORTED_ZONE_IDS`), and AZ *name* → *id* +mapping differs per account — `us-east-1a` is not the same physical zone in two +accounts. `networking_stack.py` asks CDK for `max_azs=2`, which takes the first two +alphabetically, so whether you land in a supported zone is **an accident of which +account you are in**. + +Measured on a fresh account in `us-east-1`, where the supported ids are +`use1-az1`, `use1-az2`, `use1-az4`: + +| AZ name | AZ id in that account | Supported | +|---|---|---| +| `us-east-1a` | `use1-az6` | **no** | +| `us-east-1b` | `use1-az1` | yes | + +CDK picked `1a` and `1b`. The networking stack reached `CREATE_COMPLETE` with one +unusable subnet, and nothing at deploy time objected — `unsupported_zone_ids()` is +called **only** by `check_network.py`, never by `app.py` or the runtime stack, which +pass `private_subnet_ids` through unfiltered (`app.py:383-386`). + +**The bill comes due when a runtime tries to enter the VPC**, and AgentCore's own +message is excellent: + +``` +Reason: The following subnets are in unsupported availability zones in region +us-east-1: subnet-0fa4… in us-east-1a (ID: use1-az6). Supported availability +zones are: use1-az4, use1-az1, use1-az2 +``` + +`UPDATE_FAILED` → clean `UPDATE_ROLLBACK_COMPLETE` in ~90s, so the runtime survives +in `PUBLIC` mode. Nothing is wedged; you have simply not got a VPC deployment. + +**Three things make this expensive mid-session, so plan for them:** + +1. **There is no flag.** `max_azs=2` is hardcoded and no context key, env var or + `platform.yaml` entry overrides it. The fix is a **source edit** to + `stacks/networking_stack.py` — replace `max_azs=2` with the AZ *names* that map + to supported *ids in that account*: + + ```bash + aws ec2 describe-availability-zones --region "$AWS_REGION" \ + --query 'AvailabilityZones[].{Name:ZoneName,Id:ZoneId}' --output table + ``` + ```python + availability_zones=["us-east-1b", "us-east-1c"], # ← whichever map to supported ids + ``` + +2. **You cannot apply it in place.** Changing AZs forces subnet replacement, and + CloudFormation creates the replacements before deleting the originals — so they + collide on their own CIDRs and the whole update rolls back: + `The CIDR '10.0.2.0/24' conflicts with another subnet … HandlerErrorCode: AlreadyExists`. + `check_network.py`'s advice to "redeploy the networking stack" is not enough. + **Destroy the stack, then deploy it.** Measured: destroy 4m32s, redeploy 3m32s. + +3. **Then the runtimes still need redeploying** — see below. + +Budget roughly **15 minutes** end to end from "the verify went red" to a full +`PASS`, and say so rather than debugging live. Because it is account-dependent, run +`check_network.py` in the target account *before* the session if you can. + +### Module C does not put your agents in the VPC + +Deploying the networking stack gives you a VPC, subnets, a NAT gateway and six +endpoints that **nothing is using**. Existing runtimes keep `networkMode: PUBLIC` +until they are redeployed with `enable_networking=true`. Measured: with all three +runtimes already deployed and the networking stack `CREATE_COMPLETE`, +`check_network.py --expect-public` **passed** — the agents were still public. + +```bash +ENABLE_NETWORKING=true ORG_ID=o-xxxx ./scripts/deploy.sh deploy --module 6 # orchestrator +ENABLE_NETWORKING=true ORG_ID=o-xxxx ./scripts/deploy.sh deploy --module 8 # A2A sub-agents +``` + +Measured with no source change (so no container rebuild): module 6 in 345s, module +8 in 172s. + +**`check_network.py` reports one problem at a time**, so this arrives as a second +red after you have fixed the first. The real sequence on a fresh account is: AZ +failure → fix and recreate the stack → *placement* failure → redeploy the runtimes → +`PASS`. Knowing there are two stages is the difference between one detour and two. + +### Confirming the posture for real + +```bash +.venv/bin/python scripts/check_network.py # AZ ids, then runtime placement +.venv/bin/python scripts/check_network.py --expect-public # assert a non-VPC deploy is public on purpose +aws ec2 describe-network-interfaces \ + --filters Name=interface-type,Values=agentic_ai \ + --query 'NetworkInterfaces[].[NetworkInterfaceId,AvailabilityZone,SubnetId,Status]' --output text +``` + +`--expect-public` is the underrated one: it lets you prove a non-VPC deployment is +public **deliberately** rather than accidentally. It is a real assertion in both +directions — measured returning exit 1 with +`FAIL: code-agent: expected networkMode PUBLIC, got VPC` once the runtimes moved. + +The ENI check is the evidence a compliance reviewer actually wants, and it shows +something the stack outputs do not: **AgentCore creates fewer ENIs than you have +runtimes.** Three runtimes in the VPC produced **two** `agentic_ai` ENIs, one per +private subnet, shared across runtimes. Do not treat "one ENI per agent" as the +expected shape. + +Then prove the isolation did not break anything, which is the question that +actually gets asked: + +```bash +.venv/bin/python scripts/invoke.py "Reply with exactly: VPC OK" +.venv/bin/python scripts/invoke.py --a2a code-agent "Reply with exactly: A2A VPC OK" +``` + +Both measured working in VPC mode in under 10s — Bedrock over the interface +endpoint, everything else over NAT. + +--- + +## Module E — what KMS + CloudTrail actually gives you + +`$PREFIX-security` creates: + +- a **KMS CMK** with alias `alias/$PREFIX-agentcore` and key rotation enabled. + `app.py` passes its ARN to the memory stack as `encryption_key_arn`, so memory + is CMK-encrypted when module E is deployed — this is the one place the key is + actually consumed. +- a **CloudTrail** trail `$PREFIX-agentcore-trail` into `$PREFIX-cloudtrail-`. + +Three honest caveats: + +- The trail is **`is_multi_region_trail=False`** — activity in other Regions is not + captured. Most orgs already run an org trail; do not present this as replacing it. +- The bucket is **S3-managed encryption**, not the CMK you just created, and has + `auto_delete_objects=True` with `RemovalPolicy.DESTROY`. Destroying the stack + deletes the audit log. That is right for a workshop and wrong for production. +- The CMK also has `RemovalPolicy.DESTROY`. + +**`deploy --module E` on its own fails** with `No stacks match the name(s) +…-security`, because `enable_security` defaults false and module E does not set its +own flag. Use `ENABLE_SECURITY=true ./scripts/deploy.sh deploy --module E`. See +`troubleshooting.md`. + +What the trail actually records, verified on a real deploy — this is the answer to +"so what is being audited?", and it is worth having ready: + +| Setting | Value | +|---|---| +| `IncludeManagementEvents` | `true`, `ReadWriteType: All` | +| `DataResources` | **none** — no S3/Lambda data-plane events | +| `IncludeGlobalServiceEvents` | `true` | +| `IsMultiRegionTrail` | **`false`** — hardcoded, no flag | +| `IsOrganizationTrail` | `false` | +| `LogFileValidationEnabled` | `true` | +| `KmsKeyId` | **`null`** — log files are SSE-S3, not CMK-encrypted | + +So it is a single-Region management-event trail with integrity validation, in a +bucket that deletes itself on teardown. That is a reasonable workshop artifact and +not an audit posture. Check it yourself rather than trusting the stack status: + +```bash +TRAIL=$PREFIX-agentcore-trail +aws cloudtrail get-trail --name "$TRAIL" \ + --query 'Trail.{Multi:IsMultiRegionTrail,KMS:KmsKeyId,Org:IsOrganizationTrail,Validation:LogFileValidationEnabled}' +aws cloudtrail get-event-selectors --trail-name "$TRAIL" +``` + +Module E is the prerequisite for `enable_traceability` (module 9's SNS + +EventBridge alerting on sensitive AgentCore API calls), because that rule only +fires when CloudTrail management events are being recorded. Two follow-ups nobody +remembers: **subscribe an endpoint to the SNS topic** or the alerts go nowhere, +and the topic is **not KMS-encrypted** — add a CMK if alert contents are sensitive. + +--- + +## Org guardrails (Terraform) — the enterprise conversation + +`terraform/org-guardrails/` attaches the org-scope SCPs. Run it from the +**Organizations management account or a delegated SCP administrator**, with SCPs +enabled for the org (`aws organizations enable-policy-type`). + +```hcl +module "agentcore_org_guardrails" { + source = "../../terraform/org-guardrails" + name_prefix = "agentcore" + target_ids = ["ou-abcd-1234wxyz"] +} +``` + +```bash +cd terraform/org-guardrails +terraform init +terraform apply -var 'target_ids=["ou-..."]' +``` + +**Attach to a sandbox OU first.** SCPs are additive-deny and layer on +`FullAWSAccess`, so attaching is non-destructive to existing permissions — but a +gateway hardening SCP that requires a Cedar policy engine in `ENFORCE` will block +gateway creation for every team in that OU, including the ones who never asked for +it. `target_ids` entries are validated at plan time (`r-…`, `ou-…-…`, or a 12-digit +account id) so a typo fails fast. + +### The gateway SCPs deploy as one policy, not eight + +Organizations allows **5 SCPs per target**, and `FullAWSAccess` already holds one — +4 usable. Nine individual policies would not fit any real target, so `gateway.tf` +merges the `Statement` arrays of all eight gateway documents into a single +consolidated SCP, `${name_prefix}-scp-gateway-guardrails`. SCPs are additive-deny, +so merging is semantically identical to attaching separately. + +Consequences: + +- `enable_gateway_scps` is **all-or-nothing**; subsetting means editing the map in + `gateway.tf`. +- Plan-time preconditions enforce the **5,120-character** SCP size quota and Sid + uniqueness across library files. A ninth control can fail at plan time. +- With everything on the module attaches **3** SCPs per target (consolidated + gateway + memory + identity), leaving one usable slot. A fourth standalone SCP is + the last one that fits. + +Both `enable_scp_memory_enforce_cmk` and `enable_gateway_scps` default to `true`, +so a bare `terraform apply` attaches them unless you opt out. + +### What the gateway SCPs constrain + +| SCP | Condition key | Effect | +|---|---|---| +| `require-cmk` | `KmsKeyArn` | gateway must use an approved CMK | +| `deny-no-auth` | `GatewayAuthorizerType` | blocks `NONE` — no unauthenticated gateways | +| `require-policy-engine` | `PolicyEngineArn` / `PolicyEngineMode` | requires a Cedar engine in `ENFORCE` | +| `enforce-approved-idp` | `DiscoveryUrl` | JWT gateways must use an approved IdP | +| `restrict-protocol` | `ProtocolType` | restricts explicitly-set protocols to `MCP` | +| `targets-require-private-endpoint` | `PrivateEndpointType` | targets must use a private endpoint | +| `targets-restrict-credential-provider` | `CredentialProviderType` | denies `API_KEY` / `JWT_PASSTHROUGH` | +| `targets-restrict-type` | `McpTargetConfigurationType` | allow-lists target types (lambda, mcpServer) | + +**These are control-plane controls: they constrain how a gateway may be +*configured*, not who may *invoke* one.** That distinction is the whole point of +the section. `restrict-protocol` has a matching gap worth disclosing — an +*omitted* protocol falls back to the service default and is unconstrained; only +explicitly-set values are checked. + +For invoke-time restrictions, the mechanism this repo uses is a per-resource +policy attached with `AWS::BedrockAgentCore::ResourcePolicy`, as done for Memory. + +The **"fully-private gateway" bundle** worth naming as a package: +`deny-no-auth` + `require-policy-engine` + `targets-require-private-endpoint`. + +### The identity SCP is the sharpest control here + +`scp.identity.deny-workload-token-for-userid` denies +`bedrock-agentcore:GetWorkloadAccessTokenForUserId`. **That API takes the user +identifier as an unverified string** — any principal holding the action can mint a +workload access token for any user and read that user's stored credentials out of +the token vault, with no JWT and no proof of identity anywhere in the call. + +Agents behind Runtime or Gateway inbound auth never need it: the caller's verified +token arrives with the request, and `GetWorkloadAccessTokenForJWT` is the path that +checks it. + +The exemption parameter defaults to a role ARN **that cannot exist**, so the +control denies everyone until an operator supplies a real pattern. Narrow it only +for a genuine break-glass or migration path: + +```bash +terraform apply -var 'target_ids=["ou-..."]' \ + -var 'identity_approved_principal_arn_pattern=arn:aws:iam::111122223333:role/break-glass' +``` + +The action also supports the `bedrock-agentcore:userid` condition key if you need +something narrower than an ARN exemption. Prefer removing the need over widening +the pattern. + +--- + +## Runtime IAM — what the agent's role can actually do + +This is what a compromised agent inherits, so read it rather than assuming. +`stacks/runtime_stack.py` builds the role with scoped statements: ECR pull limited +to **this component's own repository**, Logs limited to +`/aws/bedrock-agentcore/runtimes/*`, SSM limited to +`arn:aws:ssm:*:*:parameter/{project}/*`, Bedrock limited to `foundation-model/*` +plus `inference-profile/*`, and the token-vault secret read scoped to +`bedrock-agentcore-identity!default/oauth2/*`. + +Wildcards remain in three places, two of them unavoidable: + +| Statement | Resource | Why | +|---|---|---| +| `ecr:GetAuthorizationToken` | `*` | returns an account-level token; IAM accepts nothing else | +| `xray:PutTraceSegments` / `PutTelemetryRecords`, `cloudwatch:PutMetricData` | `*` | these actions take no resource ARN (`PutMetricData` is constrained by a `cloudwatch:namespace` condition instead) | +| `bedrock-agentcore:*` — gateway invoke, memory data plane, code interpreter, browser, workload tokens | `*` | **knowingly open.** Scoping needs gateway, memory and sibling-runtime ARNs, and the A2A targets do not exist when the role is built. Say this out loud in a review rather than letting someone find it. | + +Two related facts that explain real failures: + +- The memory data-plane actions (`ListEvents`, `CreateEvent`, `DeleteEvent`, + `ListSessions`, `RetrieveMemoryRecords`) are there because framework + integrations need them — LangGraph's checkpointer lists events to rehydrate a + thread. Remove them and memory-backed patterns fail at invoke while the stack + looks fine. +- `secretsmanager:GetSecretValue` on the vault path is what makes gateway tools + load. Without it the token fetch raises AccessDenied inside the MCP client and + the agent aborts. See `agent-patterns.md`. + +**The credential-provider fence is IAM and nothing else.** AgentCore does not +enforce any binding between a workload identity and the credential providers it may +read, so a shared execution role hands every agent every provider's credentials. +One workload identity and one role per trust boundary, each naming exactly one +provider — that is what `iam.identity-credential-provider-scoped` is a template for. + +One trap when adopting it: `GetResourceOauth2Token` and the +`GetWorkloadAccessToken*` actions declare several **required** resource types — the +directory and the token vault as well as the workload identity and the provider — +so a statement naming only the provider ARN authorises nothing. The resulting +`AccessDenied` is easy to "fix" by widening `Resource` to `"*"`, which defeats the +control entirely. `Deny` statements need only the ARN they target. + +--- + +## The Cognito M2M client secret is a plaintext CloudFormation export + +Find this before a reviewer does, because it is the one finding in this platform +that a reviewer can confirm in a single read-only API call. + +The `$PREFIX-auth` stack publishes the Cognito **M2M app client secret in +plaintext** as an auto-generated cross-stack output *and* a named CloudFormation +export: + +```bash +aws cloudformation describe-stacks --stack-name "$PREFIX-auth" \ + --query 'Stacks[0].Outputs[?contains(OutputKey,`ClientSecret`)].OutputKey' +# ExportsOutputFnGetAttUserPoolM2MClientDescribeCognitoUserPoolClient…ClientSecret… + +aws cloudformation list-exports \ + --query 'Exports[?contains(Name,`ClientSecret`)].Name' +# the same value, retrievable WITHOUT knowing the stack name +``` + +**This is not a misconfiguration you can flag off.** It falls out of the wiring: +`stacks/auth_stack.py` exposes the secret as a property, `app.py` passes it to a +*different* stack, and `stacks/identity_stack.py` calls `.unsafe_unwrap()` on it. +A cross-stack reference becomes a CloudFormation export, and an export carries the +**resolved value**, not the token. `list-imports` on that export names +`$PREFIX-identity`, which is the whole chain in one call. + +The property's own docstring says the value "renders as a CloudFormation token … +never literal text." That is **true of the synthesized template and false of the +deployed stack.** Do not quote the docstring as reassurance; check the outputs. + +Why it is worse than it sounds: reading it needs `cloudformation:DescribeStacks` +or `cloudformation:ListExports` — permissions handed to anyone who looks at +infrastructure, and `ListExports` accepts no resource condition, so it cannot be +scoped to one stack. The equivalent Secrets Manager read is a deliberate grant +that CloudTrail records as a secret access. + +The blast radius is bounded but real: that secret plus the `m2m-client-id` in SSM +is a complete client-credentials grant for the `agentcore/invoke` scope — enough +to call the gateway directly as a machine principal, bypassing whatever the agent +would have done. It is **not** a path to the user pool's human identities. + +What actually mitigates it, in order of how quickly it can be done: + +1. **Treat the `-auth` stack as secret-bearing** in whatever governs who may + describe stacks. This costs nothing and is the honest short answer. +2. **Rotate after any session where the outputs were on screen** — delete and + recreate the M2M client, or redeploy `-auth` into a fresh prefix. +3. **Break the cross-stack reference** if you are forking: have `-identity` read + the secret from Secrets Manager at deploy time (`{{resolve:secretsmanager:…}}`, + the pattern the accelerator already uses for the IdP secret) instead of + receiving it through `unsafe_unwrap()`. Removing the reference removes the + export. + +There is a third path, and it is the one that catches people: **the deploy prints +it.** Two separate mechanisms, and the second is much wider than the first: + +| Where | Scope | Why | +|---|---|---| +| CDK's `Outputs:` block | runs whose stack graph reaches `-auth` | includes runs that never name it — `deploy --module A` announces `Stacks: …-memory`, then prints `-auth`'s outputs because CDK pulls it in as a dependency | +| `deploy.sh`'s closing summary table | **every `deploy` run, once `-auth` exists** | `print_summary` (`scripts/deploy.sh:659`) does its own `list-stacks` for `starts_with(StackName, '$PREFIX')` and dumps all outputs of every match — it never looks at what you deployed | + +So the correct rule is the blunt one: **once module 3 has ever run in this account, +every subsequent `deploy.sh deploy` prints the secret.** Verified with +`deploy --module C`, whose graph does not touch `-auth` at all: CDK's own `Outputs:` +block listed only `-networking`, and the closing summary printed the `-auth` secret +anyway. + +`./scripts/deploy.sh export` reads the same outputs and **would** write the secret +to `workshop-outputs-.json` in the repo root, which is **not** covered by +`.gitignore`. Today it does not, and the reason is narrow enough to be worth stating +precisely: the export's merge only keeps the **last field of each output object** +(see `troubleshooting.md`), and because the secret output is a CloudFormation +*export*, its last field is `ExportName` rather than `OutputValue`. So the file gets +`…-auth:ExportsOutputFnGetAtt…UserPoolClientClientSecret…` — the export's name, not +its value. Verified on a full platform-team export: the longest lowercase-alphanumeric +run in the file is 36 characters, and a Cognito client secret is ~52. + +That is a bug shielding a leak, not a safeguard, and it is one refactor away from +inverting. If you fix the export locally, gitignore the artifact in the same change. +The file still carries the account id and every runtime, memory and VPC ARN, so it is +not shareable either way. + +Checked and **not** a leak, so do not claim it: the auth stack's custom-resource +provider Lambda does not log the response. `filter-log-events` on its log group +for `ClientSecret` / `secret` returns nothing. The exposure is the two +CloudFormation APIs above plus the deploy's own stdout — see `facilitation.md` +about screens and recordings. + +--- + +## Proving a control is real without deploying + +`cdk synth` is the cheap answer to "prove this control exists." Flag off → the +resource count is 0; flag on → the resource is present. `docs/TESTING.md` in the +repo has ready-made synth assertions per control, which is a much better response +in a security conversation than a screenshot of a green stack. + +```bash +cdk synth agentcore-workshop-dev-gateway -c enable_cedar=true | grep -c PolicyEngine +cdk synth agentcore-workshop-dev-gateway | grep -c PolicyEngine +``` + +--- + +## Never do these + +- **`cedar_mode=ENFORCE` before reading LOG_ONLY decision logs.** The shipped + permit covers one tool. Everything else stops. +- **Attach the org SCPs outside a sandbox OU on first run.** They constrain every + account under the target, including teams who did not ask. +- **Call `enable_networking=true` air-gapped.** There is a NAT route, by design. +- **Deploy `enable_networking=true` without `ORG_ID`** and assume the endpoint is + org-restricted. It has no policy at all, and only warns. +- **Say `ORG_ID` org-restricts your endpoints.** It restricts one of six. The other + five, including `bedrock-runtime`, keep the wide-open AWS default policy. +- **Say the agents are in the VPC because module C succeeded.** Runtimes stay + `PUBLIC` until redeployed with `enable_networking=true`. Prove it with + `check_network.py` plus the `agentic_ai` ENI list, and note that + `--expect-public` will happily pass in exactly this half-done state. +- **Present `iam.*` library files as deployed controls.** They are reference + policies nothing applies. +- **Put a secret in `platform.yaml`, `workshop.env`, or CDK context.** Secrets + Manager *names* only. Context ends up in `cdk.context.json` and CloudFormation + parameters. +- **Say the Cognito M2M secret only lives in Secrets Manager.** The `-auth` stack + exports it in plaintext, readable with `cloudformation:ListExports`. Section + above. +- **Rely on the egress guardrail's `DRAFT` version in production.** Pin a + published version. diff --git a/kiro/agentcore-enterprise-platform/steering/troubleshooting.md b/kiro/agentcore-enterprise-platform/steering/troubleshooting.md new file mode 100644 index 0000000..327a525 --- /dev/null +++ b/kiro/agentcore-enterprise-platform/steering/troubleshooting.md @@ -0,0 +1,1155 @@ +# Troubleshooting + +Read this when something failed. Organised by **what the person sees**, not by +what the code does. + +**Two rules that save the most time:** + +1. **Read the container logs before changing anything.** Every runtime failure so + far named its own cause there. + + ```bash + ARN=$(aws ssm get-parameter --name /$PROJECT_NAME/$ENVIRONMENT/runtimes/orchestrator/arn \ + --query Parameter.Value --output text) + aws logs filter-log-events \ + --log-group-name "/aws/bedrock-agentcore/runtimes/${ARN##*/}-DEFAULT" \ + --start-time $(( ($(date +%s) - 900) * 1000 )) \ + --query 'events[].message' --output text | grep -iE 'error|denied|traceback' + ``` + +2. **`CREATE_COMPLETE` proves nothing about behaviour.** Run the verify script. + +--- + +## Index + +| What you see | Section | +|---|---| +| `This script requires bash 4 or newer` | [macOS bash 3.2](#this-script-requires-bash-4-or-newer-or-declare--a-invalid-option) | +| Script freezes after the docker check | [npx hang](#the-script-freezes-after-the-docker-check) | +| `python3.13: NOT FOUND` | [python3.13](#python313-not-found) | +| `CERTIFICATE_VERIFY_FAILED` from a verify script | [macOS Python certs](#certificate_verify_failed-from-a-verify-script-while-aws-commands-work-fine) | +| `AWS credentials invalid or expired` | [credentials — or a disabled Region](#aws-credentials-invalid-or-expired) | +| Commands hit the wrong account | [env vars beat profile](#commands-are-hitting-the-wrong-account) | +| `CDK bootstrap failed` / schema version mismatch | [CDK CLI too old](#cdk-bootstrap-failed--and-the-advice-printed-under-it-is-probably-wrong) | +| `No stacks match the name(s) …-security` / `…-networking` | [module needs its flag](#no-stacks-match-the-names-prefix-networking-or--security) | +| Deploy failed, CDK output unhelpful | [CloudFormation events](#a-stack-failed-and-the-cdk-output-is-not-enough) | +| Model not available / access denied | [Bedrock models](#bedrock-says-the-model-is-not-available-or-access-is-denied) | +| Deploy stops asking for an org id | [ORG_ID](#the-deploy-stops-asking-for-an-organizations-id) | +| `subnets are in unsupported availability zones` | [AZ ids, and the fix is a source edit](#the-following-subnets-are-in-unsupported-availability-zones-in-region-) | +| `CIDR … conflicts with another subnet` after an AZ change | [destroy before redeploying](#the-cidr-1002024-conflicts-with-another-subnet-after-changing-azs) | +| `networkMode is PUBLIC, not VPC` | [module C does not move runtimes](#check_networkpy-networkmode-is-public-not-vpc--the-vpc-exists-but-the-agent-is-not-in-it) | +| `workshop-outputs-*.json` stack outputs are garbage | [export merge is broken](#workshop-outputs-stampjson-has-no-usable-stack-outputs) | +| Changed agent code, redeploy did nothing | [image tags](#i-changed-agent-code-and-the-redeploy-changed-nothing) | +| `invoke.py` prints a wall of `data: {…}` JSON | [streaming pattern, not a crash](#invokepy-printed-hundreds-of-data--lines-instead-of-an-answer) | +| Memory-using pattern forgets across sessions | [long-term memory is off by default](#a-memory-using-pattern-recalls-within-a-session-but-not-across-sessions) | +| `platform.yaml is invalid` | [config validation](#platformyaml-is-invalid) | +| Stale answers keep coming back | [workshop.env](#old-answers-keep-coming-back) | +| Module 6 has been silent for minutes | [not a hang](#module-6-has-been-silent-for-eight-minutes) | +| I pressed Ctrl-C mid-deploy | [AWS keeps going](#i-pressed-ctrl-c-during-a-deploy) | +| `--dry-run` deployed for real | [wrong action, or misspelled flag](#i-passed---dry-run-and-it-deployed-anyway) | +| I deployed way more than I expected | [`--profile` scope](#i-deployed-far-more-than-i-expected) | +| Invoke returns `Unauthorized` | [JWT](#invoke-returns-unauthorized) | +| Invoke returns HTTP 424 | [container serves the wrong protocol](#invoke-returns-http-424) | +| A2A invoke returns 200, no answer | [wrong JSON-RPC envelope](#an-a2a-invoke-returns-http-200-and-the-agent-never-answered) | +| `Invalid length for parameter runtimeSessionId` | [33-char minimum](#invalid-length-for-parameter-runtimesessionid) | +| `Authorization method mismatch` | [SigV4 vs JWT](#authorization-method-mismatch-on-invoke) | +| Agent reports no tools | [gateway tools](#the-agent-reports-no-tools) | +| Memory-backed pattern fails at invoke | [runtime role](#a-memory-backed-pattern-fails-at-invoke) | +| Module 9 verify: `status is PENDING` | [Transaction Search is async](#module-9s-verify-fails-with-trace-segment-destination-is-cloudwatchlogs-but-status-is-pending) | +| No traces anywhere | [Transaction Search](#no-traces-appear-anywhere) | +| `batch-get-traces` returns nothing | [1% sampling](#batch-get-traces-returns-nothing-for-my-trace) | +| Destroy fails on an export | [teardown order](#destroy---stack-fails-with-export--cannot-be-deleted) | +| `destroy` exited 1, stacks still up | [it aborts at the first failure, and networking is last](#destroy-exited-non-zero-and-half-the-platform-is-still-standing) | +| `DELETE_FAILED` / `NotStabilized` on a runtime | [a timeout, not a failure — retry](#delete_failed-request-timed-out-while-deleting-awsbedrockagentcoreruntime) | +| Networking destroy fails on subnets | [ENI drain](#destroying-the-networking-stack-fails-on-subnets-or-security-groups) | +| Deleted observability, Transaction Search still on | [account-scoped](#deleting-the-observability-stack-did-not-disable-transaction-search) | + +--- + +## Local setup + +### `This script requires bash 4 or newer` (or `declare: -A: invalid option`) + +macOS ships bash 3.2 (2007) as `/bin/bash`; the script needs associative arrays. +The guard is the **first thing in the file** (`scripts/deploy.sh:5`, on +`BASH_VERSINFO`), so it fires before any subcommand runs — `ls`, `config` and +`workshop` all stop identically, and it exits 1. Its own advice is correct and +complete: + +``` +ERROR: This script requires bash 4 or newer (you are running bash 3.2.57(1)-release). + +macOS ships bash 3.2 as /bin/bash. To fix: + 1. brew install bash + 2. Run the script with the new bash explicitly: + bash scripts/deploy.sh deploy +``` + +```bash +brew install bash +bash scripts/deploy.sh deploy # explicit new bash +# or: hash -r so PATH picks up /opt/homebrew/bin/bash +``` + +You will only see the raw `declare: -A: invalid option` if the guard was removed or +you are sourcing pieces of the script by hand. **Do not go looking for that string** — +on a stock checkout the clean message above is what appears. + +### The script freezes after the docker check + +An old checkout. Older copies probed the CDK CLI with a bare `npx cdk --version`; +when `aws-cdk` is not in the npx cache, npx asks "Ok to proceed?" and — with +output suppressed and no TTY — waits forever. The fix (`npx --no-install cdk +--version`) is already in `check_prereqs`. Meanwhile: + +```bash +npm install -g aws-cdk +``` + +### `python3.13 NOT FOUND` + +The scripts require exactly `python3.13` on PATH, not `python3`. Install it, then +rebuild the venv: + +```bash +python3.13 -m venv .venv && source .venv/bin/activate +pip install -r requirements.txt +``` + +### `CERTIFICATE_VERIFY_FAILED` from a verify script, while `aws` commands work fine + +``` +ssl.SSLCertVerificationError: [SSL: CERTIFICATE_VERIFY_FAILED] certificate verify +failed: unable to get local issuer certificate (_ssl.c:1028) +``` + +Seen from `test_gateway.py`, `invoke.py` or `deploy.sh verify` — typically at +`utils.py` → `get_m2m_token`, the step that fetches a Cognito token. **The stack is +fine. The gateway is fine.** Nothing is wrong in AWS. + +The tell is the asymmetry: every `aws` CLI call and every boto3 call in the same run +succeeds, and only the accelerator's own Python scripts fail. That is because +botocore ships its own CA bundle (`certifi`), while these scripts call +`urllib.request.urlopen` with no `context=` argument and so fall back to OpenSSL's +default path — and on a **python.org macOS installer** build of `python3.13` that +path does not exist: + +```bash +python3.13 -c "import ssl; print(ssl.get_default_verify_paths().cafile)" # None +``` + +Two fixes. The official one, run once per machine: + +```bash +"/Applications/Python 3.13/Install Certificates.command" +``` + +Or, where a participant cannot or would rather not run that, a per-shell override +that needs no privileges: + +```bash +export SSL_CERT_FILE=$(.venv/bin/python -m certifi) +``` + +Either makes the whole verify layer work — one env var turned a failing +`test_gateway.py` into `tools/list` returning both targets and a successful +`tools/call`. + +Affects `python3.13` from the python.org installer specifically. Homebrew and +`uv`-managed interpreters link a real trust store and never see this. + +**Why it can end a run:** module 5's stack reaches `CREATE_COMPLETE`, then its +verify fails. Under `NON_INTERACTIVE=1` the script aborts the whole guided run +(`Aborting (NON_INTERACTIVE=1 cannot prompt)`); at a terminal it offers to +continue. Fix the certs and resume with `--from`, do not skip the verify. + +### Do I need Docker? + +**No.** Images build in AWS CodeBuild. `check_prereqs` reports docker as optional +and continues. You only need a container runtime to build and run an agent image +locally while developing. + +--- + +## Credentials and permissions + +### `AWS credentials invalid or expired` + +Refresh and retry. Long deploys can outlive a session, so this can also appear +*mid-run*: + +```bash +aws sso login # or your credential process +aws sts get-caller-identity +``` + +If a deploy died mid-stack, re-run the same command — CDK picks up from current +stack state, and `UPDATE_ROLLBACK_COMPLETE` is safe to deploy onto again. + +**But check the Region before you touch your credentials** — the message is +misattributed. `check_credentials` (`scripts/deploy.sh:291-297`) tests +`aws sts get-caller-identity &>/dev/null` and prints this on *any* non-zero exit, +having discarded the actual error. Measured against an opt-in Region that is not +enabled for the account: + +``` +[ERROR] AWS credentials invalid or expired. +[ERROR] Run 'aws sso login' or configure credentials, then retry. +``` + +The credentials were valid. The real error, visible only when you run the call +yourself: + +```bash +aws sts get-caller-identity --region eu-central-2 +# An error occurred (InvalidClientTokenId) … The security token included in the +# request is invalid. ← STS in a disabled opt-in Region +aws sts get-caller-identity --region us-east-11 +# Could not connect to the endpoint URL: "https://sts.us-east-11.amazonaws.com/" +# ← a typo'd Region name +``` + +So `aws sso login` can never fix it. Run `get-caller-identity` yourself against +`$AWS_REGION`, and enable the Region (Account settings → Regions) or fix the +spelling. Region-enablement takes minutes and is account-wide. + +### Commands are hitting the wrong account + +`AWS_ACCESS_KEY_ID` / `AWS_SESSION_TOKEN` in the environment beat `AWS_PROFILE`. +That is almost always the cause. + +```bash +unset AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY AWS_SESSION_TOKEN +aws sts get-caller-identity --query Account --output text +``` + +Also note: an **empty** `AWS_PROFILE=""` is worse than unset — the CLI reports +`The config profile () could not be found`. Unset it, do not blank it. + +### CDK bootstrap failed — and the advice printed under it is probably wrong + +The script prints this on any bootstrap failure: + +``` +[ERROR] CDK bootstrap failed for aws:///. Output: +... +[ERROR] Common causes: wrong AWS account/profile, or missing IAM permissions +[ERROR] to create the CDK bootstrap stack (CDKToolkit). Fix and retry. +``` + +**Read the CDK output above those two lines before believing them.** On a fresh +clone the most common cause is neither of the ones named — it is a CDK CLI that is +too old for the `aws-cdk-lib` pip just installed: + +``` +This CDK CLI is not compatible with the CDK library used by your application. +(Cloud assembly schema version mismatch: Maximum schema version supported is +53.x.x, but found 54.0.0. You need at least CLI version 2.1138.0 to read this +manifest.) +``` + +Fix the **CLI**, not the library: + +```bash +npm install -g aws-cdk@latest +cdk --version # must be >= the version the error named +``` + +Why it happens, and why it will keep happening: `requirements.txt` pins +`aws-cdk-lib>=2.265.0` with no upper bound, so `pip install -r requirements.txt` +always pulls the newest library, which emits the newest cloud-assembly schema. The +repo has no `package.json` and no `node_modules`, so `npx --no-install cdk` +resolves whatever CDK CLI is installed **globally** — often one installed months +ago and never touched. The library moves on every clone; the CLI does not. + +`check_prereqs` will not catch it. It prints a green `✓ cdk: 2.1119.0` because it +tests that the CLI *exists*, never that it is compatible with the installed +library. So the run looks healthy right up to the bootstrap step, then dies in +about ten seconds. + +Do **not** "fix" this by pinning `aws-cdk-lib` down to match an old CLI. The +version the error message names is the floor; install it or newer. + +The two causes the script *does* name are real, just rarer here. Once the CLI is +current and it still fails, then confirm the account and the IAM permission to +create `CDKToolkit`: + +```bash +aws sts get-caller-identity --query '[Account,Arn]' --output text +aws cloudformation describe-stacks --stack-name CDKToolkit \ + --query 'Stacks[0].[StackStatus,LastUpdatedTime]' --output text +``` + +--- + +## Deploy failures + +### `No stacks match the name(s) -networking` (or `-security`) + +``` +No stacks match the name(s) agentcore-workshop-dev-security +[ERROR] Failed to deploy agentcore-workshop-dev-security +[ERROR] Check CloudFormation console for details. +``` + +**Ignore the advice — there is nothing in the CloudFormation console to look at.** +The stack was never synthesized, so CloudFormation has never heard of it. The script +exits 1, correctly, but points you at the wrong place. + +Cause: `deploy --module C` and `deploy --module E` name a stack that only exists in +the CDK app when a feature flag is on, and **neither module turns its own flag on.** +`app.py` defaults `enable_networking=false` and `enable_security=false`, so a bare +standalone deploy of either asks CDK for a stack that is not in the app. Export the +flag: + +```bash +ENABLE_NETWORKING=true ./scripts/deploy.sh deploy --module C +ENABLE_SECURITY=true ./scripts/deploy.sh deploy --module E +``` + +**Module 8 is the exception, and it is the reason this surprises people.** +`enable_a2a` defaults **`true`** at app level (`app.py:83`), so `deploy --module 8` +just works — and the guided loop additionally re-exports `ENABLE_A2A=true` at +`scripts/deploy.sh:901` so that a profile which set it false (`greenfield`, +`migration`, `security-focused`) does not break module 8 mid-walk. There is no +equivalent line for C or E. So "module 8 needed no flag" is true and does not +generalise. + +Inside `workshop --profile …` none of this bites: `platform-team` and +`security-focused` already set the flags they need. The trap is only the standalone +`deploy --module` path — which is exactly what people use to redo one module. + +Confirm what the app currently defines before blaming the deploy: + +```bash +./scripts/deploy.sh ls # stacks with current flags +ENABLE_SECURITY=true ./scripts/deploy.sh ls | grep security +``` + +### A stack failed and the CDK output is not enough + +Ask CloudFormation directly — the resource-level reason is what you want: + +```bash +aws cloudformation describe-stack-events --stack-name \ + --query "StackEvents[?contains(ResourceStatus,'FAILED')].{r:LogicalResourceId,reason:ResourceStatusReason}" \ + --output json | head -30 +``` + +### Bedrock says the model is not available, or access is denied + +Two independent causes, and they need different fixes: + +- **Model access not enabled** in this account/Region → enable it in the Bedrock + console (Model access), then retry. +- **The model id has aged out.** Dated model ids get marked Legacy and are + rejected in fresh accounts. Every pattern takes an override rather than + hardcoding: + + ```bash + export MODEL_ID=us.anthropic.claude-sonnet-5 # or your current inference profile + ./scripts/deploy.sh deploy --stack $PREFIX-runtime-orchestrator + ``` + +Prefer a cross-region inference profile id over a dated model id. + +**Match the exception to the cause — three distinct signatures, all measured +directly against `bedrock-runtime converse`, which is the fastest way to test a +model id without redeploying anything:** + +```bash +aws bedrock-runtime converse --model-id "$MODEL_ID" \ + --messages '[{"role":"user","content":[{"text":"say OK"}]}]' \ + --inference-config '{"maxTokens":10}' +``` + +| Exception | Message | What it means | +|---|---|---| +| `ResourceNotFoundException` | `Access denied. This Model is marked by provider as Legacy and you have not been actively using the model in the last 30 days.` | the id aged out — **this is what a hardcoded model default eventually becomes**, and the wording says "Access denied" even though access is not the problem | +| `ValidationException` | `Invocation of model ID … with on-demand throughput isn't supported. Retry your request with the ID or ARN of an inference profile` | you dropped the `us.` prefix on a model that is inference-profile-only | +| `ValidationException` | `The provided model identifier is invalid.` | the id does not exist at all — a typo, or a retired dated id | + +Note the first one is a `ResourceNotFoundException`, not `AccessDeniedException`. +Searching for "AccessDenied" in the container logs will miss it. + +**One place in the repo carries an id that is already invalid:** +`workshop-simulation/existing-ec2-agent/agent.py:13` pins +`anthropic.claude-sonnet-4-20250514`, which returns +`The provided model identifier is invalid.` today. The `agent-code/*` defaults +(`us.anthropic.claude-sonnet-4-6`, `us.anthropic.claude-opus-4-6-v1`) were still +resolving when this was measured — but set `MODEL_ID` explicitly rather than +finding out live. + +### The deploy stops asking for an Organizations id + +`enable_resource_policies=true` renders `aws:PrincipalOrgID` into the Memory +resource policy and cannot be built without it: + +```bash +export ORG_ID=$(aws organizations describe-organization --query Organization.Id --output text) +``` + +Interactively the script prompts for an `o-xxxx` value and only fails if you +leave it empty; with `NON_INTERACTIVE=1` it is a hard stop with the fix printed. +If the account is not in an Organization, leave `enable_resource_policies` off — +which means not using the `security-focused` profile as shipped. + +**This gate also fires in `--dry-run`, before the plan is printed.** So +`workshop --dry-run --profile security-focused` produces no plan at all without +`ORG_ID`: with a terminal it stops and waits at the prompt, and with stdin closed +it warns and exits 1. To *preview* the plan, any `o-…`-shaped value works, because +dry-run makes no AWS calls: + +```bash +ORG_ID=o-preview0 ./scripts/deploy.sh workshop --dry-run --profile security-focused +``` + +Different failure mode, same variable: `enable_networking=true` **without** +`ORG_ID` does not fail. It warns and creates the AgentCore VPC endpoint with **no +policy at all**. Treat that warning as an error anywhere it matters. + +### `The following subnets are in unsupported availability zones in region …` + +The runtime redeploy fails, not the networking deploy: + +``` +Reason: The following subnets are in unsupported availability zones in region +us-east-1: subnet-0fa4… in us-east-1a (ID: use1-az6). Supported availability +zones are: use1-az4, use1-az1, use1-az2 +HandlerErrorCode: NotStabilized +``` + +**The message is completely accurate — trust it.** It rolls back cleanly +(`UPDATE_ROLLBACK_COMPLETE`, ~90s) and the runtime keeps working in `PUBLIC` mode, +so nothing is wedged. You just do not have a VPC deployment yet. + +Why the networking stack was green: AgentCore supports a limited set of AZ **ids** +per Region, `networking_stack.py` asks CDK for `max_azs=2` which takes the first two +AZ *names* alphabetically, and name → id mapping differs per account. Nothing +validates this at deploy time — `unsupported_zone_ids()` is called only by +`check_network.py`. Catch it before the runtime does: + +```bash +.venv/bin/python scripts/check_network.py +aws ec2 describe-availability-zones --region "$AWS_REGION" \ + --query 'AvailabilityZones[].{Name:ZoneName,Id:ZoneId}' --output table +``` + +**The fix is a source edit plus a destroy — not a flag and not a redeploy.** There +is no context key, env var or `platform.yaml` entry for AZs. In +`stacks/networking_stack.py`, replace `max_azs=2` with the names that map to +supported ids *in this account*: + +```python +availability_zones=["us-east-1b", "us-east-1c"], +``` + +Then read the next entry before you run `deploy`. + +### `The CIDR '10.0.2.0/24' conflicts with another subnet` after changing AZs + +``` +Resource handler returned message: "The CIDR '10.0.2.0/24' conflicts with another +subnet …" HandlerErrorCode: AlreadyExists +``` + +Changing AZs forces subnet **replacement**, and CloudFormation creates the +replacements before deleting the originals — so they collide with their own CIDRs +and all four subnets fail. The stack rolls back to its previous, still-broken AZs. + +`check_network.py`'s advice to "redeploy the networking stack" is not sufficient. +**Destroy it first:** + +```bash +./scripts/deploy.sh destroy --stack "$PREFIX-networking" # ~4m30s +ENABLE_NETWORKING=true ORG_ID=o-xxxx ./scripts/deploy.sh deploy --module C # ~3m30s +``` + +Safe to destroy if no runtime ever entered the VPC — no `agentic_ai` ENIs exist to +block it. If runtimes *were* in the VPC, see "Destroying the networking stack fails +on subnets or security groups" below. + +### `check_network.py`: `networkMode is PUBLIC, not VPC — the VPC exists but the agent is not in it` + +Working as intended, and the most common surprise in module C. **Deploying the +networking stack does not move existing runtimes into it.** You have a VPC, a NAT +gateway and six endpoints that nothing is using. + +```bash +ENABLE_NETWORKING=true ORG_ID=o-xxxx ./scripts/deploy.sh deploy --module 6 # ~345s +ENABLE_NETWORKING=true ORG_ID=o-xxxx ./scripts/deploy.sh deploy --module 8 # ~172s +``` + +Note the corollary: **`check_network.py --expect-public` will pass** in this state, +because the agents genuinely are public. That is not confirmation you are safe. + +`check_network.py` also stops at its first failure, so on a fresh account expect two +rounds — AZ ids first, then placement. + +### `workshop-outputs-.json` has no usable stack outputs + +`stack_outputs` is a list of single characters and every `OutputKey` is missing. +`./scripts/deploy.sh export` merges each stack's JSON by splitting on whitespace and +re-parsing the fragments, so the documents get shredded; the failures are swallowed +by `except: pass`. Measured: 769 one-character entries, zero recoverable +key/value pairs. + +The mechanism is worth knowing, because it decides what does and does not end up in +the file. Splitting pretty-printed JSON on whitespace yields tokens like +`"OutputKey":` and `"someValue",` — both fail `json.loads` because of the trailing +colon or comma. The only tokens that parse are bare quoted strings, i.e. the **last +field of each output object**, and `result.extend()` on a parsed string appends its +characters one at a time. So each output contributes at most its final field's +characters, in order, concatenated with its neighbours' — which is why rejoining the +list gives a readable but unusable run of ARNs, URLs, ids and CloudFormation export +names with no separators. + +**The `ssm_parameters` half of the file is fine** — one API call, no merge, and it +carries no secrets. For stack outputs, query CloudFormation directly: + +```bash +aws cloudformation describe-stacks --stack-name "$PREFIX-gateway" \ + --query 'Stacks[0].Outputs' --output json +``` + +Also note the file lands in the repo root and is not gitignored. + +### `invoke.py` printed hundreds of `data: {…}` lines instead of an answer + +Not a crash, and not a deployment problem. **`invoke.py` prints whatever the pattern +emits**, and `langgraph-agent` (and the other streaming patterns) emit raw SSE — one +`data:` chunk per few tokens. Measured: 176 lines for a one-sentence reply, exit code +0. The `orchestrator` pattern prints a single `{"status": "success", "response": …}` +line, which is why the difference reads as a regression. + +The text is in the `content[].text` fields. To read it as a sentence: + +```bash +.venv/bin/python scripts/invoke.py "…" \ + | python3 -c 'import sys,json +print("".join(p["text"] for l in sys.stdin if l.startswith("data: ") + for p in (json.loads(l[6:]).get("content") or []) if isinstance(p,dict) and p.get("text")))' +``` + +Do **not** reach for `--agui` — that is for the `agui-*` patterns and fails on +protocol against an `HTTP` runtime. + +### A memory-using pattern recalls within a session but not across sessions + +Working as configured. Measured on `langgraph-agent` with Memory `ACTIVE`: recall in +the same `--session` worked; the same question in a new session returned `NO RECORD`. + +The events are being stored — `list-sessions` shows them. What is missing is a +strategy that turns them into recallable facts: + +```bash +aws bedrock-agentcore-control get-memory --memory-id "$MEMORY_ID" \ + --query 'memory.strategies[].{type:type,status:status}' +# → [{"type": "USER_PREFERENCE", "status": "ACTIVE"}] ← no semantic strategy +``` + +Semantic fact extraction only exists when `use_long_term_memory=true`, and it defaults +to `false` (`app.py:142-143`) because it costs more: + +```bash +USE_LONG_TERM_MEMORY=true ./scripts/deploy.sh deploy --module A +``` + +Related: if you are checking whether *anything* was written, look under the right +actor. For `invoke.py` (M2M) the `actor_id` is the **app client id**, not a username — +`list-actors` returns the value of `/{project}/{env}/auth/m2m-client-id`. + +### I changed agent code and the redeploy changed nothing + +Image tags are a content hash of the source plus the selected pattern, and +CodeBuild only reruns when the hash changes. Identical tags across two deploys +means no rebuild happened: + +```bash +aws ecr describe-images --repository-name $PREFIX-orchestrator \ + --query 'sort_by(imageDetails,&imagePushedAt)[-5:].{tags:imageTags,pushed:imagePushedAt}' +``` + +If you expected a rebuild, confirm you edited a file **inside the build context** +(`agent-code/`), not something excluded from it. + +**The timing is the giveaway.** A module 6 redeploy with nothing changed is a +27.7s no-op end to end — measured — and every stack in the chain reports +`✅ (no changes)` with `Deployment time: 0s`. If your "rebuild" came back +in under a minute, it did not build anything. A real rebuild goes quiet for +minutes while CodeBuild works. So this cuts both ways: the same absence of output +that means "nothing happened" here means "be patient" on a first deploy. + +### `platform.yaml is invalid` + +Validation is deliberately strict and reports **every** problem at once, before +any AWS call. Unknown keys are errors, not no-ops — a typo would otherwise +silently do nothing. + +```bash +.venv/bin/python -m infra_utils.platform_config platform.yaml +``` + +### Old answers keep coming back + +The wizard remembers answers in `workshop.env`, and `platform.yaml` overrides it. + +```bash +./scripts/deploy.sh config # effective values, with their source +./scripts/deploy.sh config --reset # delete saved answers +``` + +### Module 6 has been silent for eight minutes + +Not a hang. CodeBuild is building an **arm64** container image remotely. Expect +~7 minutes on a first build. Confirm if you want to see it move: + +```bash +aws codebuild list-builds-for-project --project-name $PREFIX-build-orchestrator \ + --query 'ids[0]' --output text +``` + +or just watch ECR for a new pushed image. + +### I pressed Ctrl-C during a deploy + +**The script stops. AWS does not.** Measured, interrupting a module 6 rebuild +while CloudFormation was mid-update: + +| What | What happened | +|---|---| +| the script and the CDK CLI | both gone immediately, no grace period, no "stopping deployment" | +| what the log said | `[ERROR] Failed to deploy ` / `[ERROR] Check CloudFormation console for details.` — **byte-identical to a real failure** | +| the CloudFormation stack | still `UPDATE_IN_PROGRESS`, then `UPDATE_COMPLETE` on its own | +| the CodeBuild build | ran to `SUCCEEDED` | +| the runtime | `agentRuntimeVersion` bumped, `status: READY` | + +So the interrupted deploy **succeeded**, and the only thing that failed was the +script's ability to watch it. Do not react to that error text — ask AWS: + +```bash +aws cloudformation describe-stacks --stack-name $PREFIX-runtime-orchestrator \ + --query 'Stacks[0].[StackStatus,LastUpdatedTime]' --output text +``` + +- `UPDATE_COMPLETE` / `CREATE_COMPLETE` → nothing to do. A re-run came back + `(no changes)` in 27.6s. +- `UPDATE_IN_PROGRESS` → **wait.** Do not retry; a second `cdk deploy` against a + stack that is mid-update cannot proceed. Poll the command above. +- `UPDATE_ROLLBACK_COMPLETE` → safe to deploy onto again; re-run the same command. + +The one thing an interrupt can genuinely cost you is a **guided run's place in the +sequence**, since the loop dies with the script. Resume with +`workshop --from ` — the modules before it are skipped explicitly, one +`Skipping module N` line each, and the dependency stacks it walks through come back +`(no changes)`. + +### I passed `--dry-run` and it deployed anyway + +**First: which action did you run it on?** `--dry-run` is only honoured by +`workshop`. The `deploy` case reads `NON_INTERACTIVE`, then calls `cdk_bootstrap` +and deploys — it never tests `DRY_RUN` at all (`scripts/deploy.sh:837-865`). So +`deploy --module 3 --dry-run`, spelled perfectly, is a real deploy. Measured +against a Region that had never been touched: `Total time: 58.51s`, a `CDKToolkit` +bootstrap stack, a live Cognito user pool with three app clients and a hosted +domain, and the M2M client secret printed in the summary table. No warning. + +If that is what happened, tear it down before anything else, and remember the +Region — `destroy` only knows about the Region you point it at: + +```bash +AWS_REGION= ./scripts/deploy.sh destroy +``` + +Then sweep for the CDK staging bucket and any `/aws/lambda/${PREFIX}…` log groups; +see "Tearing down" in `deploy.md`. To actually preview: `workshop --dry-run`, or +`synth` (renders templates, creates nothing). + +**Second: check the spelling.** The argument parser drops anything it does not recognise — +**silently, with no warning** — so `--dryrun`, `--dry_run` or `--dryRun` leaves +`DRY_RUN=0` and the run is real. Same for `--modul 6`, `--profil greenfield`, +`--frm 6`: the flag vanishes and its value is swallowed with it, so the deploy +proceeds unscoped. + +Flag names are never validated. Flag **values** are validated unevenly, and the +two gaps both fail the same expensive way — by deploying the whole app: + +| Value | Bad value does what | +|---|---| +| `--module` | hard error, valid list printed (`scripts/deploy.sh:812`) | +| `--from` | hard error (`:872`) | +| `AGENT_PATTERN` | hard error, valid list printed (`:139-141`) | +| `--profile` | hard error **only on `workshop`** (`:779`) — on `deploy` it is ignored | +| `--team` | **never validated** | + +An unvalidated bad value falls past the `elif` chain that sets `CDK_STACKS` +(`:810-817`), leaving it empty, and an empty stack list means `cdk deploy --all`. +So `deploy --team platfrom` and `deploy --profile greenfeild` each deploy every +stack the app defines, with default feature flags, and report success. Confirm +from the echoed `Workshop Module … → Stacks:` or `Team … → Stacks:` line that the +scope was actually applied. + +Read the output rather than trusting what you typed. A real dry run prints both of +these, and no `═══ Deploying ═══` header after them: + +``` +[INFO] Dry run: skipping prerequisite and credential checks (no AWS calls) +[INFO] DRY RUN — nothing will be deployed +``` + +The `═══ Prerequisite Checks ═══` banner and your account id mean AWS calls are +happening — which is normal for `synth`/`diff` but means a `deploy` is underway. + +### I deployed far more than I expected + +First rule out a misspelled `--module` / `--team` (above) — the value is dropped +with the flag. + +Then: `--profile` sets the profile's **feature flags** and then runs +`cdk deploy --all` — every stack the app defines, not just the profile's modules. +Scope it: + +```bash +./scripts/deploy.sh deploy --profile

--module +./scripts/deploy.sh deploy --profile

--team +./scripts/deploy.sh workshop --profile

# module-by-module +``` + +Extra stacks are not dangerous, but with a networking profile they can be +expensive. Check what landed with +`aws cloudformation describe-stacks --query "Stacks[?starts_with(StackName,'$PREFIX')].StackName"` +and destroy what you did not want. + +--- + +## The agent does not work + +### Invoke returns `Unauthorized` + +Applies to the patterns that verify identity themselves — `strands-agent`, +`langgraph-agent`, `claude-sdk-*`, `agui-*` (via `agent-code/shared/auth.py`). +**The default `orchestrator` pattern reads no token at all, so it never produces +this.** + +Those agents check the caller's JWT — signature against the issuer's JWKS pinned +to RS256, plus expiry, issuer and client — rather than trusting that the runtime +authorizer ran. The response is deliberately generic; **the reason is in the +container logs.** Common causes: + +- `COGNITO_ISSUER_URL` / `COGNITO_ALLOWED_CLIENTS` not injected into the runtime + → the agent refuses rather than decoding unverified. +- A token from a different user pool, or an expired one. +- An M2M token whose client id is not in the allowed clients list. + +One asymmetry worth knowing: a missing **issuer** is a hard reject, but an +**empty** `COGNITO_ALLOWED_CLIENTS` degrades quietly to "any client of the +correct issuer." Set both. + +### Invoke returns HTTP 424 + +**424 means the container is serving the wrong protocol — not that you sent the +wrong payload.** The repo's own `docs/TROUBLESHOOTING.md` and the docstring at +`scripts/invoke.py:112` say a `{"prompt": ...}` body to an A2A runtime "gets you a +424." Measured against a working `code-agent`, it does not; see the next section. +Keep the two apart or you will redeploy a healthy image: + +| Cause | Symptom | Fix | +|---|---|---| +| Image built on `BedrockAgentCoreApp` (HTTP `/invocations`, port 8080) while the runtime is registered `A2A` | **424**, clean container logs | rebuild on `agent-code/shared/a2a_serve.py` | +| Right image, wrong envelope from the client | **200** + JSON-RPC `-32600` | fix the call, use `invoke.py --a2a` | + +The 424 case is a real defect this repo shipped once — stacks clean, logs clean, +every invoke 424 — which is why `tests/test_a2a_contract.py` now guards it +statically. If a sub-agent you wrote 424s, check it serves the contract: `POST /` +(JSON-RPC), `GET /.well-known/agent-card.json`, `GET /ping` returning +`{"status": "Healthy"}` — all on `0.0.0.0:9000`. + +### An A2A invoke returns HTTP 200 and the agent never answered + +Read the body. A correctly built A2A sub-agent accepts the connection and rejects +the *envelope* at the JSON-RPC layer, so the HTTP status is 200 and the failure is +inside the payload: + +```json +{"jsonrpc":"2.0","error":{"code":-32600,"message":"Request payload validation error", + "data":[{"type":"missing","loc":["method"],"msg":"Field required", + "input":{"prompt":"…"}}]}} +``` + +`missing field: method` with your `{"prompt": …}` echoed back as `input` is the +tell: you sent the orchestrator's HTTP shape to a JSON-RPC endpoint. A2A runtimes +want `message/send`. Do not go looking for a 424 and do not redeploy — use the path +that builds the envelope for you: + +```bash +.venv/bin/python scripts/invoke.py --a2a code-agent "Reply with exactly: A2A OK" +``` + +Anything hand-rolling `invoke_agent_runtime` against a sub-agent needs the JSON-RPC +envelope **and** SigV4 (not a Bearer token) **and** a `runtimeSessionId` of at least +33 characters. + +Otherwise compare the runtime's protocol against what its code serves: + +```bash +aws bedrock-agentcore-control get-agent-runtime --agent-runtime-id \ + --query '[agentRuntimeVersion,protocolConfiguration]' +``` + +Expected: A2A sub-agents report `A2A`, an MCP-server runtime `MCP`, the +orchestrator `HTTP` — except `agui-*` patterns, which report `AGUI`. + +### `Invalid length for parameter runtimeSessionId` + +``` +botocore.exceptions.ParamValidationError: Parameter validation failed: +Invalid length for parameter runtimeSessionId, value: 32, valid min length: 33 +``` + +Twenty lines of traceback, exit 1, and no AWS call was made — this is client-side +botocore validation. **Read the last line; the message is accurate.** Session ids +must be **≥33 characters** and `invoke.py` does not enforce it, so `--session` lets a +short one through. + +What makes this confusing is that the two invoke paths disagree, and the strict one +is the one people reach for second: + +| Path | 32-char session id | +|---|---| +| orchestrator (`invoke.py --session`) | **accepted** — travels in the request body | +| A2A (`invoke.py --a2a --session`) | **rejected** — goes to boto3 `InvokeAgentRuntime` | + +So an id that has worked all week starts failing the first time someone points it at +a sub-agent. Use something safely long everywhere: + +```bash +.venv/bin/python scripts/invoke.py --a2a code-agent --session "session-$(uuidgen | tr -d -)" "…" +``` + +### `Authorization method mismatch` on invoke + +The runtime's inbound auth and your request disagree, and it cuts **both** ways: + +| Runtime | Expects | Call it with | +|---|---|---| +| orchestrator (`HTTP`/`AGUI`/`MCP`) | Bearer JWT (CUSTOM_JWT authorizer) | `invoke.py` / `invoke.py --agui` | +| A2A sub-agents (`A2A`) | **SigV4** — no authorizer; IAM `InvokeAgentRuntime` | `invoke.py --a2a ` | + +A2A is not a client-facing protocol, so those runtimes deliberately get no JWT +authorizer. Sending a bearer token to one is rejected exactly like sending SigV4 +to the orchestrator. `scripts/invoke.py` picks the right mechanism per target. + +### The agent reports no tools + +**First: is this the `orchestrator` pattern?** It ships with no tools by design. +Tools are consumed by `strands-agent`, `langgraph-agent`, `claude-sdk-*` and +`agui-*`. + +Otherwise gateway tools load through AgentCore Identity — the agent exchanges M2M +credentials for a gateway token via the token vault. Two failure shapes, and +telling them apart saves real time: + +- **Silently no tools** — the agent answers, just without them. Only two + conditions do this, both `return None` in `create_gateway_mcp_client()` + (`agent-code/strands-agent/tools/gateway.py:79-93`, duplicated per + tool-consuming pattern): `GATEWAY_CREDENTIAL_PROVIDER_NAME` unset, or the + gateway URL unresolvable. + Check both env vars on the runtime and look for `[GATEWAY]` warnings in the + container logs — the client logs why it gave up. +- **The invoke fails outright** — if the runtime role cannot read the vault's + secret (`secretsmanager:GetSecretValue` on + `bedrock-agentcore-identity!default/oauth2/*`), the token fetch raises + AccessDenied inside the MCP client and the agent aborts rather than degrading. + An invoke that *errors* instead of *answering* points here. + +Then confirm the gateway side is healthy and the tool is registered: + +```bash +.venv/bin/python scripts/test_gateway.py +``` + +Also: the built-in web-search connector only exists in some Regions and is +Region-gated in `app.py`, so in an unsupported Region the tool is absent **by +design**. + +A tool that is listed but whose calls fail is usually a missing gateway-role +action for the connector, or a missing Lambda invoke permission. A +`Unsupported tool: …` error from your own Lambda means the handler is dispatching +on the full `target___tool` name instead of the suffix after `___`. + +### A memory-backed pattern fails at invoke + +Framework memory integrations call data-plane actions — LangGraph's checkpointer +lists events to rehydrate a thread. If the **runtime role** lacks `ListEvents` / +`CreateEvent` / `RetrieveMemoryRecords`, the pattern fails at invoke while the +stack looks fine. + +`scripts/test_memory.py` cannot tell you this: it uses **your local +credentials**, so it passes regardless. Check the role directly: + +```bash +aws iam list-role-policies --role-name -orchestrator-role +``` + +--- + +## Observability + +### Module 9's verify fails with `trace segment destination is CloudWatchLogs but status is PENDING` + +``` +Checking /agentcore-workshop/dev in us-east-1 +FAIL: trace segment destination is CloudWatchLogs but status is PENDING +``` + +Nothing is broken. Enabling Transaction Search is **asynchronous and slower than the +stack that requests it**, and `check_observability.py` runs about a second after +`CREATE_COMPLETE`. It has no retry, so on a first-ever enablement in an account it +fails by design of the timing, not of the platform. + +Measured on a fresh account: stack complete at `15:06:19Z`, still `PENDING` two +minutes later, `ACTIVE` at `15:14:41Z` — **8m22s**. Watch it directly rather than +guessing: + +```bash +aws xray get-trace-segment-destination --region $AWS_REGION +# {"Destination": "CloudWatchLogs", "Status": "PENDING"} → wait +# {"Destination": "CloudWatchLogs", "Status": "ACTIVE"} → re-run the verify +``` + +Then simply re-run it — it passes with no redeploy: + +``` +PASS: trace segment destination is CloudWatchLogs (ACTIVE) +PASS: X-Ray span delivery policy present (…-transaction-search-xray) +PASS: 3 vended log deliveries active +OK: logs and traces are being accepted +``` + +Two consequences worth planning around. In a guided run this is the **last** module +of `greenfield`, so the session's final step is the one most likely to go red — +pre-empt it. And under `NON_INTERACTIVE=1` a failed verify aborts the run +(`Aborting (NON_INTERACTIVE=1 cannot prompt)`), so an automated walk of the full +sequence needs either a wait before module 9 or a re-run afterwards. + +Only the first enablement in an account is slow. Once `ACTIVE`, it stays that way — +including after teardown. + +### No traces appear anywhere + +Runtimes emit OTLP spans even when the account cannot receive them: while the +X-Ray trace segment destination is still `XRay`, **every batch is rejected with +HTTP 400 and the deployment still reports success.** The observability stack sets +the destination, so the usual cause is deploying with +`enable_transaction_search=false`. + +```bash +.venv/bin/python scripts/check_observability.py +aws xray get-trace-segment-destination # expect CloudWatchLogs / ACTIVE +``` + +### `batch-get-traces` returns nothing for my trace + +Expected, not a failure. With Transaction Search **all** spans are searchable in +the `aws/spans` log group, while the classic X-Ray APIs only serve the indexed +sample (default rule: 1%). Search the way the console does: + +```bash +.venv/bin/python scripts/check_observability.py --spans # needs an invoke in the last hour +``` + +Also give it time — span delivery lags an invocation by a minute or two, which is +exactly why `--spans` is not part of module 9's verify. + +--- + +## Teardown + +### `destroy --stack` fails with "Export … cannot be deleted" + +Working as intended, and **nothing was deleted**. Measured on +`destroy --stack $PREFIX-auth` with three consumers standing: exit 1 after 22s, and +the stack went `DELETE_IN_PROGRESS` → straight back to `CREATE_COMPLETE`. + +``` +agentcore-…-auth | 1 | CREATE_COMPLETE | AWS::CloudFormation::Stack + Delete canceled. Cannot delete export agentcore-…-auth:ExportsOutputRefUserPool… + as it is in use by agentcore-…-gateway, agentcore-…-identity and + agentcore-…-runtime-orchestrator. +[ERROR] If the error above names an export in use, another stack depends on this one. +[ERROR] Destroy the dependents first, or run 'destroy --all' for the whole environment. +``` + +A targeted destroy passes `--exclusively`, so CloudFormation refuses rather than +cascading into the dependents — which would take the platform out from under other +teams. **The message names every consumer**, which makes it a dependency-discovery +tool as much as an error: run it to find out who depends on a stack before you plan +a change. Either destroy those first, or take the whole environment down: + +```bash +./scripts/deploy.sh destroy # cascade is intended here +``` + +One footnote on that error text: `--all` is not a real flag. The parser ends in +`*) shift ;;`, so `destroy --all` silently drops it and works only because a bare +`destroy` already destroys everything. Do not go looking for `--all` elsewhere. + +### `destroy` exited non-zero and half the platform is still standing + +**Expected, and the half still standing is the expensive half.** A bare `destroy` +runs `cdk destroy --all --force` (`scripts/deploy.sh:590-595`), and CDK **stops at +the first stack it cannot delete** rather than skipping it and continuing. + +Because the order is reverse-dependency — observability → runtime stacks → gateway, +memory, identity, auth, security → **networking last** — a failure in an early, +cheap runtime stack leaves the NAT gateway and all five interface endpoints running. +Measured: 693s, failed on the fourth of ten stacks, exit 1, six stacks left up +including `-networking`, NAT still `available`. + +```bash +# What is left +aws cloudformation list-stacks \ + --query "StackSummaries[?StackStatus!='DELETE_COMPLETE'].[StackName,StackStatus]" \ + --output text | sort + +# Is anything still on an hourly meter? +aws ec2 describe-nat-gateways --filter Name=state,Values=available \ + --query 'NatGateways[].NatGatewayId' --output text +``` + +Clear the failed stack (next entry is the usual cause), then **re-run +`./scripts/deploy.sh destroy`** to take out the remainder. + +### `DELETE_FAILED`: "Request timed out while deleting AWS::BedrockAgentCore::Runtime" + +``` +DELETE_FAILED (The following resource(s) failed to delete: [Runtime]. ): +Resource handler returned message: "Request timed out while deleting +AWS::BedrockAgentCore::Runtime" (HandlerErrorCode: NotStabilized) +``` + +**`NotStabilized` means CloudFormation gave up waiting, not that the delete +failed.** In the measured case the runtime was already gone at the service level +while the stack sat in `DELETE_FAILED`. Confirm that before doing anything drastic — +if the list is empty, there is nothing left to clean up by hand: + +```bash +aws bedrock-agentcore-control list-agent-runtimes \ + --query 'agentRuntimes[].[agentRuntimeName,status]' --output text +``` + +Then just retry the delete. **Measured: 34 seconds to `DELETE_COMPLETE`** on a stack +that had timed out minutes earlier. + +```bash +aws cloudformation delete-stack --stack-name "$PREFIX-runtime-" +aws cloudformation wait stack-delete-complete --stack-name "$PREFIX-runtime-" +``` + +Ignore the handler's advice to "delete it from the AWS console" — a plain retry is +enough, and the console does the same call. Afterwards re-run +`./scripts/deploy.sh destroy` for the stacks the abort skipped. + +### Destroying the networking stack fails on subnets or security groups + +**It fails fast and then you wait — those are two different clocks.** Measured: the +stack reached `DELETE_FAILED` about 17 minutes after the delete started. It does not +sit in `DELETE_IN_PROGRESS` for hours. The ~8 hours is how long the ENIs may take to +release before a *retry* will succeed. + +The three failure reasons, verbatim, so they are searchable: + +``` +RuntimeSecurityGroup resource sg-… has a dependent object (Service: Ec2, Status Code: 400) +VPCPrivateSubnet1 The subnet 'subnet-…' has dependencies and cannot be deleted. +VPCPrivateSubnet2 The subnet 'subnet-…' has dependencies and cannot be deleted. +``` + +Neither message mentions ENIs, AgentCore, or runtimes — which is why this reads as a +mystery the first time. The "dependent object" is an `agentic_ai` ENI. + +AgentCore leaves those ENIs behind for up to ~8 hours after runtimes stop +using a VPC. Measured: both ENIs still `in-use` after every runtime had been deleted +and `list-agent-runtimes` came back **empty** — the ENI outlives the runtime that +created it by hours. + +```bash +aws ec2 describe-network-interfaces \ + --filters Name=interface-type,Values=agentic_ai \ + --query 'NetworkInterfaces[].{id:NetworkInterfaceId,status:Status,subnet:SubnetId}' +``` + +**Exactly three resources get stuck, and they are the cheap ones:** the runtime +security group (the ENIs hold it) and the two private subnets. Measured on a real +teardown, at the same moment: NAT gateway `deleted`, all five interface endpoints +gone, public subnets gone — only `RuntimeSecurityGroup` and +`VPCPrivateSubnet{1,2}` left `DELETE_IN_PROGRESS`. + +**Do not try to delete the ENIs by hand.** They report +`RequesterManaged: false` and an empty `Description`, which makes them look like +ordinary account-owned interfaces, and they are not: + +``` +$ aws ec2 delete-network-interface --network-interface-id eni-… +InvalidParameterValue: Network interface 'eni-…' is currently in use. +``` + +The attachment is an `ela-attach-…` with `InstanceId: None` and +`InstanceOwnerId: amazon-aws` — a service-managed attachment with no instance to +detach from, so `detach-network-interface` is not an option either. There is no +force. The only move is to come back later and re-run the destroy. + +The filter is the only reliable way to find them, since `Description` is blank and +`RequesterManaged` lies: match on `interface-type=agentic_ai`. + +If `-networking` is the *only* stack left, NAT and the endpoints went with the rest +of it and the wait costs nothing meaningful. **Confirm that it really is the only one +left** — if the destroy aborted earlier in the sequence, networking was never +attempted and the hourly meters are still running. See the abort entry above. + +**Do not let the drain hold the rest of the teardown hostage.** `cdk destroy --all` +works through stacks serially, so while it sits on `-networking` it has not yet +touched `-auth`, `-gateway`, `-identity` or `-security` — none of which depend on +networking. Leaving `-auth` up is the part that matters, because that is the stack +whose CloudFormation export carries the M2M client secret in plaintext. + +Take them out directly instead of waiting. Dependents of `-auth` first, then `-auth`: + +```bash +for s in gateway identity security; do + aws cloudformation delete-stack --stack-name "$PREFIX-$s" +done +for s in gateway identity security; do + aws cloudformation wait stack-delete-complete --stack-name "$PREFIX-$s" +done +aws cloudformation delete-stack --stack-name "$PREFIX-auth" +aws cloudformation wait stack-delete-complete --stack-name "$PREFIX-auth" +``` + +Measured: all four gone, `-auth` in 34s, leaving only `-networking` waiting on its +ENIs. Then re-run `./scripts/deploy.sh destroy` later to finish networking. + +### Deleting the observability stack did not disable Transaction Search + +Deliberate. The trace segment destination is account- and Region-scoped, and +other workloads may depend on it by the time you tear this down. Revert it +yourself if you really want to: + +```bash +aws xray update-trace-segment-destination --destination XRay +``` + +--- + +## Still stuck + +Collect these first — they answer the first three questions anyone will have: + +```bash +./scripts/deploy.sh config +aws sts get-caller-identity +aws cloudformation describe-stacks \ + --query "Stacks[?starts_with(StackName,'$PREFIX')].{n:StackName,s:StackStatus}" --output table +./scripts/deploy.sh verify +``` diff --git a/kiro/agentcore-enterprise-platform/steering/verify.md b/kiro/agentcore-enterprise-platform/steering/verify.md new file mode 100644 index 0000000..5acc238 --- /dev/null +++ b/kiro/agentcore-enterprise-platform/steering/verify.md @@ -0,0 +1,415 @@ +# Verifying + +Read this when you need to prove a layer works — or prove it does not. + +**A stack reaching `CREATE_COMPLETE` proves nothing about behaviour.** These +scripts exist because stacks have completed successfully while the thing they +promise was broken. Runtimes emitted spans for weeks while X-Ray rejected every +batch. Run the checks. + +All commands from the repo root with `.venv` active. Env defaults: +`AWS_REGION=us-east-1`, `PROJECT_NAME=agentcore-workshop`, `ENVIRONMENT=dev`. + +**On macOS, prove the certs work before you trust any failure here.** These scripts +fetch the Cognito token with bare `urllib`, not boto3, so on a python.org build of +`python3.13` they all die with `CERTIFICATE_VERIFY_FAILED` while every `aws` command +keeps working — a failure that looks like a broken gateway and is not: + +```bash +python3.13 -c "import ssl; print(ssl.get_default_verify_paths().cafile)" # None == broken +"/Applications/Python 3.13/Install Certificates.command" # the fix +``` + +See `troubleshooting.md` → `CERTIFICATE_VERIFY_FAILED`. + +--- + +## The whole set + +```bash +./scripts/deploy.sh verify # every check this config promises; non-zero on failure +.venv/bin/python scripts/invoke.py "Reply with exactly: WORKSHOP OK" +.venv/bin/python scripts/invoke.py --tools # tools registered on the GATEWAY +.venv/bin/python scripts/invoke.py --agui "…" # agui-* patterns (typed SSE) +.venv/bin/python scripts/invoke.py --a2a code-agent "…" # A2A sub-agent (JSON-RPC + SigV4) +.venv/bin/python scripts/test_gateway.py # gateway direct: tools/list + tools/call +.venv/bin/python scripts/test_memory.py # Memory data-plane operations +.venv/bin/python scripts/test_agent.py # interactive chat with a runtime +.venv/bin/python scripts/check_observability.py # logs + traces actually accepted +.venv/bin/python scripts/check_observability.py --spans # …and spans searchable end to end +.venv/bin/python scripts/check_network.py # runtimes really are in the VPC +``` + +--- + +## `invoke.py` — the main tool + +``` +.venv/bin/python scripts/invoke.py [PROMPT] [--session ID] [--tools] [--agui] [--a2a [COMPONENT]] +``` + +| Flag | Does | +|---|---| +| *(positional)* | the prompt | +| `--session ID` | reuse a session — see the length caveat below | +| `--tools` | list the gateway's MCP tools instead of invoking the agent | +| `--agui` | use the AG-UI protocol — required for `agui-*` patterns | +| `--a2a [COMPONENT]` | invoke an A2A sub-agent over JSON-RPC; defaults to `code-agent` | + +Prefer `invoke.py` over hand-rolled calls: it picks the right auth mechanism per +target — Bearer JWT for the orchestrator, SigV4 for A2A sub-agents — which is +exactly the thing people get wrong. + +**Session ids: the ≥33-character rule applies on one path and not the other, and +`invoke.py` enforces it on neither.** `new_session_id()` builds a 40-char id and the +`--session` help says `>=33 chars`, but the value is passed straight through +unvalidated. Both behaviours are measured: + +| Path | 32-char id | 33-char id | +|---|---|---| +| orchestrator (HTTP, `--session`) | **accepted** — a 15-char id worked | fine | +| A2A (`--a2a`, boto3 `InvokeAgentRuntime`) | **rc=1**, unhandled traceback | fine | + +So a session id that works all week on the orchestrator breaks the moment someone +tries it against a sub-agent. The boundary is exact — 33 passes, 32 does not — and +the last line of the traceback says so plainly: + +``` +botocore.exceptions.ParamValidationError: Parameter validation failed: +Invalid length for parameter runtimeSessionId, value: 32, valid min length: 33 +``` + +It is client-side botocore validation, so no AWS call is made. The message is good; +what makes it look like a crash is that `invoke.py` does not catch it, so a room sees +twenty lines of stack trace. **Read the last line, not the traceback.** Just use ids +≥33 characters everywhere: `session-$(uuidgen | tr -d -)`. + +**And `--session` does not give the default agent a memory.** Reusing a session id +does not make the `orchestrator` pattern recall anything — it never reads Memory. See +`agent-patterns.md`; a recall demo needs `strands-agent` or `langgraph-agent`. + +### `--tools` answers a narrower question than it looks like + +`--tools` and `test_gateway.py` talk to the gateway with a machine token. They +tell you what is **registered on the gateway**. That is not the same question as +what a given agent **loaded**. + +```bash +.venv/bin/python scripts/invoke.py --tools +# sample-tool___text_analysis_tool, web-search___WebSearch +``` + +To see the agent's own view, ask a tool-using pattern: + +```bash +.venv/bin/python scripts/invoke.py "List the names of the tools you have available. Names only." +``` + +The default `orchestrator` pattern has **no tools**, so it correctly answers +nothing. Use `strands-agent`, `langgraph-agent`, `claude-sdk-*` or `agui-*` for +that question. + +--- + +## `verify.py` — the one check for the whole deployment + +```bash +./scripts/deploy.sh verify # or: .venv/bin/python scripts/verify.py +``` + +The broadest single "is this platform actually working" probe, and the first thing +to collect when asking anyone for help. It does not add a check of its own — it +asks the deployment contract what *this* configuration promises and runs the tool +that proves each promise (`scripts/verify.py:56-81`): + +| Stack in the footprint | Check it runs | +|---|---| +| `-gateway` | `test_gateway.py` — `tools/list` then a real `tools/call` | +| `-memory` | `test_memory.py` | +| `-observability` | `check_observability.py` | +| `-networking` | `check_network.py` | +| `-runtime-orchestrator` | `invoke.py` (adds `--agui` for `agui-*` patterns) | +| `-runtime-code-agent` | `invoke.py --a2a code-agent` | +| `-runtime-research-agent` | `invoke.py --a2a research-agent` | + +**Any failed check exits 1** (`scripts/verify.py:110-113`). That is the point of +it, and it is worth saying explicitly because its predecessor did the opposite. + +Two things to read carefully before treating a pass as proof: + +- **It verifies what the configuration promises, not what the account contains.** + The footprint comes from `expected_stacks()` against `platform.yaml` — schema + defaults if that file is absent, env vars winning over both + (`scripts/verify.py:44-52`). So a manifest with networking off will not run + `check_network.py` even when a VPC is deployed and the runtimes are outside it. + `OK: all 4 checks passed` means four claims held, and the header line prints the + footprint it derived — read that line, not just the verdict. +- **Four stacks in the footprint have no check at all**: `-auth`, `-identity`, + `-security`, and any `uc-*` use-case stack. `verify` is silent on identity and + on the security stack, so it is not the answer to "are the controls working." + It also runs `check_observability.py` without `--spans`, so span searchability + is not covered. + +In a federated deployment the footprint is role-dependent, so `verify.py` calls +`sts:GetCallerIdentity` and lets the account decide which half it is checking +(`scripts/verify.py:85-90`). Running it against the wrong account does not fail — +it checks a different, smaller thing. + +**If you are looking at an older checkout, this command did not exist and its +predecessor could not fail.** `scripts/test.py` was the README's headline health +check; it swallowed invoke exceptions and printed them as expected, hardcoded the +default project name, marked zero discovered SSM parameters as a success, and had +no non-zero exit path anywhere — a broken deployment ended with `Done.` and exit +0. Its most alarming symptom was a red `✗ AccessDeniedException … Authorization +method mismatch` on step 2 of a perfectly healthy platform, because boto3's +`InvokeAgentRuntime` cannot authenticate against a `CUSTOM_JWT` runtime at all. +If someone reports that output, they are on a pre-`verify.py` checkout: the answer +is `git pull` and `deploy.sh verify`, not a configuration change. + +--- + +## `test_gateway.py` — the gateway directly + +``` +.venv/bin/python scripts/test_gateway.py [--project NAME] [--env ENV] +``` + +Fetches an M2M token, runs `tools/list`, then a real `tools/call`. This is +module 5's and module 7's verify, and it is the fastest way to tell "the gateway +is broken" from "the agent cannot reach the gateway." + +--- + +## `test_memory.py` — Memory operations + +``` +.venv/bin/python scripts/test_memory.py [--memory-arn ARN] [--project NAME] [--env ENV] +``` + +`--memory-arn` overrides the SSM lookup. + +**What it does not prove:** it uses **your local credentials**, so it passes +regardless of whether the *runtime role* can reach Memory. Framework memory +integrations call data-plane actions — LangGraph's checkpointer lists events to +rehydrate a thread — so a runtime role missing `ListEvents` / `CreateEvent` / +`RetrieveMemoryRecords` fails at invoke while this script and the stack both look +fine. To check the role: + +```bash +aws iam list-role-policies --role-name -orchestrator-role +``` + +--- + +## `test_agent.py` — interactive chat + +``` +.venv/bin/python scripts/test_agent.py [--project NAME] [--env ENV] [--component COMPONENT] +``` + +`--component` defaults to `orchestrator`. Useful in a session when you want to +hand someone a prompt loop rather than re-running `invoke.py`. + +--- + +## `check_observability.py` — the tracing claim + +``` +.venv/bin/python scripts/check_observability.py [--spans] +``` + +Three checks against live state: + +1. the trace segment destination is `CloudWatchLogs` and `ACTIVE` +2. a CloudWatch Logs resource policy lets X-Ray write the span log groups +3. vended log delivery exists for each monitored AgentCore resource + +`--spans` adds the end-to-end proof: it finds a span emitted in the last hour in +`aws/spans` and queries it back by `traceId` through Logs Insights — the same +path the Transaction Search console uses. It needs **at least one agent +invocation in the last hour** and is opt-in precisely because span delivery lags +invocation by a minute or two, which would make module 9's gate flaky. + +```bash +.venv/bin/python scripts/invoke.py "hi" +sleep 120 +.venv/bin/python scripts/check_observability.py --spans +# PASS: trace searchable (5 spans, service agentcore_workshop_dev_orchestrator.DEFAULT) +``` + +**Do not "fix" this:** `aws xray batch-get-traces` and `get-trace-summaries` +return nothing for most traces. With Transaction Search, *all* spans are +searchable in `aws/spans`, while the classic X-Ray APIs only serve the indexed +sample (default rule: 1%). An empty trace-API result is expected, not a delivery +failure. + +```bash +aws xray get-trace-segment-destination # expect CloudWatchLogs / ACTIVE +``` + +**`PENDING` there is the one failure of this script that means "wait", not "fix".** +On a first-ever enablement in an account the destination is still `PENDING` when the +stack completes, so check 1 fails: + +``` +FAIL: trace segment destination is CloudWatchLogs but status is PENDING +``` + +Measured: **8m22s** from `CREATE_COMPLETE` to `ACTIVE`. Poll the command above and +re-run; all four checks then pass with nothing redeployed: + +``` +PASS: trace segment destination is CloudWatchLogs (ACTIVE) +PASS: X-Ray span delivery policy present (…-transaction-search-xray) +PASS: 3 vended log deliveries active +OK: logs and traces are being accepted +``` + +--- + +## `check_network.py` — the isolation claim + +``` +.venv/bin/python scripts/check_network.py [--expect-public] +``` + +Verifies runtime network placement. Two uses: + +- After module C, plain `check_network.py` proves runtimes are actually in the + VPC's private subnets — not merely that a VPC exists. +- `--expect-public` asserts the opposite, which is how you confirm a non-VPC + deployment is public **deliberately** rather than accidentally. + +**It checks two independent things and stops at the first failure**, so on a fresh +account you will meet them one at a time: + +1. **AZ ids.** AgentCore supports a limited set per Region and AZ name → id mapping + differs per account, so `max_azs=2` can land you outside the supported set with + the stack still `CREATE_COMPLETE`. Fixing this needs a source edit *and* a stack + destroy — see `troubleshooting.md` before you start. +2. **Runtime placement.** Deploying module C does not move existing runtimes into + the VPC; they stay `PUBLIC` until redeployed. + +Because of (2), **`--expect-public` passing is not evidence of anything on its own.** +Measured: with the networking stack `CREATE_COMPLETE` and all three runtimes still +public, `--expect-public` reported `OK: network placement matches the deployment's +claim` and exited 0. It is answering the question you asked, not the one you meant. +Once the runtimes moved it correctly returned exit 1 with +`FAIL: code-agent: expected networkMode PUBLIC, got VPC`. + +A full pass looks like this: + +``` +PASS: private subnets are in supported AZs (use1-az2, use1-az1) +PASS: code-agent is in the VPC (2 subnets) +PASS: orchestrator is in the VPC (2 subnets) +PASS: research-agent is in the VPC (2 subnets) +OK: network placement matches the deployment's claim +``` + +Corroborate with the ENIs themselves — this is the artifact a compliance reviewer +asks for: + +```bash +aws ec2 describe-network-interfaces \ + --filters Name=interface-type,Values=agentic_ai \ + --query 'NetworkInterfaces[].[NetworkInterfaceId,AvailabilityZone,SubnetId,Status]' --output text +``` + +Expect **fewer ENIs than runtimes** — three runtimes in the VPC produced two +`agentic_ai` ENIs, one per private subnet, shared across runtimes. That is normal; +"one ENI per agent" is not the shape. + +Finish by proving isolation did not break the thing you isolated: + +```bash +.venv/bin/python scripts/invoke.py "Reply with exactly: VPC OK" +.venv/bin/python scripts/invoke.py --a2a code-agent "Reply with exactly: A2A VPC OK" +``` + +--- + +## Confirming what a deploy actually produced + +Stack status alone is weak. These two answer "is the thing I asked for running?" + +```bash +# Protocol + version + image the runtime is really on +ARN=$(aws ssm get-parameter --name /$PROJECT_NAME/$ENVIRONMENT/runtimes/orchestrator/arn \ + --query Parameter.Value --output text) +aws bedrock-agentcore-control get-agent-runtime --agent-runtime-id "${ARN##*/}" \ + --query '[agentRuntimeVersion,protocolConfiguration,agentRuntimeArtifact]' +# agui-* patterns → AGUI; A2A sub-agents → A2A; everything else → HTTP + +# Did CodeBuild actually rebuild? Identical tags across two deploys means it did not. +aws ecr describe-images --repository-name $PREFIX-orchestrator \ + --query 'sort_by(imageDetails,&imagePushedAt)[-5:].{tags:imageTags,pushed:imagePushedAt}' +``` + +--- + +## The local dashboard + +Status only, runs on your machine, no AWS resources: + +```bash +.venv/bin/python dashboard/monitor.py & +python3 -m http.server 8888 -d dashboard/public +# http://localhost:8888 +``` + +Good for a facilitated session — it gives the room something to watch during +module 6's silent seven minutes. + +--- + +## Local checks that need no AWS account + +Useful before a session, or in CI: + +```bash +make lint # ruff +make validate-controls # control-library ↔ catalog.yaml consistency +make test-controls # pytest tests/ -q +make check-shell # shellcheck + deploy-config + workshop-flow checks +.venv/bin/python -m infra_utils.platform_config platform.yaml +./scripts/deploy.sh workshop --dry-run --profile

+``` + +`make check-shell` uses `BASH ?= /opt/homebrew/bin/bash` — override `BASH` if +your bash 4+ lives elsewhere. + +`cdk synth` is the cheap way to prove a feature flag does what it claims without +deploying. `docs/TESTING.md` in the repo has ready-made synth assertions per +control — flag off → resource count 0, flag on → resource present — which is a +good answer to "prove this control is real" in a security conversation. + +--- + +## When a verify fails + +**A failing verify is useful information, not a dead end.** The guided run offers +to continue; if you say yes, write down which module failed. Later modules build +on it, and the failure usually explains a stranger symptom two modules later. + +Read the container logs before changing anything — every runtime failure so far +named its own cause there: + +```bash +aws logs filter-log-events \ + --log-group-name "/aws/bedrock-agentcore/runtimes/-DEFAULT" \ + --start-time $(( ($(date +%s) - 900) * 1000 )) \ + --query 'events[].message' --output text | grep -iE 'error|denied|traceback' +``` + +Then go to `troubleshooting.md`. + +## Collect these before asking anyone for help + +```bash +./scripts/deploy.sh config +aws sts get-caller-identity +aws cloudformation describe-stacks \ + --query "Stacks[?starts_with(StackName,'$PREFIX')].{n:StackName,s:StackStatus}" --output table +./scripts/deploy.sh verify +``` diff --git a/scripts/check-kiro-power.sh b/scripts/check-kiro-power.sh new file mode 100755 index 0000000..85d2c4e --- /dev/null +++ b/scripts/check-kiro-power.sh @@ -0,0 +1,576 @@ +#!/usr/bin/env bash +# +# Validate the Kiro power in kiro/agentcore-enterprise-platform against Kiro's +# power format, against Kiro's installer rules, and — the part that matters — +# against this repository's own source. +# +# bash scripts/check-kiro-power.sh +# +# No AWS calls, no network. Runs in CI on every pull request. +# +# The power's whole value is that a reader can trust what it says without +# checking. A hallucinated flag reads exactly like a real one and only fails in +# front of a user, and a renamed flag is indistinguishable from a hallucinated +# one a month later. So this script pins the power's claims to the tree it ships +# with: every file:line citation resolves and is in bounds, every restatement of +# a profile's module sequence matches PROFILE_MODULES in scripts/deploy.sh, and +# every --flag cited next to one of our scripts exists in that script. Shipping +# the power here rather than in a repository of its own is what makes that +# possible: a rename breaks the build instead of quietly breaking a session. +# +set -uo pipefail + +REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +cd "$REPO_ROOT" || exit 1 + +NAME="agentcore-enterprise-platform" +POWER_DIR="kiro/$NAME" +status=0 + +fail() { echo " FAIL $*"; status=1; } +ok() { echo " ok $*"; } + +if [ ! -f "$POWER_DIR/POWER.md" ]; then + echo "FAIL: no power at $POWER_DIR/POWER.md" + exit 1 +fi + +echo "== Power format: $POWER_DIR/POWER.md ==" +python3 - "$POWER_DIR" "$NAME" <<'PY' || status=1 +import pathlib +import re +import sys + +power_dir, name = pathlib.Path(sys.argv[1]), sys.argv[2] +text = (power_dir / "POWER.md").read_text() +bad = False + + +def fail(msg): + global bad + print(f" FAIL {msg}") + bad = True + + +m = re.match(r"^---\n(.*?)\n---\n", text, re.S) +if not m: + fail("no YAML frontmatter block at the top of POWER.md") + sys.exit(1) +fm = m.group(1) + +# Kiro's PowerFrontmatterSchema takes exactly these five. There is no version, +# tags, repository or license field; adding one is an error, not an extension. +keys = re.findall(r"^([A-Za-z][A-Za-z0-9_]*):", fm, re.M) +allowed = ["name", "displayName", "description", "keywords", "author"] +extra = [k for k in keys if k not in allowed] +missing = [k for k in allowed if k not in keys] +if extra: + fail(f"fields that do not exist in the power format: {extra}") +if missing: + fail(f"required fields missing: {missing}") +if len(keys) != len(set(keys)): + fail("duplicate frontmatter keys") +if not extra and not missing: + print(" ok exactly the 5 allowed fields") + + +def scalar(key): + mm = re.search(rf"^{key}:\s*(.+)$", fm, re.M) + return mm.group(1).strip().strip('"').strip("'") if mm else None + + +# Kiro warns when these differ, and the power installs under the frontmatter +# name — so a mismatch means the directory a contributor edits and the power a +# user installs have different names. +if scalar("name") != name: + fail(f"name ({scalar('name')!r}) must equal the directory name ({name!r})") +else: + print(" ok name == directory name") + +desc = scalar("description") or "" +if not desc: + fail("description is empty") +else: + sentences = [s for s in re.split(r"(?<=[.!?])\s+", desc.strip()) if s] + if len(sentences) > 3: + fail(f"description is {len(sentences)} sentences; the limit is 3") + else: + print(f" ok description is {len(sentences)} sentence(s), {len(desc)} chars") + +kw_block = re.search(r"^keywords:\s*\n((?:\s*-\s*.+\n)+)", fm, re.M) +if kw_block: + kws = re.findall(r"-\s*(.+)", kw_block.group(1)) +elif scalar("keywords"): + kws = [k for k in scalar("keywords").strip("[]").split(",") if k.strip()] +else: + kws = [] +kws = [k.strip().strip('"').strip("'") for k in kws] + +if not 5 <= len(kws) <= 7: + fail(f"{len(kws)} keywords; the guidance is 5-7") +else: + print(f" ok {len(kws)} keywords") + +# A power that activates on "aws" or "deploy" gets uninstalled. Broad keywords +# are the fastest way there, so they are a failure rather than a style note. +broad = {"aws", "deploy", "test", "agent", "agents", "ai", "cloud", "python", + "cdk", "security", "bedrock", "mcp", "workshop", "platform"} +offenders = [k for k in kws if k.lower() in broad] +if offenders: + fail(f"keywords too broad (false-positive activation): {offenders}") + +# The steering index and the steering directory must agree in both directions. +# A steering file POWER.md never names is unreachable; a name POWER.md routes to +# that does not exist sends the agent looking for nothing. +on_disk = {f.name for f in (power_dir / "steering").glob("*.md")} +referenced = set(re.findall(r"`([a-z0-9-]+\.md)`", text)) & on_disk +if unreferenced := on_disk - referenced: + fail(f"steering files never referenced from POWER.md: {sorted(unreferenced)}") +for ref in sorted(re.findall(r"^\| `([a-z0-9-]+\.md)` \|", text, re.M)): + if ref not in on_disk: + fail(f"POWER.md's steering index points at a file that does not exist: {ref}") +if not bad: + print(f" ok steering index covers all {len(on_disk)} files, no dangling refs") + +sys.exit(1 if bad else 0) +PY + +echo +echo "== Installability (Kiro rejects a power containing anything else) ==" +# Kiro's validatePowerDirectory throws before install if the directory holds a +# hidden file at ANY depth, a script, an archive, or a credential-shaped name. +# The failure message does not name the offending rule, so a stray .DS_Store +# makes the power simply refuse to install. .DS_Store is gitignored here, which +# keeps the committed tree clean but not a contributor's working copy. +python3 - "$POWER_DIR" <<'PY' || status=1 +import pathlib +import sys + +power_dir = pathlib.Path(sys.argv[1]) +bad = False + +# From the extension's DISALLOWED_FILE_PATTERNS. Matched as substrings of the +# filename, lowercased, exactly as Kiro matches them. +DISALLOWED = [ + ".env", ".key", ".pem", ".p12", "id_rsa", "id_dsa", "id_ecdsa", "id_ed25519", + "credentials", ".secret", "token", "password", "api_key", "apikey", ".npmrc", + ".aws", ".exe", ".dll", ".com", ".msi", ".scr", ".so", ".dylib", ".bin", + ".run", ".sh", ".bash", ".zsh", ".bat", ".cmd", ".ps1", ".vbs", ".vbe", + ".js", ".jsx", ".ts", ".tsx", ".py", ".rb", ".pl", ".php", ".zip", ".tar", + ".gz", ".tgz", ".bz2", ".xz", ".7z", ".rar", ".db", ".sqlite", ".sqlite3", + ".sql", "node_modules", "package-lock.json", "yarn.lock", "pnpm-lock.yaml", +] + +def is_allowed(rel, name, is_dir): + """Kiro's isAllowedFile, faithfully. Checked BEFORE the disallowed patterns, + which is why mcp.json is fine despite containing '.js' as a substring.""" + if is_dir: + return len(rel.parts) == 1 and name in ("steering", ".git") + if len(rel.parts) == 1 and name in ("POWER.md", "mcp.json"): + return True + if rel.parts[0] == "steering": + return name.lower().endswith(".md") + return False + + +for p in sorted(power_dir.rglob("*")): + rel = p.relative_to(power_dir) + if is_allowed(rel, p.name, p.is_dir()): + continue + if p.name.startswith("."): + print(f" FAIL {rel}: hidden files make the power refuse to install") + bad = True + continue + hit = [d for d in DISALLOWED if d in p.name.lower()] + if hit: + print(f" FAIL {rel}: filename matches Kiro's disallowed pattern {hit[0]!r}") + bad = True + +# Only steering/ may be a directory, and only .md may live in it. Everything +# else at the top level is dropped on install even when it validates, so a file +# put there is invisible to the agent that is supposed to read it. +for p in sorted(power_dir.iterdir()): + if p.is_dir(): + if p.name != "steering": + print(f" FAIL {p.name}/: only steering/ is copied on install;" + " anything else is silently dropped") + bad = True + elif p.name not in ("POWER.md", "mcp.json"): + print(f" FAIL {p.name}: only POWER.md and mcp.json are copied on install") + bad = True + +steering = power_dir / "steering" +if not steering.is_dir(): + print(" FAIL no steering/ directory") + bad = True +else: + for p in sorted(steering.iterdir()): + if p.is_dir(): + print(f" FAIL steering/{p.name}/: the installer copies only files" + " directly under steering/, so a subdirectory never arrives") + bad = True + elif p.suffix != ".md": + print(f" FAIL steering/{p.name}: only .md is allowed under steering/") + bad = True + +if not bad: + n = len(list(steering.glob("*.md"))) + print(f" ok POWER.md, mcp.json and {n} steering file(s); nothing Kiro would" + " reject or silently drop") +sys.exit(1 if bad else 0) +PY + +echo +echo "== mcp.json ==" +python3 - "$POWER_DIR" <<'PY' || status=1 +import json +import pathlib +import sys + +p = pathlib.Path(sys.argv[1]) / "mcp.json" +if not p.exists(): + print(" skip the power ships no mcp.json") + sys.exit(0) +bad = False +try: + cfg = json.loads(p.read_text()) +except json.JSONDecodeError as e: + print(f" FAIL not valid JSON: {e}") + sys.exit(1) + +if set(cfg) != {"mcpServers"}: + print(f" FAIL top-level keys must be exactly ['mcpServers'], got {sorted(cfg)}" + " (display metadata belongs in POWER.md frontmatter)") + bad = True +else: + print(" ok mcpServers only, no display metadata") + +for srv, conf in cfg.get("mcpServers", {}).items(): + if "command" not in conf and "url" not in conf: + print(f" FAIL server {srv!r} has neither 'command' nor 'url'") + bad = True + continue + # Kiro deletes autoApprove/allowedTools from a power's mcp.json on load: a + # power cannot grant itself auto-approval. Shipping them is inert, and it + # invites POWER.md to promise a posture the runtime will not honour. + for dead in ("autoApprove", "allowedTools"): + if dead in conf: + print(f" FAIL {srv}: {dead!r} is deleted by Kiro on load; remove it" + " (approvals are the user's to give)") + bad = True + print(f" ok {srv}: {conf.get('command', conf.get('url'))}") +sys.exit(1 if bad else 0) +PY + +echo +echo "== Runbooks are routed from POWER.md ==" +# A runbook the agent cannot find is dead weight, and it fails invisibly: the +# model answers from reference material instead and nobody notices the procedure +# was skipped. +python3 - "$POWER_DIR" <<'PY' || status=1 +import pathlib +import sys + +power_dir = pathlib.Path(sys.argv[1]) +power_md = (power_dir / "POWER.md").read_text() +runbooks = sorted((power_dir / "steering").glob("runbook-*.md")) +if not runbooks: + print(" skip no runbooks") + sys.exit(0) +bad = False +for rb in runbooks: + if f"`{rb.name}`" not in power_md: + print(f" FAIL POWER.md does not route to `{rb.name}`, so the agent will" + " not find it") + bad = True +if not bad: + print(f" ok {len(runbooks)} runbook(s), each routed from POWER.md") +sys.exit(1 if bad else 0) +PY + +echo +echo "== Link integrity ==" +# troubleshooting.md routes by a symptom index of anchor links, so a broken +# anchor silently costs a reader the fix they came for. GitHub's slug is: +# lowercase, drop everything that is not a word char / space / hyphen, then +# replace EACH space with one hyphen -- runs are not collapsed, which is why an +# em-dash or an ellipsis in a heading yields a double hyphen. +python3 - "$POWER_DIR" <<'PY' || status=1 +import pathlib +import re +import sys + +power_dir = pathlib.Path(sys.argv[1]) + + +def slug(h): + h = h.strip().lower().replace("`", "") + h = re.sub(r"[^\w\s-]", "", h) + return h.replace(" ", "-") + + +files = [power_dir / "POWER.md", *sorted(power_dir.glob("steering/*.md"))] +readme = power_dir.parent / "README.md" +if readme.exists(): + files.append(readme) + +bad = False +anchors = rels = 0 +for f in files: + text = f.read_text() + heads = {slug(m.group(1)) for m in re.finditer(r"^#{1,6}\s+(.*)$", text, re.M)} + for m in re.finditer(r"\]\((#[^)]+|[A-Za-z0-9_./-]+\.md(?:#[^)]+)?)\)", text): + target = m.group(1) + line = text[: m.start()].count("\n") + 1 + if target.startswith("#"): + anchors += 1 + if target[1:] not in heads: + print(f" FAIL {f}:{line} dead anchor {target}") + bad = True + else: + rels += 1 + if not (f.parent / target.split("#", 1)[0]).exists(): + print(f" FAIL {f}:{line} link to missing file {target}") + bad = True +if not bad: + print(f" ok {anchors} anchor(s) and {rels} relative link(s) resolve" + f" across {len(files)} file(s)") +sys.exit(1 if bad else 0) +PY + +echo +echo "== Source citations resolve in this tree ==" +python3 - "$POWER_DIR" <<'PY' || status=1 +import pathlib +import re +import sys + +power_dir = pathlib.Path(sys.argv[1]) +root = pathlib.Path(".") +# platform.yaml and workshop.env are created by the deploy wizard, so they are +# cited legitimately without existing in a clean checkout. +USER_CREATED = {"platform.yaml", "workshop.env"} +# Paths this repository has retired. The power names them on purpose: someone on +# an older checkout will report a symptom the current tree cannot produce, and +# "that command was replaced, git pull" is the answer. Each entry is asserted to +# be *absent* below, so a resurrected path fails this check instead of hiding in +# an allowlist. +RETIRED = { + "scripts/test.py": "replaced by scripts/verify.py / deploy.sh verify", +} +pat = re.compile( + r"\b((?:scripts|stacks|config|agent-code|docs|tests|tools|infra_utils|dashboard)" + r"/[A-Za-z0-9_./-]+?\.(?:sh|py|ya?ml|md)" + r"|app\.py|requirements\.txt|platform\.yaml|workshop\.env)" + r"(?::(\d+)(?:[-–](\d+))?)?" +) +cites = {} +for f in [power_dir / "POWER.md", *sorted(power_dir.glob("steering/*.md"))]: + text = f.read_text() + for m in pat.finditer(text): + line = text[: m.start()].count("\n") + 1 + cites.setdefault((m.group(1), m.group(2), m.group(3)), []).append(f"{f}:{line}") + +bad = False +for (path, lo, hi), where in sorted(cites.items(), key=lambda kv: (kv[0][0], kv[0][1] or "")): + if path in USER_CREATED: + continue + target = root / path + if path in RETIRED: + if target.exists(): + print(f" FAIL {path} is listed as retired ({RETIRED[path]}) but exists" + f" (cited at {where[0]})") + bad = True + continue + if not target.exists(): + print(f" FAIL {path} does not exist in this repo (cited at {where[0]})") + bad = True + continue + if lo: + n = sum(1 for _ in target.open(errors="replace")) + if int(hi or lo) > n: + print(f" FAIL {path}:{lo}-{hi or lo} is past EOF ({n} lines)" + f" (cited at {where[0]})") + bad = True +if not bad: + n = len([k for k in cites if k[0] not in USER_CREATED and k[0] not in RETIRED]) + r = len([k for k in cites if k[0] in RETIRED]) + print(f" ok {n} source citation(s) resolve, all line ranges in bounds") + if r: + print(f" ok {r} reference(s) to retired path(s), still absent: " + f"{', '.join(sorted({k[0] for k in cites if k[0] in RETIRED}))}") +sys.exit(1 if bad else 0) +PY + +echo +echo "== Profile sequences match PROFILE_MODULES ==" +# The profile picker is where this power's value concentrates, and the same five +# sequences are restated across many files. Pin them to the source so an +# upstream reordering cannot leave a dozen copies quietly stale. +python3 - "$POWER_DIR" <<'PY' || status=1 +import pathlib +import re +import sys + +power_dir = pathlib.Path(sys.argv[1]) +truth = dict(re.findall(r'PROFILE_MODULES\[([a-z-]+)\]="([^"]+)"', + pathlib.Path("scripts/deploy.sh").read_text())) +if not truth: + print(" FAIL could not read PROFILE_MODULES from scripts/deploy.sh") + sys.exit(1) + +files = [power_dir / "POWER.md", *sorted(power_dir.glob("steering/*.md"))] +bad, checked = False, 0 +for f in files: + for i, line in enumerate(f.read_text().splitlines(), 1): + for prof, seq in truth.items(): + if f"`{prof}`" not in line: + continue + for s in re.findall(r"(?${}-]+))*)" +) +bad, checked = False, 0 +for f in files: + text = f.read_text() + for m in pat.finditer(text): + script = "scripts/deploy.sh" if m.group(1) == "deploy.sh" else m.group(1) + target = root / script + if not target.exists(): + continue + src = target.read_text() + for flag in re.findall(r"(? is the documentation convention +# but bash reads it as a redirect, so normalise it first -- otherwise every +# block with a placeholder is a false positive and the gate gets ignored. +blocks = 0 +for f in files: + text = f.read_text() + for m in re.finditer(r"```bash\n(.*?)```", text, re.S): + blocks += 1 + block = re.sub(r"<[A-Za-z0-9_ .|-]+>", "PLACEHOLDER", m.group(1)) + r = subprocess.run(["bash", "-n"], input=block, capture_output=True, text=True) + if r.returncode: + line = text[: m.start()].count("\n") + 1 + print(f" FAIL {f}:{line} bash syntax: {r.stderr.strip().splitlines()[-1]}") + bad = True +if not bad: + print(f" ok {blocks} bash block(s) parse") + +# 2. No bare `python`. `source .venv/bin/activate` does not survive between an +# agent's tool calls, so a bare `python scripts/...` fails with a +# ModuleNotFoundError that looks nothing like its real cause. This repo's own +# MODULE_VERIFY table uses .venv/bin/python; so should every command here. +hits = [] +for f in files: + for i, line in enumerate(f.read_text().splitlines(), 1): + if re.search(r"(?/dev/null); then + echo "$hits" | sed 's/^/ /' + fail "credential-shaped string" + else + ok "no credential-shaped strings" + fi + + # 111122223333 is the documentation placeholder CONTRIBUTING.md asks for. + if hits=$(grep -nIoE '\b[0-9]{12}\b' "${FILES[@]}" 2>/dev/null | grep -v '111122223333'); then + echo "$hits" | sed 's/^/ /' + fail "12-digit number that may be a real account id (use 111122223333)" + else + ok "no real-looking account ids" + fi + + # Placeholders in commands are fine; a concrete id from a real run is not. + RESOURCE_IDS='\b(eni|vpc|subnet|sg|ami|rtb|igw|nat|eipalloc)-[0-9a-f]{8,17}\b' + RESOURCE_IDS+='|\b[a-z]{2}-[a-z]+-[0-9]_[A-Za-z0-9]{9}\b' + RESOURCE_IDS+='|\bo-[a-z0-9]{10,32}\b' + if hits=$(grep -nIoE "$RESOURCE_IDS" "${FILES[@]}" 2>/dev/null | grep -v 'o-example123'); then + echo "$hits" | sed 's/^/ /' + fail "concrete resource id from a real account — use a placeholder" + else + ok "no concrete resource ids" + fi +fi + +echo +if (( status )); then + echo "FAILED" +else + echo "All checks passed." +fi +exit $status