diff --git a/.sqlx/query-3097d5ce169708bf80fddd1f466a2d6f6ca511533c6c3f278ecf8c08ad3b8e9e.json b/.sqlx/query-3097d5ce169708bf80fddd1f466a2d6f6ca511533c6c3f278ecf8c08ad3b8e9e.json new file mode 100644 index 00000000..4eca7fd6 --- /dev/null +++ b/.sqlx/query-3097d5ce169708bf80fddd1f466a2d6f6ca511533c6c3f278ecf8c08ad3b8e9e.json @@ -0,0 +1,331 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO upload_tokens (\n token_hash, org_id, identity_id, service_instance_id,\n service_key, action_key, request, credential_ref,\n declared_sha256, declared_size_bytes, declared_mime, declared_filename,\n max_bytes, filename_param, result_spec, expires_at\n )\n VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15,\n now() + make_interval(secs => $16))\n RETURNING id, token_hash, org_id, identity_id, service_instance_id,\n service_key, action_key, request, credential_ref,\n declared_sha256, declared_size_bytes, declared_mime, declared_filename,\n max_bytes, filename_param, result_spec, stored_media_path, stored_sha256,\n stored_size_bytes,\n stored_mime, stored_filename, completed_at,\n created_at, expires_at, consumed_at", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Uuid", + "origin": { + "Table": { + "table": "upload_tokens", + "name": "id" + } + } + }, + { + "ordinal": 1, + "name": "token_hash", + "type_info": "Bytea", + "origin": { + "Table": { + "table": "upload_tokens", + "name": "token_hash" + } + } + }, + { + "ordinal": 2, + "name": "org_id", + "type_info": "Uuid", + "origin": { + "Table": { + "table": "upload_tokens", + "name": "org_id" + } + } + }, + { + "ordinal": 3, + "name": "identity_id", + "type_info": "Uuid", + "origin": { + "Table": { + "table": "upload_tokens", + "name": "identity_id" + } + } + }, + { + "ordinal": 4, + "name": "service_instance_id", + "type_info": "Uuid", + "origin": { + "Table": { + "table": "upload_tokens", + "name": "service_instance_id" + } + } + }, + { + "ordinal": 5, + "name": "service_key", + "type_info": "Text", + "origin": { + "Table": { + "table": "upload_tokens", + "name": "service_key" + } + } + }, + { + "ordinal": 6, + "name": "action_key", + "type_info": "Text", + "origin": { + "Table": { + "table": "upload_tokens", + "name": "action_key" + } + } + }, + { + "ordinal": 7, + "name": "request", + "type_info": "Jsonb", + "origin": { + "Table": { + "table": "upload_tokens", + "name": "request" + } + } + }, + { + "ordinal": 8, + "name": "credential_ref", + "type_info": "Jsonb", + "origin": { + "Table": { + "table": "upload_tokens", + "name": "credential_ref" + } + } + }, + { + "ordinal": 9, + "name": "declared_sha256", + "type_info": "Text", + "origin": { + "Table": { + "table": "upload_tokens", + "name": "declared_sha256" + } + } + }, + { + "ordinal": 10, + "name": "declared_size_bytes", + "type_info": "Int8", + "origin": { + "Table": { + "table": "upload_tokens", + "name": "declared_size_bytes" + } + } + }, + { + "ordinal": 11, + "name": "declared_mime", + "type_info": "Text", + "origin": { + "Table": { + "table": "upload_tokens", + "name": "declared_mime" + } + } + }, + { + "ordinal": 12, + "name": "declared_filename", + "type_info": "Text", + "origin": { + "Table": { + "table": "upload_tokens", + "name": "declared_filename" + } + } + }, + { + "ordinal": 13, + "name": "max_bytes", + "type_info": "Int8", + "origin": { + "Table": { + "table": "upload_tokens", + "name": "max_bytes" + } + } + }, + { + "ordinal": 14, + "name": "filename_param", + "type_info": "Text", + "origin": { + "Table": { + "table": "upload_tokens", + "name": "filename_param" + } + } + }, + { + "ordinal": 15, + "name": "result_spec", + "type_info": "Jsonb", + "origin": { + "Table": { + "table": "upload_tokens", + "name": "result_spec" + } + } + }, + { + "ordinal": 16, + "name": "stored_media_path", + "type_info": "Text", + "origin": { + "Table": { + "table": "upload_tokens", + "name": "stored_media_path" + } + } + }, + { + "ordinal": 17, + "name": "stored_sha256", + "type_info": "Text", + "origin": { + "Table": { + "table": "upload_tokens", + "name": "stored_sha256" + } + } + }, + { + "ordinal": 18, + "name": "stored_size_bytes", + "type_info": "Int8", + "origin": { + "Table": { + "table": "upload_tokens", + "name": "stored_size_bytes" + } + } + }, + { + "ordinal": 19, + "name": "stored_mime", + "type_info": "Text", + "origin": { + "Table": { + "table": "upload_tokens", + "name": "stored_mime" + } + } + }, + { + "ordinal": 20, + "name": "stored_filename", + "type_info": "Text", + "origin": { + "Table": { + "table": "upload_tokens", + "name": "stored_filename" + } + } + }, + { + "ordinal": 21, + "name": "completed_at", + "type_info": "Timestamptz", + "origin": { + "Table": { + "table": "upload_tokens", + "name": "completed_at" + } + } + }, + { + "ordinal": 22, + "name": "created_at", + "type_info": "Timestamptz", + "origin": { + "Table": { + "table": "upload_tokens", + "name": "created_at" + } + } + }, + { + "ordinal": 23, + "name": "expires_at", + "type_info": "Timestamptz", + "origin": { + "Table": { + "table": "upload_tokens", + "name": "expires_at" + } + } + }, + { + "ordinal": 24, + "name": "consumed_at", + "type_info": "Timestamptz", + "origin": { + "Table": { + "table": "upload_tokens", + "name": "consumed_at" + } + } + } + ], + "parameters": { + "Left": [ + "Bytea", + "Uuid", + "Uuid", + "Uuid", + "Text", + "Text", + "Jsonb", + "Jsonb", + "Text", + "Int8", + "Text", + "Text", + "Int8", + "Text", + "Jsonb", + "Float8" + ] + }, + "nullable": [ + false, + false, + false, + false, + true, + true, + true, + false, + false, + true, + true, + true, + true, + false, + true, + true, + true, + true, + true, + true, + true, + true, + false, + false, + true + ] + }, + "hash": "3097d5ce169708bf80fddd1f466a2d6f6ca511533c6c3f278ecf8c08ad3b8e9e" +} diff --git a/.sqlx/query-40a800fc3ba2e0365df92e2bf82221d56594ea8a556547a10c32e9b34733b712.json b/.sqlx/query-40a800fc3ba2e0365df92e2bf82221d56594ea8a556547a10c32e9b34733b712.json new file mode 100644 index 00000000..0967501e --- /dev/null +++ b/.sqlx/query-40a800fc3ba2e0365df92e2bf82221d56594ea8a556547a10c32e9b34733b712.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO media_descriptors (\n org_id, service_instance_id, service_key, media_path,\n sha256, mime, size_bytes, filename, source\n )\n VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)\n ON CONFLICT (org_id, service_instance_id, media_path) DO UPDATE\n SET sha256 = COALESCE(EXCLUDED.sha256, media_descriptors.sha256),\n mime = COALESCE(EXCLUDED.mime, media_descriptors.mime),\n size_bytes = COALESCE(EXCLUDED.size_bytes, media_descriptors.size_bytes),\n filename = COALESCE(EXCLUDED.filename, media_descriptors.filename),\n service_key = COALESCE(EXCLUDED.service_key, media_descriptors.service_key),\n last_seen_at = now()", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Uuid", + "Uuid", + "Text", + "Text", + "Text", + "Text", + "Int8", + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "40a800fc3ba2e0365df92e2bf82221d56594ea8a556547a10c32e9b34733b712" +} diff --git a/.sqlx/query-59036059e060c78c025778ff57e335b4e7a41ca9fc4ebadb01324b2346026d8e.json b/.sqlx/query-59036059e060c78c025778ff57e335b4e7a41ca9fc4ebadb01324b2346026d8e.json new file mode 100644 index 00000000..0940fc13 --- /dev/null +++ b/.sqlx/query-59036059e060c78c025778ff57e335b4e7a41ca9fc4ebadb01324b2346026d8e.json @@ -0,0 +1,162 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT id, org_id, service_instance_id, service_key, media_path,\n sha256, mime, size_bytes, filename, source, first_seen_at, last_seen_at\n FROM media_descriptors\n WHERE org_id = $1\n AND service_instance_id IS NOT DISTINCT FROM $2\n AND media_path = $3", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Uuid", + "origin": { + "Table": { + "table": "media_descriptors", + "name": "id" + } + } + }, + { + "ordinal": 1, + "name": "org_id", + "type_info": "Uuid", + "origin": { + "Table": { + "table": "media_descriptors", + "name": "org_id" + } + } + }, + { + "ordinal": 2, + "name": "service_instance_id", + "type_info": "Uuid", + "origin": { + "Table": { + "table": "media_descriptors", + "name": "service_instance_id" + } + } + }, + { + "ordinal": 3, + "name": "service_key", + "type_info": "Text", + "origin": { + "Table": { + "table": "media_descriptors", + "name": "service_key" + } + } + }, + { + "ordinal": 4, + "name": "media_path", + "type_info": "Text", + "origin": { + "Table": { + "table": "media_descriptors", + "name": "media_path" + } + } + }, + { + "ordinal": 5, + "name": "sha256", + "type_info": "Text", + "origin": { + "Table": { + "table": "media_descriptors", + "name": "sha256" + } + } + }, + { + "ordinal": 6, + "name": "mime", + "type_info": "Text", + "origin": { + "Table": { + "table": "media_descriptors", + "name": "mime" + } + } + }, + { + "ordinal": 7, + "name": "size_bytes", + "type_info": "Int8", + "origin": { + "Table": { + "table": "media_descriptors", + "name": "size_bytes" + } + } + }, + { + "ordinal": 8, + "name": "filename", + "type_info": "Text", + "origin": { + "Table": { + "table": "media_descriptors", + "name": "filename" + } + } + }, + { + "ordinal": 9, + "name": "source", + "type_info": "Text", + "origin": { + "Table": { + "table": "media_descriptors", + "name": "source" + } + } + }, + { + "ordinal": 10, + "name": "first_seen_at", + "type_info": "Timestamptz", + "origin": { + "Table": { + "table": "media_descriptors", + "name": "first_seen_at" + } + } + }, + { + "ordinal": 11, + "name": "last_seen_at", + "type_info": "Timestamptz", + "origin": { + "Table": { + "table": "media_descriptors", + "name": "last_seen_at" + } + } + } + ], + "parameters": { + "Left": [ + "Uuid", + "Uuid", + "Text" + ] + }, + "nullable": [ + false, + false, + true, + true, + false, + true, + true, + true, + true, + false, + false, + false + ] + }, + "hash": "59036059e060c78c025778ff57e335b4e7a41ca9fc4ebadb01324b2346026d8e" +} diff --git a/.sqlx/query-611166c422a6f9de16bf29f3b661a283fbb0bb2674818732405de934d52552df.json b/.sqlx/query-611166c422a6f9de16bf29f3b661a283fbb0bb2674818732405de934d52552df.json new file mode 100644 index 00000000..f87c1126 --- /dev/null +++ b/.sqlx/query-611166c422a6f9de16bf29f3b661a283fbb0bb2674818732405de934d52552df.json @@ -0,0 +1,12 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM upload_tokens WHERE expires_at < now()", + "describe": { + "columns": [], + "parameters": { + "Left": [] + }, + "nullable": [] + }, + "hash": "611166c422a6f9de16bf29f3b661a283fbb0bb2674818732405de934d52552df" +} diff --git a/.sqlx/query-c5280f756bf2cc59edb8231c1dc05d13565a45d609967eaed2f155f3539ba88c.json b/.sqlx/query-c5280f756bf2cc59edb8231c1dc05d13565a45d609967eaed2f155f3539ba88c.json new file mode 100644 index 00000000..18375aa2 --- /dev/null +++ b/.sqlx/query-c5280f756bf2cc59edb8231c1dc05d13565a45d609967eaed2f155f3539ba88c.json @@ -0,0 +1,316 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE upload_tokens\n SET consumed_at = now()\n WHERE token_hash = $1 AND expires_at > now() AND consumed_at IS NULL\n RETURNING id, token_hash, org_id, identity_id, service_instance_id,\n service_key, action_key, request, credential_ref,\n declared_sha256, declared_size_bytes, declared_mime, declared_filename,\n max_bytes, filename_param, result_spec, stored_media_path, stored_sha256,\n stored_size_bytes,\n stored_mime, stored_filename, completed_at,\n created_at, expires_at, consumed_at", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Uuid", + "origin": { + "Table": { + "table": "upload_tokens", + "name": "id" + } + } + }, + { + "ordinal": 1, + "name": "token_hash", + "type_info": "Bytea", + "origin": { + "Table": { + "table": "upload_tokens", + "name": "token_hash" + } + } + }, + { + "ordinal": 2, + "name": "org_id", + "type_info": "Uuid", + "origin": { + "Table": { + "table": "upload_tokens", + "name": "org_id" + } + } + }, + { + "ordinal": 3, + "name": "identity_id", + "type_info": "Uuid", + "origin": { + "Table": { + "table": "upload_tokens", + "name": "identity_id" + } + } + }, + { + "ordinal": 4, + "name": "service_instance_id", + "type_info": "Uuid", + "origin": { + "Table": { + "table": "upload_tokens", + "name": "service_instance_id" + } + } + }, + { + "ordinal": 5, + "name": "service_key", + "type_info": "Text", + "origin": { + "Table": { + "table": "upload_tokens", + "name": "service_key" + } + } + }, + { + "ordinal": 6, + "name": "action_key", + "type_info": "Text", + "origin": { + "Table": { + "table": "upload_tokens", + "name": "action_key" + } + } + }, + { + "ordinal": 7, + "name": "request", + "type_info": "Jsonb", + "origin": { + "Table": { + "table": "upload_tokens", + "name": "request" + } + } + }, + { + "ordinal": 8, + "name": "credential_ref", + "type_info": "Jsonb", + "origin": { + "Table": { + "table": "upload_tokens", + "name": "credential_ref" + } + } + }, + { + "ordinal": 9, + "name": "declared_sha256", + "type_info": "Text", + "origin": { + "Table": { + "table": "upload_tokens", + "name": "declared_sha256" + } + } + }, + { + "ordinal": 10, + "name": "declared_size_bytes", + "type_info": "Int8", + "origin": { + "Table": { + "table": "upload_tokens", + "name": "declared_size_bytes" + } + } + }, + { + "ordinal": 11, + "name": "declared_mime", + "type_info": "Text", + "origin": { + "Table": { + "table": "upload_tokens", + "name": "declared_mime" + } + } + }, + { + "ordinal": 12, + "name": "declared_filename", + "type_info": "Text", + "origin": { + "Table": { + "table": "upload_tokens", + "name": "declared_filename" + } + } + }, + { + "ordinal": 13, + "name": "max_bytes", + "type_info": "Int8", + "origin": { + "Table": { + "table": "upload_tokens", + "name": "max_bytes" + } + } + }, + { + "ordinal": 14, + "name": "filename_param", + "type_info": "Text", + "origin": { + "Table": { + "table": "upload_tokens", + "name": "filename_param" + } + } + }, + { + "ordinal": 15, + "name": "result_spec", + "type_info": "Jsonb", + "origin": { + "Table": { + "table": "upload_tokens", + "name": "result_spec" + } + } + }, + { + "ordinal": 16, + "name": "stored_media_path", + "type_info": "Text", + "origin": { + "Table": { + "table": "upload_tokens", + "name": "stored_media_path" + } + } + }, + { + "ordinal": 17, + "name": "stored_sha256", + "type_info": "Text", + "origin": { + "Table": { + "table": "upload_tokens", + "name": "stored_sha256" + } + } + }, + { + "ordinal": 18, + "name": "stored_size_bytes", + "type_info": "Int8", + "origin": { + "Table": { + "table": "upload_tokens", + "name": "stored_size_bytes" + } + } + }, + { + "ordinal": 19, + "name": "stored_mime", + "type_info": "Text", + "origin": { + "Table": { + "table": "upload_tokens", + "name": "stored_mime" + } + } + }, + { + "ordinal": 20, + "name": "stored_filename", + "type_info": "Text", + "origin": { + "Table": { + "table": "upload_tokens", + "name": "stored_filename" + } + } + }, + { + "ordinal": 21, + "name": "completed_at", + "type_info": "Timestamptz", + "origin": { + "Table": { + "table": "upload_tokens", + "name": "completed_at" + } + } + }, + { + "ordinal": 22, + "name": "created_at", + "type_info": "Timestamptz", + "origin": { + "Table": { + "table": "upload_tokens", + "name": "created_at" + } + } + }, + { + "ordinal": 23, + "name": "expires_at", + "type_info": "Timestamptz", + "origin": { + "Table": { + "table": "upload_tokens", + "name": "expires_at" + } + } + }, + { + "ordinal": 24, + "name": "consumed_at", + "type_info": "Timestamptz", + "origin": { + "Table": { + "table": "upload_tokens", + "name": "consumed_at" + } + } + } + ], + "parameters": { + "Left": [ + "Bytea" + ] + }, + "nullable": [ + false, + false, + false, + false, + true, + true, + true, + false, + false, + true, + true, + true, + true, + false, + true, + true, + true, + true, + true, + true, + true, + true, + false, + false, + true + ] + }, + "hash": "c5280f756bf2cc59edb8231c1dc05d13565a45d609967eaed2f155f3539ba88c" +} diff --git a/.sqlx/query-e823441feac386d76bab49e867a95880e0c1bd902d6b334a5195c4ed50da80bc.json b/.sqlx/query-e823441feac386d76bab49e867a95880e0c1bd902d6b334a5195c4ed50da80bc.json new file mode 100644 index 00000000..fad61e9e --- /dev/null +++ b/.sqlx/query-e823441feac386d76bab49e867a95880e0c1bd902d6b334a5195c4ed50da80bc.json @@ -0,0 +1,12 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE upload_tokens SET expires_at = now() - interval '1 hour'", + "describe": { + "columns": [], + "parameters": { + "Left": [] + }, + "nullable": [] + }, + "hash": "e823441feac386d76bab49e867a95880e0c1bd902d6b334a5195c4ed50da80bc" +} diff --git a/.sqlx/query-ee2d1755b5979138ad7c0fe70417ee5c7a6634032ecb2872cd817f1015519e74.json b/.sqlx/query-ee2d1755b5979138ad7c0fe70417ee5c7a6634032ecb2872cd817f1015519e74.json new file mode 100644 index 00000000..7ca61852 --- /dev/null +++ b/.sqlx/query-ee2d1755b5979138ad7c0fe70417ee5c7a6634032ecb2872cd817f1015519e74.json @@ -0,0 +1,33 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE upload_tokens\n SET completed_at = now(), stored_media_path = $2, stored_sha256 = $3,\n stored_size_bytes = $4, stored_mime = $5, stored_filename = $6\n WHERE id = $1 AND completed_at IS NULL\n RETURNING id", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Uuid", + "origin": { + "Table": { + "table": "upload_tokens", + "name": "id" + } + } + } + ], + "parameters": { + "Left": [ + "Uuid", + "Text", + "Text", + "Int8", + "Text", + "Text" + ] + }, + "nullable": [ + false + ] + }, + "hash": "ee2d1755b5979138ad7c0fe70417ee5c7a6634032ecb2872cd817f1015519e74" +} diff --git a/DECISIONS.md b/DECISIONS.md index 74889695..7c23d7bd 100644 --- a/DECISIONS.md +++ b/DECISIONS.md @@ -616,3 +616,33 @@ The gateway never follows a page. `gmail.yaml` is the only shipped template anno **A malformed declaration is dropped, not fatal** — D67's leniency, and D73's before it. An action whose `pagination` block is misspelled goes back to being exactly as unpaged as it was, with an authoring error the template author can see. What is *not* lenient is a `page_size.param` naming a parameter the action does not declare: that is an error, because it reads as a bound and applies none, which is the precise silent unboundedness the key exists to end. **One exemplar, not a sweep.** `gmail.yaml` was picked over the more urgent unbounded cases because it demonstrates both halves at once — `maxResults` declares no default, and `nextPageToken` sits after the `messages` array, the exact payload D74's key cap was ranked for. Annotating twenty templates in the same change would have made the mechanism hard to review and the template judgements impossible to argue with individually. The follow-up closes with a corpus gate shaped like `shipped_mutating_actions_declare_disclose`. + +## D-NEXT: Bytes move into a service the same way they move out — a capability the gateway proxies, never the service's own credential + +**Date**: 2026-08-31 +**Decision**: An MCP action may declare `x-overslash-upload` (bare alias `upload:`), naming a plain-HTTP byte route on the service instance's own origin: `{path, method, filename_param, auth, max_bytes, result}`. An action carrying it is **served by the gateway** — the two dispatch sites intercept it before `mcp_caller::invoke`, so the upstream never sees a tool by that name — and what it returns is a one-shot `upload_url` on `POST|PUT /v1/uploads/{token}`. Redeeming that URL streams the request body through to the byte route with the instance's bearer re-resolved from the vault at that moment. `upload_tokens` is a **separate table** from `download_tokens` and its claim consumes. + +The organizing invariant: **everything the reviewer approved is fixed at mint time; the anonymous redemption leg contributes only bytes.** The filename, the route, the ceiling and the declared content all come off the token; the redeemer contributes bytes and at most a `Content-Type` hint. + +Optional `sha256` and `size_bytes` arguments bind a capability to one file. Size is enforced *during* the stream and cuts the transfer; sha256 is verified after the last byte and, on a mismatch, the redemption answers 422 and **withholds the upstream's reference** so nothing downstream can name those bytes. + +A `media_descriptors` ledger records every descriptor the gateway sees — in *both* directions — and `x-overslash-resolve` grows a non-tool `source: media` target that reads it, so a `send_file` approval's primary row reads `invoice.pdf (application/pdf, 240912 bytes)` instead of `/media/<64 hex>`. + +**Rationale**: `services/whatsapp.yaml` said it outright: originating bytes "is out-of-band work for whoever operates the container — not something an agent can do through this service." `send_file` therefore only ever *forwarded* a `media_path` some earlier `download_media` produced. An agent could re-send a file someone sent it and could not send a file it made. + +**The direct route — hand the agent the container's URL and token — is the one thing the vault exists to prevent.** There is no narrower credential to mint: the container has a single static `AUTH_TOKEN` with no scopes, audience or expiry, and it authorizes `/mcp`, which is every WhatsApp write. Proxying also buys what the direct route structurally cannot: the gateway sees the descriptor, so it can say what a later approval is actually about. + +**The token layer is where the download mirror stops.** `download_tokens` is deliberately multi-use so a dropped transfer can resume with `curl -C -`, and its `claim` bumps a counter unconditionally. Redeeming a download twice re-fetches the same bytes; redeeming an upload twice stores two *different* payloads under one authorization, so "what the reviewer approved" would stop having an answer. Bolting a consume onto a table whose contract is the opposite invites a later change to the shared claim to quietly make uploads replayable — hence a second table, whose declared/stored column split also makes a started-and-abandoned push legible rather than inferable. + +**There are two dispatch sites, and the second is the one that matters.** `upload_media` is `risk: write`, so for any gated agent the *first* call is replayed from a stored payload after approval, through `stored_call::run_mcp` — and replay resolves nothing: it holds a URL and a tool name, not an action key, so it cannot look the declaration back up. Intercepting only in `call_mcp::dispatch` would work perfectly until a human said yes, at which point the replay would dispatch `tools/call {name: "upload_media"}` and get `-32603 Unknown tool`. The spec therefore rides on `StoredMcpCall`, the same answer D56 gave the timeout cascade and D75 gave pagination, for the same reason. + +**Declaring it as a tool, rather than a platform action, is what buys the approval.** A platform action would give the same in-process dispatch and lose the whole point: `ext::READS` records `Ext::Disclose` as *not* read at `Pos::PlatformAction`, and the `shipped_mutating_actions_declare_disclose` gate skips `Runtime::Platform` for that reason, so the approval would render the bare `{runtime, action, params, service}` projection. The cost is honest and small: the container-catalog drift guard learns a `GATEWAY_SYNTHESIZED` list, teaching it that "exposed" and "served as a tool" are now different sets. + +**A hash can be detected, not prevented, and the design says so rather than implying otherwise.** It is not known until the last byte, which is after the upstream has them. Withholding the reference is what makes detecting it worth anything — nothing downstream can name bytes it was never told the name of — but the bytes do linger upstream as an unreferenced orphan, and an operator who believes the check is preventive will not think about that. Size is different and *is* prevented: the meter cuts the transfer mid-stream, and the verdict is read from the meter rather than matched against the transport error's text, because that text arrives as a generic reqwest failure an unknown number of `source()` hops down. + +**The ledger write is deliberately not gated on `deliver: "url"`.** The template's own guidance tells agents to call `download_media` *without* it when they want the reference to forward — which is the common case — so recording only on the deferred path would have left exactly the calls that produce references to later approvals unrecorded, and every one of those approvals showing a bare hash. Lifting the jq evaluation above the mint gate costs one already-paid batch on a path that was not paying it. + +**`source: media` rides inside `x-overslash-resolve` rather than growing the extension vocabulary.** `ParamResolver` already models mutually-exclusive targets (`get` for HTTP, `tool` for MCP) behind `has_one_target`, and `.resolved.` already reaches the approval projection, the audit `disclosed` block and the dashboard. A new projection key would have meant a second enrichment path with its own template vocabulary, null for every service but one; a heuristic scan for sha256-shaped arguments would have meant enriching a field nobody declared, and the entire value of the enrichment is that a reviewer can trust it. `scope:` is *refused* alongside `source:` — `scope` canonicalizes the permission key, and collapsing every file sharing a filename onto one grant is a silent authorization widening that costs three lines to make unrepresentable. + +**Bytes the gateway never handled stay un-enriched, and that is the correct outcome.** A `media_path` pushed to the container out of band is not in the ledger and the File row falls back to the raw path. Probing the upstream at approval time would put a network round-trip and a credential use inside the approval path for a cosmetic gain. The fallback is lossless — the reviewer still sees exactly the string the call will send — so a miss is less helpful, never misleading. + diff --git a/SCHEMA.sql b/SCHEMA.sql index d6e6719f..d113a098 100644 --- a/SCHEMA.sql +++ b/SCHEMA.sql @@ -2,10 +2,10 @@ -- PostgreSQL database dump -- -\restrict Wa9cVbD20aiSD1mg8hdX4ucJSCAI1Zo8GH1ywdS2KyvkPycAg0afifQTGjdzEay +\restrict ITpRAwCBQB1Eujdumlb2lXtfTcabr3ewsacbHizs9jNwJBpKN4XyV8oROCASLy3 --- Dumped from database version 16.13 (Debian 16.13-1.pgdg12+1) --- Dumped by pg_dump version 16.14 (Ubuntu 16.14-0ubuntu0.24.04.1) +-- Dumped from database version 16.14 (Debian 16.14-1.pgdg12+1) +-- Dumped by pg_dump version 16.15 (Ubuntu 16.15-0ubuntu0.24.04.1) SET statement_timeout = 0; SET lock_timeout = 0; @@ -32,10 +32,6 @@ CREATE SCHEMA public; COMMENT ON SCHEMA public IS 'standard public schema'; -SET default_tablespace = ''; - -SET default_table_access_method = heap; - -- -- Name: events_notify(); Type: FUNCTION; Schema: public; Owner: - -- @@ -50,6 +46,10 @@ END; $$; +SET default_tablespace = ''; + +SET default_table_access_method = heap; + -- -- Name: api_keys; Type: TABLE; Schema: public; Owner: - -- @@ -99,6 +99,13 @@ CREATE TABLE public.approvals ( ); +-- +-- Name: COLUMN approvals.tags; Type: COMMENT; Schema: public; Owner: - +-- + +COMMENT ON COLUMN public.approvals.tags IS 'System-derived `namespace:value` metadata tags describing the gated call (sql:*, table:*, service:*, host:*, risk:*). Never caller-supplied.'; + + -- -- Name: audit_log; Type: TABLE; Schema: public; Owner: - -- @@ -121,18 +128,25 @@ CREATE TABLE public.audit_log ( ); +-- +-- Name: COLUMN audit_log.tags; Type: COMMENT; Schema: public; Owner: - +-- + +COMMENT ON COLUMN public.audit_log.tags IS 'System-derived `namespace:value` metadata tags. Searchable via GET /v1/audit?tag=. Populated on action/approval events; other events carry an empty array.'; + + -- -- Name: COLUMN audit_log.actor_name; Type: COMMENT; Schema: public; Owner: - -- -COMMENT ON COLUMN public.audit_log.actor_name IS 'Name of identity_id as of write time. Historical by design (D56) — the row records the name the actor had when they acted, not their current one.'; +COMMENT ON COLUMN public.audit_log.actor_name IS 'Name of identity_id as of write time. Historical by design (D59) — the row records the name the actor had when they acted, not their current one.'; -- -- Name: COLUMN audit_log.owner_user_name; Type: COMMENT; Schema: public; Owner: - -- -COMMENT ON COLUMN public.audit_log.owner_user_name IS 'Name of the root user of the actor''s identity chain, as of write time. Root, not direct parent: a sub-agent resolves to the human at the top, matching the audit table''s User column.'; +COMMENT ON COLUMN public.audit_log.owner_user_name IS 'Name of the owning user (identities.owner_id, a flattened pointer to the root user) as of write time, or the actor''s own name when the actor is a user. Historical by design (D59).'; -- @@ -162,10 +176,18 @@ CREATE TABLE public.byoc_credentials ( encrypted_client_id bytea NOT NULL, encrypted_client_secret bytea NOT NULL, created_at timestamp with time zone DEFAULT now() NOT NULL, - updated_at timestamp with time zone DEFAULT now() NOT NULL + updated_at timestamp with time zone DEFAULT now() NOT NULL, + metadata jsonb DEFAULT '{}'::jsonb NOT NULL ); +-- +-- Name: COLUMN byoc_credentials.metadata; Type: COMMENT; Schema: public; Owner: - +-- + +COMMENT ON COLUMN public.byoc_credentials.metadata IS 'Opaque caller-supplied key/value claim (provenance tag). Echoed verbatim; cleared/rewritten whenever the encrypted client pair is replaced.'; + + -- -- Name: call_results; Type: TABLE; Schema: public; Owner: - -- @@ -225,10 +247,25 @@ CREATE TABLE public.connections ( created_at timestamp with time zone DEFAULT now() NOT NULL, updated_at timestamp with time zone DEFAULT now() NOT NULL, keep boolean DEFAULT false NOT NULL, + reauth_required boolean DEFAULT false NOT NULL, account_picture text ); +-- +-- Name: COLUMN connections.keep; Type: COMMENT; Schema: public; Owner: - +-- + +COMMENT ON COLUMN public.connections.keep IS 'When true, this connection is never auto-deleted by service deletion, even when no service references it.'; + + +-- +-- Name: COLUMN connections.reauth_required; Type: COMMENT; Schema: public; Owner: - +-- + +COMMENT ON COLUMN public.connections.reauth_required IS 'When true, the connection must be re-authorized before use (e.g. its pinned BYOC client was replaced). Cleared when fresh tokens are written.'; + + -- -- Name: download_tokens; Type: TABLE; Schema: public; Owner: - -- @@ -262,6 +299,27 @@ CREATE TABLE public.download_tokens ( COMMENT ON TABLE public.download_tokens IS 'Capability tokens for deferred (out-of-band) byte delivery. Minted by POST /v1/actions/call with deliver:"url", redeemed by GET /v1/downloads/{token}.'; +-- +-- Name: COLUMN download_tokens.token_hash; Type: COMMENT; Schema: public; Owner: - +-- + +COMMENT ON COLUMN public.download_tokens.token_hash IS 'sha256 of the raw token; the raw value exists only in the minted URL.'; + + +-- +-- Name: COLUMN download_tokens.request; Type: COMMENT; Schema: public; Owner: - +-- + +COMMENT ON COLUMN public.download_tokens.request IS 'Replayable upstream request {method,url,headers,body}. Names secrets rather than carrying them; inline credential headers are rejected at mint time.'; + + +-- +-- Name: COLUMN download_tokens.credential_ref; Type: COMMENT; Schema: public; Owner: - +-- + +COMMENT ON COLUMN public.download_tokens.credential_ref IS 'How to re-resolve the upstream credential at fetch time. Never the credential itself.'; + + -- -- Name: COLUMN download_tokens.call_result_id; Type: COMMENT; Schema: public; Owner: - -- @@ -398,6 +456,41 @@ CREATE TABLE public.executions ( ); +-- +-- Name: COLUMN executions.tags; Type: COMMENT; Schema: public; Owner: - +-- + +COMMENT ON COLUMN public.executions.tags IS 'Copied verbatim from the originating approval at insert time — an execution can never disagree with what its approver saw.'; + + +-- +-- Name: COLUMN executions.request; Type: COMMENT; Schema: public; Owner: - +-- + +COMMENT ON COLUMN public.executions.request IS 'Credential-free stored call payload, same shape as approvals.replay_payload. NOT NULL marks this row as worker-run (async).'; + + +-- +-- Name: COLUMN executions.lease_expires_at; Type: COMMENT; Schema: public; Owner: - +-- + +COMMENT ON COLUMN public.executions.lease_expires_at IS 'While status=executing, the instant after which the claiming worker is presumed dead and the row may be reclaimed. Renewed by heartbeat.'; + + +-- +-- Name: COLUMN executions.attempts; Type: COMMENT; Schema: public; Owner: - +-- + +COMMENT ON COLUMN public.executions.attempts IS 'Attempts that ended by losing their lease. Incremented only by the reclaim sweep, never by the claim.'; + + +-- +-- Name: COLUMN executions.cancel_requested; Type: COMMENT; Schema: public; Owner: - +-- + +COMMENT ON COLUMN public.executions.cancel_requested IS 'Cooperative cancel. Stops Overslash waiting on the upstream; does not cancel the upstream operation itself.'; + + -- -- Name: group_grants; Type: TABLE; Schema: public; Owner: - -- @@ -420,7 +513,7 @@ CREATE TABLE public.group_grants ( -- Name: COLUMN group_grants.auto_approve_reads; Type: COMMENT; Schema: public; Owner: - -- -COMMENT ON COLUMN public.group_grants.auto_approve_reads IS 'DEPRECATED - derived mirror of (auto_approve_level <> ''none''). Read auto_approve_level instead; this column is dropped once the API alias is removed.'; +COMMENT ON COLUMN public.group_grants.auto_approve_reads IS 'DEPRECATED — derived mirror of (auto_approve_level <> ''none''). Read auto_approve_level instead; this column is dropped once the API alias is removed.'; -- @@ -490,6 +583,21 @@ CREATE TABLE public.identity_groups ( ); +-- +-- Name: magic_link_tokens; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.magic_link_tokens ( + id uuid DEFAULT gen_random_uuid() NOT NULL, + token_hash bytea NOT NULL, + email text NOT NULL, + next_path text, + created_at timestamp with time zone DEFAULT now() NOT NULL, + expires_at timestamp with time zone NOT NULL, + redeemed_at timestamp with time zone +); + + -- -- Name: mcp_client_agent_bindings; Type: TABLE; Schema: public; Owner: - -- @@ -579,6 +687,34 @@ CREATE TABLE public.mcp_upstream_tokens ( ); +-- +-- Name: media_descriptors; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.media_descriptors ( + id uuid DEFAULT gen_random_uuid() NOT NULL, + org_id uuid NOT NULL, + service_instance_id uuid, + service_key text, + media_path text NOT NULL, + sha256 text, + mime text, + size_bytes bigint, + filename text, + source text NOT NULL, + first_seen_at timestamp with time zone DEFAULT now() NOT NULL, + last_seen_at timestamp with time zone DEFAULT now() NOT NULL, + CONSTRAINT media_descriptors_source_check CHECK ((source = ANY (ARRAY['download'::text, 'upload'::text]))) +); + + +-- +-- Name: TABLE media_descriptors; Type: COMMENT; Schema: public; Owner: - +-- + +COMMENT ON TABLE public.media_descriptors IS 'What the gateway recorded about bytes it moved, so an approval that references them can describe them instead of showing a bare content hash. Best-effort: bytes that never passed through the gateway are simply absent.'; + + -- -- Name: oauth_connection_flows; Type: TABLE; Schema: public; Owner: - -- @@ -600,7 +736,8 @@ CREATE TABLE public.oauth_connection_flows ( created_user_agent text, return_url text, upgrade_connection_id uuid, - service_instance_id uuid + service_instance_id uuid, + pin_service_instance_ids uuid[] DEFAULT '{}'::uuid[] NOT NULL ); @@ -643,6 +780,13 @@ CREATE TABLE public.oauth_mcp_clients ( ); +-- +-- Name: COLUMN oauth_mcp_clients.org_id; Type: COMMENT; Schema: public; Owner: - +-- + +COMMENT ON COLUMN public.oauth_mcp_clients.org_id IS 'Org this DCR client is locked to (stamped from the subdomain at registration). NULL = root/multi-org: usable on any subdomain, absent from any org''s admin MCP-Clients list.'; + + -- -- Name: oauth_preview_origins; Type: TABLE; Schema: public; Owner: - -- @@ -736,7 +880,6 @@ CREATE TABLE public.orgs ( subagent_idle_timeout_secs integer DEFAULT 14400 NOT NULL, subagent_archive_retention_days integer DEFAULT 30 NOT NULL, approval_auto_bubble_secs integer DEFAULT 300 NOT NULL, - allow_user_templates boolean DEFAULT false NOT NULL, global_templates_enabled boolean DEFAULT true NOT NULL, allow_unsigned_secret_provide boolean DEFAULT true NOT NULL, is_personal boolean DEFAULT false NOT NULL, @@ -746,16 +889,49 @@ CREATE TABLE public.orgs ( creator_user_id uuid, audit_response_body_mode text DEFAULT 'off'::text NOT NULL, headless boolean DEFAULT false NOT NULL, + require_invite_admission boolean DEFAULT true NOT NULL, + managed_signin_allowed_domains text[] DEFAULT '{}'::text[] NOT NULL, + trial_ends_at timestamp with time zone, allow_services_outside_catalog boolean DEFAULT false NOT NULL, + user_template_policy text DEFAULT 'none'::text NOT NULL, call_timeout_ms integer, max_call_timeout_ms integer, CONSTRAINT orgs_approval_auto_bubble_secs_check CHECK ((approval_auto_bubble_secs >= 0)), CONSTRAINT orgs_audit_response_body_mode_check CHECK ((audit_response_body_mode = ANY (ARRAY['off'::text, 'errors_only'::text, 'all'::text]))), CONSTRAINT orgs_call_timeout_bounds CHECK ((((call_timeout_ms IS NULL) OR ((call_timeout_ms >= 1000) AND (call_timeout_ms <= 600000))) AND ((max_call_timeout_ms IS NULL) OR ((max_call_timeout_ms >= 1000) AND (max_call_timeout_ms <= 600000))) AND ((call_timeout_ms IS NULL) OR (max_call_timeout_ms IS NULL) OR (call_timeout_ms <= max_call_timeout_ms)))), - CONSTRAINT orgs_plan_check CHECK ((plan = ANY (ARRAY['standard'::text, 'free_unlimited'::text]))) + CONSTRAINT orgs_plan_check CHECK ((plan = ANY (ARRAY['standard'::text, 'free_unlimited'::text, 'trial'::text]))), + CONSTRAINT orgs_user_template_policy_check CHECK ((user_template_policy = ANY (ARRAY['none'::text, 'restrictive'::text, 'full'::text]))) ); +-- +-- Name: COLUMN orgs.allow_services_outside_catalog; Type: COMMENT; Schema: public; Owner: - +-- + +COMMENT ON COLUMN public.orgs.allow_services_outside_catalog IS 'When false (default), non-admins cannot instantiate global templates outside the curated catalog.'; + + +-- +-- Name: COLUMN orgs.user_template_policy; Type: COMMENT; Schema: public; Owner: - +-- + +COMMENT ON COLUMN public.orgs.user_template_policy IS 'Whether org members may create user-namespace layers: none | restrictive (reserved) | full.'; + + +-- +-- Name: COLUMN orgs.call_timeout_ms; Type: COMMENT; Schema: public; Owner: - +-- + +COMMENT ON COLUMN public.orgs.call_timeout_ms IS 'Default upstream timeout in ms for action calls in this org. NULL inherits the deployment default (CALL_TIMEOUT_MS). Overridden per template action and per call.'; + + +-- +-- Name: COLUMN orgs.max_call_timeout_ms; Type: COMMENT; Schema: public; Owner: - +-- + +COMMENT ON COLUMN public.orgs.max_call_timeout_ms IS 'Ceiling on any resolved call timeout in this org, in ms. NULL inherits CALL_TIMEOUT_MAX_MS. A caller asking for more is rejected; a template or org default above it is clamped.'; + + -- -- Name: pending_checkouts; Type: TABLE; Schema: public; Owner: - -- @@ -910,6 +1086,7 @@ CREATE TABLE public.service_instances ( updated_at timestamp with time zone DEFAULT now() NOT NULL, is_system boolean DEFAULT false NOT NULL, url text, + use_default_connection boolean DEFAULT true NOT NULL, credentials jsonb DEFAULT '{}'::jsonb NOT NULL, discovered_tools jsonb, discovered_at timestamp with time zone, @@ -919,6 +1096,27 @@ CREATE TABLE public.service_instances ( ); +-- +-- Name: COLUMN service_instances.credentials; Type: COMMENT; Schema: public; Owner: - +-- + +COMMENT ON COLUMN public.service_instances.credentials IS 'Per-scheme secret bindings: {securityScheme key -> secret NAME in the org vault}. Names only, never values. Empty map falls back to legacy secret_name for the sole instance-source scheme.'; + + +-- +-- Name: COLUMN service_instances.discovered_tools; Type: COMMENT; Schema: public; Owner: - +-- + +COMMENT ON COLUMN public.service_instances.discovered_tools IS 'MCP tools/list result for this instance (array of {name, description, input_schema, output_schema}). NULL = never resynced. Overlaid on the template''s authored tools at read time.'; + + +-- +-- Name: COLUMN service_instances.config; Type: COMMENT; Schema: public; Owner: - +-- + +COMMENT ON COLUMN public.service_instances.config IS 'Per-instance non-secret param values: {param name -> scalar}. Only params the template marks x-overslash-instance-config may appear. Never secrets — those are vault references in credentials.'; + + -- -- Name: service_templates; Type: TABLE; Schema: public; Owner: - -- @@ -934,12 +1132,90 @@ CREATE TABLE public.service_templates ( hosts text[] DEFAULT '{}'::text[] NOT NULL, created_at timestamp with time zone DEFAULT now() NOT NULL, updated_at timestamp with time zone DEFAULT now() NOT NULL, - openapi jsonb NOT NULL, + openapi jsonb, status text DEFAULT 'active'::text NOT NULL, + extends text, + delta jsonb, + CONSTRAINT service_templates_layer_shape CHECK ((((extends IS NULL) AND (delta IS NULL) AND (openapi IS NOT NULL)) OR ((extends IS NOT NULL) AND (delta IS NOT NULL)))), CONSTRAINT service_templates_status_check CHECK ((status = ANY (ARRAY['draft'::text, 'active'::text]))) ); +-- +-- Name: COLUMN service_templates.extends; Type: COMMENT; Schema: public; Owner: - +-- + +COMMENT ON COLUMN public.service_templates.extends IS 'Base template key for a derived layer (delta over a live base). NULL = standalone full-doc layer.'; + + +-- +-- Name: COLUMN service_templates.delta; Type: COMMENT; Schema: public; Owner: - +-- + +COMMENT ON COLUMN public.service_templates.delta IS 'Derived-layer content: masks (allowlist/denylist/action_patch/hidden/relabel) + extensions (actions/hosts). NULL = standalone.'; + + +-- +-- Name: upload_tokens; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.upload_tokens ( + id uuid DEFAULT gen_random_uuid() NOT NULL, + token_hash bytea NOT NULL, + org_id uuid NOT NULL, + identity_id uuid NOT NULL, + service_instance_id uuid, + service_key text, + action_key text, + request jsonb NOT NULL, + credential_ref jsonb DEFAULT '{}'::jsonb NOT NULL, + declared_sha256 text, + declared_size_bytes bigint, + declared_mime text, + declared_filename text, + max_bytes bigint NOT NULL, + filename_param text, + result_spec jsonb, + stored_media_path text, + stored_sha256 text, + stored_size_bytes bigint, + stored_mime text, + stored_filename text, + completed_at timestamp with time zone, + created_at timestamp with time zone DEFAULT now() NOT NULL, + expires_at timestamp with time zone NOT NULL, + consumed_at timestamp with time zone +); + + +-- +-- Name: TABLE upload_tokens; Type: COMMENT; Schema: public; Owner: - +-- + +COMMENT ON TABLE public.upload_tokens IS 'Single-use capability tokens for pushing bytes into a service. Minted by an action carrying x-overslash-upload, redeemed by POST /v1/uploads/{token}.'; + + +-- +-- Name: COLUMN upload_tokens.token_hash; Type: COMMENT; Schema: public; Owner: - +-- + +COMMENT ON COLUMN public.upload_tokens.token_hash IS 'sha256 of the raw token; the raw value exists only in the minted URL.'; + + +-- +-- Name: COLUMN upload_tokens.declared_sha256; Type: COMMENT; Schema: public; Owner: - +-- + +COMMENT ON COLUMN public.upload_tokens.declared_sha256 IS 'Content hash the caller declared at mint time. Verified against the stream during redemption; a mismatch refuses the descriptor so no later call can reference the bytes.'; + + +-- +-- Name: COLUMN upload_tokens.consumed_at; Type: COMMENT; Schema: public; Owner: - +-- + +COMMENT ON COLUMN public.upload_tokens.consumed_at IS 'Set by the claim. A row with consumed_at but no completed_at is a push that started and did not land.'; + + -- -- Name: user_org_memberships; Type: TABLE; Schema: public; Owner: - -- @@ -1210,6 +1486,22 @@ ALTER TABLE ONLY public.identity_groups ADD CONSTRAINT identity_groups_pkey PRIMARY KEY (identity_id, group_id); +-- +-- Name: magic_link_tokens magic_link_tokens_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.magic_link_tokens + ADD CONSTRAINT magic_link_tokens_pkey PRIMARY KEY (id); + + +-- +-- Name: magic_link_tokens magic_link_tokens_token_hash_key; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.magic_link_tokens + ADD CONSTRAINT magic_link_tokens_token_hash_key UNIQUE (token_hash); + + -- -- Name: mcp_client_agent_bindings mcp_client_agent_bindings_pkey; Type: CONSTRAINT; Schema: public; Owner: - -- @@ -1266,6 +1558,14 @@ ALTER TABLE ONLY public.mcp_upstream_tokens ADD CONSTRAINT mcp_upstream_tokens_pkey PRIMARY KEY (id); +-- +-- Name: media_descriptors media_descriptors_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.media_descriptors + ADD CONSTRAINT media_descriptors_pkey PRIMARY KEY (id); + + -- -- Name: oauth_connection_flows oauth_connection_flows_pkey; Type: CONSTRAINT; Schema: public; Owner: - -- @@ -1330,7 +1630,6 @@ ALTER TABLE ONLY public.org_idp_configs ADD CONSTRAINT org_idp_configs_pkey PRIMARY KEY (id); - -- -- Name: org_subscriptions org_subscriptions_pkey; Type: CONSTRAINT; Schema: public; Owner: - -- @@ -1459,6 +1758,22 @@ ALTER TABLE ONLY public.service_templates ADD CONSTRAINT service_templates_pkey PRIMARY KEY (id); +-- +-- Name: upload_tokens upload_tokens_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.upload_tokens + ADD CONSTRAINT upload_tokens_pkey PRIMARY KEY (id); + + +-- +-- Name: upload_tokens upload_tokens_token_hash_key; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.upload_tokens + ADD CONSTRAINT upload_tokens_token_hash_key UNIQUE (token_hash); + + -- -- Name: user_org_memberships user_org_memberships_pkey; Type: CONSTRAINT; Schema: public; Owner: - -- @@ -1514,24 +1829,24 @@ CREATE INDEX billing_email_log_user_id ON public.billing_email_log USING btree ( -- --- Name: email_unsubscribe_tokens_user_id; Type: INDEX; Schema: public; Owner: - +-- Name: call_results_expiry_idx; Type: INDEX; Schema: public; Owner: - -- -CREATE INDEX email_unsubscribe_tokens_user_id ON public.email_unsubscribe_tokens USING btree (user_id); +CREATE INDEX call_results_expiry_idx ON public.call_results USING btree (expires_at); -- --- Name: call_results_expiry_idx; Type: INDEX; Schema: public; Owner: - +-- Name: download_tokens_expiry_idx; Type: INDEX; Schema: public; Owner: - -- -CREATE INDEX call_results_expiry_idx ON public.call_results USING btree (expires_at); +CREATE INDEX download_tokens_expiry_idx ON public.download_tokens USING btree (expires_at); -- --- Name: download_tokens_expiry_idx; Type: INDEX; Schema: public; Owner: - +-- Name: email_unsubscribe_tokens_user_id; Type: INDEX; Schema: public; Owner: - -- -CREATE INDEX download_tokens_expiry_idx ON public.download_tokens USING btree (expires_at); +CREATE INDEX email_unsubscribe_tokens_user_id ON public.email_unsubscribe_tokens USING btree (user_id); -- @@ -1793,6 +2108,13 @@ CREATE INDEX idx_identity_groups_group ON public.identity_groups USING btree (gr CREATE INDEX idx_identity_groups_identity ON public.identity_groups USING btree (identity_id); +-- +-- Name: idx_magic_link_tokens_expires; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX idx_magic_link_tokens_expires ON public.magic_link_tokens USING btree (expires_at); + + -- -- Name: idx_mcp_client_agent_bindings_agent; Type: INDEX; Schema: public; Owner: - -- @@ -2052,6 +2374,13 @@ CREATE INDEX idx_service_instances_owner ON public.service_instances USING btree CREATE UNIQUE INDEX idx_service_instances_user_name ON public.service_instances USING btree (org_id, owner_identity_id, name) WHERE (owner_identity_id IS NOT NULL); +-- +-- Name: idx_service_templates_extends; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX idx_service_templates_extends ON public.service_templates USING btree (org_id, extends) WHERE (extends IS NOT NULL); + + -- -- Name: idx_service_templates_org; Type: INDEX; Schema: public; Owner: - -- @@ -2108,6 +2437,13 @@ CREATE INDEX idx_users_personal_org ON public.users USING btree (personal_org_id CREATE INDEX idx_webhook_deliveries_retry ON public.webhook_deliveries USING btree (next_retry_at) WHERE ((delivered_at IS NULL) AND (attempts < 5)); +-- +-- Name: media_descriptors_ref_idx; Type: INDEX; Schema: public; Owner: - +-- + +CREATE UNIQUE INDEX media_descriptors_ref_idx ON public.media_descriptors USING btree (org_id, service_instance_id, media_path) NULLS NOT DISTINCT; + + -- -- Name: org_idp_configs_one_default_per_org; Type: INDEX; Schema: public; Owner: - -- @@ -2143,6 +2479,13 @@ CREATE UNIQUE INDEX service_action_embeddings_org_unique ON public.service_actio CREATE UNIQUE INDEX service_action_embeddings_user_unique ON public.service_action_embeddings USING btree (org_id, owner_identity_id, template_key, action_key) WHERE (tier = 'user'::text); +-- +-- Name: upload_tokens_expiry_idx; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX upload_tokens_expiry_idx ON public.upload_tokens USING btree (expires_at); + + -- -- Name: users_overslash_idp_unique; Type: INDEX; Schema: public; Owner: - -- @@ -2395,6 +2738,14 @@ ALTER TABLE ONLY public.executions ADD CONSTRAINT executions_approval_id_fkey FOREIGN KEY (approval_id) REFERENCES public.approvals(id) ON DELETE CASCADE; +-- +-- Name: executions executions_identity_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.executions + ADD CONSTRAINT executions_identity_id_fkey FOREIGN KEY (identity_id) REFERENCES public.identities(id) ON DELETE CASCADE; + + -- -- Name: executions executions_org_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - -- @@ -2403,6 +2754,14 @@ ALTER TABLE ONLY public.executions ADD CONSTRAINT executions_org_id_fkey FOREIGN KEY (org_id) REFERENCES public.orgs(id) ON DELETE CASCADE; +-- +-- Name: executions executions_service_instance_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.executions + ADD CONSTRAINT executions_service_instance_id_fkey FOREIGN KEY (service_instance_id) REFERENCES public.service_instances(id) ON DELETE SET NULL; + + -- -- Name: group_grants group_grants_group_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - -- @@ -2587,6 +2946,22 @@ ALTER TABLE ONLY public.mcp_upstream_tokens ADD CONSTRAINT mcp_upstream_tokens_connection_id_fkey FOREIGN KEY (connection_id) REFERENCES public.mcp_upstream_connections(id) ON DELETE CASCADE; +-- +-- Name: media_descriptors media_descriptors_org_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.media_descriptors + ADD CONSTRAINT media_descriptors_org_id_fkey FOREIGN KEY (org_id) REFERENCES public.orgs(id) ON DELETE CASCADE; + + +-- +-- Name: media_descriptors media_descriptors_service_instance_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.media_descriptors + ADD CONSTRAINT media_descriptors_service_instance_id_fkey FOREIGN KEY (service_instance_id) REFERENCES public.service_instances(id) ON DELETE CASCADE; + + -- -- Name: oauth_connection_flows oauth_connection_flows_actor_identity_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - -- @@ -2851,6 +3226,30 @@ ALTER TABLE ONLY public.service_templates ADD CONSTRAINT service_templates_owner_identity_id_fkey FOREIGN KEY (owner_identity_id) REFERENCES public.identities(id) ON DELETE CASCADE; +-- +-- Name: upload_tokens upload_tokens_identity_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.upload_tokens + ADD CONSTRAINT upload_tokens_identity_id_fkey FOREIGN KEY (identity_id) REFERENCES public.identities(id) ON DELETE CASCADE; + + +-- +-- Name: upload_tokens upload_tokens_org_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.upload_tokens + ADD CONSTRAINT upload_tokens_org_id_fkey FOREIGN KEY (org_id) REFERENCES public.orgs(id) ON DELETE CASCADE; + + +-- +-- Name: upload_tokens upload_tokens_service_instance_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.upload_tokens + ADD CONSTRAINT upload_tokens_service_instance_id_fkey FOREIGN KEY (service_instance_id) REFERENCES public.service_instances(id) ON DELETE CASCADE; + + -- -- Name: user_org_memberships user_org_memberships_org_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - -- @@ -2903,5 +3302,5 @@ ALTER TABLE ONLY public.webhook_subscriptions -- PostgreSQL database dump complete -- -\unrestrict Wa9cVbD20aiSD1mg8hdX4ucJSCAI1Zo8GH1ywdS2KyvkPycAg0afifQTGjdzEay +\unrestrict ITpRAwCBQB1Eujdumlb2lXtfTcabr3ewsacbHizs9jNwJBpKN4XyV8oROCASLy3 diff --git a/STATUS.md b/STATUS.md index d30d27ec..5b419e8a 100644 --- a/STATUS.md +++ b/STATUS.md @@ -205,6 +205,7 @@ - Typed `reauth_required` + `needs_authentication` error envelopes (PR #259); structured 400 + dry-run `POST /v1/actions/validate` (PR #256); MCP tools/call surfaces the same typed envelopes (PR #263). - Stable webhook envelope with routing headers (PR #258); connection lifecycle events emit webhooks (PR #260). - Re-fetchable call results (DECISIONS D61): a `verbose: false` render that truncates stores the full `ActionResult` (encrypted, `call_results` table, migration 111) and stamps `_full_result.download_url` into the same envelope as the cropped body. Redeemed through the existing `GET /v1/downloads/{token}`, which now branches on `call_result_id` to serve stored bytes instead of replaying upstream. `CALL_RESULT_MAX_BYTES` (1 MB, `0` disables); shares `DOWNLOAD_TOKEN_TTL_SECS`. Side effect: `deliver: "url"` now effectively works for OAuth-authenticated services on the re-fetch path, since a result-backed token dials nothing. Complements D57, which mints a token for the *same request* when the transport cap trips and no body ever existed; this one serves *stored bytes* and dials nothing. Truncation itself is unchanged — priority-aware compaction is in TODO.md §3. +- Proxy uploads — bytes move *into* a service (DECISIONS D-NEXT): an MCP action declaring `x-overslash-upload` is served by the gateway rather than forwarded, and mints a one-shot `upload_url` on `POST|PUT /v1/uploads/{token}` (anonymous, per-IP throttled, `upload_tokens` + `media_descriptors`, migration 116). Redemption streams the request body straight through to the byte route on the instance's own origin with the bearer re-resolved from the vault; nothing about the container's URL or token reaches the caller. Optional `sha256` / `size_bytes` bind the capability to one file — size is cut mid-stream, a hash mismatch answers 422 and withholds the stored reference. Intercepted at *both* dispatch sites, including approval replay (`StoredMcpCall.upload`), because a `risk: write` upload reaches the replay path first for any gated agent. `UPLOAD_TOKEN_TTL_SECS` (900), `UPLOAD_MAX_BYTES` (100 MiB); audited as `action.uploaded` with declared *and* measured. `services/whatsapp.yaml` gains `upload_media`; `send_file` / `send_audio_message` resolve `media_path` through the new ledger (`resolve: {source: media}`), so an approval's primary row names the file instead of hashing it — including for references seen on a plain `download_media` with no `deliver: "url"`. Bytes pushed to the container out of band stay un-enriched and fall back to the raw path. - Layered call timeouts (DECISIONS D56): `timeout_ms` per call, `x-overslash-timeout_ms` per action, `x-overslash-default_timeout_ms` per service, `call_timeout_ms` / `max_call_timeout_ms` per org, `CALL_TIMEOUT_MS` / `CALL_TIMEOUT_MAX_MS` per deployment. 504 carries `timeout_source`. The previously-unbounded inline path is now bounded; streaming bounds time-to-first-byte and guards the transfer with a per-chunk idle timeout. `CALL_TIMEOUT_MS` is pinned to 110000 in Cloud Run for the rollout, to be removed once audit percentiles confirm nothing legitimate lives above 30s. - Generic pagination (DECISIONS D75): `x-overslash-pagination` on an operation or MCP tool maps the corpus's six spellings of page size and six of continuation onto one vocabulary — `page_size: {param, default, max}` plus `next: {style, param, from}` over four styles (`cursor`, `offset`, `page`, RFC 8288 `link`). `page_size.default` seeds the parameter's own `default:` at compile time, so the bound reaches the wire through `apply_defaults` and shows up on `/v1/search` rows without a second mechanism. A paged result carries `_pagination: {has_more, next: {service, action, params}}` — the *delta* for the next call, in the action's own parameter names — derived from the raw body and headers, so a jq `filter` or the compact render cannot cost the caller the page. `/v1/search` rows gain `paginated: true`; `ActionSummary` gains a `pagination` object and the service actions tab a "pages" pill; the `response_too_large` 502 hint names the page-size parameter. The async worker and approval replay carry the declaration and the call's arguments on the stored payload (`StoredPagination`), since a stored call is a resolved request with no action key left to look it up from — the same answer D56 gave the timeout cascade. The gateway never follows a page. Builds on D74. `services/gmail.yaml` is the only shipped template annotated so far — the corpus sweep and the MCP-side templates are a follow-up. - List-heavy actions have a middle gear (DECISIONS D57). `GET /v1/search` action rows now carry `params` — the action's caller-supplied contract (name/type/required/description/enum/default), required-first then alphabetical, `instance-config` params excluded — so a declared paging parameter is discoverable instead of folklore. `filter` is declared on the `overslash_call` / `overslash_read` MCP tool schemas and forwarded (bare jq string lifted into `{lang, expr}`), and now actually applies on the MCP-runtime and platform-runtime forks, which previously accepted it and silently ignored it. A `response_too_large` 502 carries a pre-minted `download_url` + `expires_at` for the same request (best-effort: OAuth-injected services and inline raw-HTTP credentials still get the plain 502), and the compact truncation hint leads with narrowing rather than `verbose=true`. `services/metabase.yaml` gains `limit`/`offset`/`archived` on `search`, the real `f` enum + `model_id` on `list_cards`, and new `popular_items` / `recents` actions; `export_query` now declares a `responses:` block so it compiles to a binary response type instead of being buffered against the size cap. diff --git a/crates/overslash-api/src/config/from_env.rs b/crates/overslash-api/src/config/from_env.rs index 33d67b30..48a3885a 100644 --- a/crates/overslash-api/src/config/from_env.rs +++ b/crates/overslash-api/src/config/from_env.rs @@ -158,6 +158,20 @@ impl Config { .and_then(|s| s.parse().ok()) .filter(|n| *n > 0) .unwrap_or(900), // 15 min + upload_token_ttl_secs: env::var("UPLOAD_TOKEN_TTL_SECS") + .ok() + .and_then(|s| s.parse().ok()) + .filter(|n| *n > 0) + .unwrap_or(900), // 15 min + // Matches the reference container's own default. Set it lower to + // bound what a redemption can push; raising it past what the + // upstream accepts only moves the rejection later, to a 413 from + // the upstream after the bytes have already crossed the wire. + upload_max_bytes: env::var("UPLOAD_MAX_BYTES") + .ok() + .and_then(|s| s.parse().ok()) + .filter(|n| *n > 0) + .unwrap_or(100 * 1024 * 1024), // 100 MiB // No `.filter(|n| *n > 0)` here, unlike the TTL above: 0 is a // meaningful value — it turns result storage off — whereas a // zero-second token lifetime is only ever a misconfiguration. diff --git a/crates/overslash-api/src/config/mod.rs b/crates/overslash-api/src/config/mod.rs index 3c77a0c2..427d9ebd 100644 --- a/crates/overslash-api/src/config/mod.rs +++ b/crates/overslash-api/src/config/mod.rs @@ -143,6 +143,22 @@ pub struct Config { /// long enough that an agent can hand the URL to a shell and let a large /// file finish transferring, including a retry or two. pub download_token_ttl_secs: i64, + /// Lifetime of an upload capability token. + /// + /// Separate from the download TTL because it is bounding a different + /// thing. A download URL leaks read access to bytes that already exist; an + /// upload URL is a window in which someone can push bytes the gateway + /// already authorized, so the cost of a long window is that the *approved* + /// description and the *actual* payload drift further apart in time. + pub upload_token_ttl_secs: i64, + /// Hard ceiling on the bytes one upload redemption may push, whatever a + /// template declares. + /// + /// A template's own `max_bytes` can only lower this. The body is metered + /// chunk by chunk and the transfer is cut the moment it goes over, so this + /// bounds work actually done rather than work merely promised — a caller + /// that lies in `Content-Length` gets the same answer, just later. + pub upload_max_bytes: u64, /// Ceiling on the plaintext size of a stored call result. /// /// A truncated compact render stores the full `ActionResult` so the same @@ -925,6 +941,8 @@ pub(crate) mod tests { audit_response_body_max_bytes: 0, filter_timeout_ms: 0, download_token_ttl_secs: 900, + upload_token_ttl_secs: 900, + upload_max_bytes: 100 * 1024 * 1024, call_result_max_bytes: 1024 * 1024, dashboard_url: "/".into(), dashboard_origin: "*".into(), diff --git a/crates/overslash-api/src/lib.rs b/crates/overslash-api/src/lib.rs index bdf6bd24..5d39402e 100644 --- a/crates/overslash-api/src/lib.rs +++ b/crates/overslash-api/src/lib.rs @@ -346,6 +346,17 @@ pub async fn create_app(mut config: Config) -> anyhow::Result { |n| tracing::info!("Expired {n} download_tokens"), ) .await; + // The inbound half. Kept separate from the sweep above rather + // than folded into it because the tables are separate for a + // reason — an upload token is single-use, a download token is + // not — and a shared prune would be the first place that + // distinction quietly stopped mattering. + instrumented_step( + "upload_token_expiry", + async { overslash_db::repos::upload_token::prune_expired(&db).await }, + |n| tracing::info!("Expired {n} upload_tokens"), + ) + .await; // Stored results for truncated compact renders (D61). Ordering // against the sweep above is irrelevant: the FK from // `download_tokens.call_result_id` cascades, so pruning a @@ -594,6 +605,10 @@ pub async fn create_app(mut config: Config) -> anyhow::Result { // layer because that layer keys on an API-key prefix these requests // don't have — the handler throttles per-IP itself. .merge(routes::downloads::router()) + // Upload redemption, and the same reasoning inverted: the process + // pushing the bytes is not the caller either, and the token is the sole + // authority for one push. + .merge(routes::uploads::router()) .merge(stripe_webhook_routes) .merge(validate_routes) .merge(rate_limited_routes) diff --git a/crates/overslash-api/src/routes/actions/approval_detail.rs b/crates/overslash-api/src/routes/actions/approval_detail.rs index 3738fb1c..413e9330 100644 --- a/crates/overslash-api/src/routes/actions/approval_detail.rs +++ b/crates/overslash-api/src/routes/actions/approval_detail.rs @@ -204,6 +204,7 @@ mod tests { redact: Vec::new(), oauth_injected: false, download: None, + upload: None, params: HashMap::new(), resolved: HashMap::new(), canonical: HashMap::new(), diff --git a/crates/overslash-api/src/routes/actions/call_mcp.rs b/crates/overslash-api/src/routes/actions/call_mcp.rs index cbc588c9..4cc680b1 100644 --- a/crates/overslash-api/src/routes/actions/call_mcp.rs +++ b/crates/overslash-api/src/routes/actions/call_mcp.rs @@ -17,7 +17,69 @@ use overslash_db::scopes::OrgScope; use super::call::UpstreamErrored; use super::*; use super::{approval_detail::*, dto::CallRequest, dto::ResolvedMeta}; -use crate::{AppState, extractors::AuthContext, services::audit_capture, services::mcp_caller}; +use crate::{ + AppState, extractors::AuthContext, services::audit_capture, services::mcp_caller, + services::proxy_upload, +}; + +/// Audit a minted upload capability and render it as an ordinary call result. +/// +/// No upstream call happened, so there is no `action.executed` row to write. +/// Recording the mint instead is what keeps a gateway-served action from being +/// a hole in the trail: without it, the only evidence would be a later +/// `action.uploaded` with nothing before it explaining who was allowed to push. +#[allow(clippy::too_many_arguments)] +async fn mint_response( + state: &AppState, + ext: &axum::http::Extensions, + scope: &OrgScope, + result: overslash_core::types::ActionResult, + req: &CallRequest, + meta: &ResolvedMeta, + auth: &AuthContext, + identity_id: Uuid, + ip: Option<&str>, + call_tags: Vec, +) -> Result { + let _ = scope + .clone() + .log_audit_tagged( + AuditEntry { + org_id: auth.org_id, + identity_id: Some(identity_id), + action: "action.deferred", + resource_type: req.service.as_deref(), + resource_id: None, + detail: serde_json::json!({ + "runtime": "mcp", + "service": req.service, + "action": req.action, + "cause": "upload_requested", + // The descriptor, minus nothing: it carries a capability + // URL, and the raw token in it is exactly what the audit + // trail must not hand to whoever can read audit rows. + "response": { "skipped": "capability" }, + }), + description: meta.description.as_deref(), + ip_address: ip, + }, + &tags::with_outcome(call_tags, false), + ) + .await; + + let rendered = + super::render_stored(state, ext, &result, req, meta, auth.org_id, identity_id).await; + Ok(( + StatusCode::OK, + Json(CallResponse::Called { + result: rendered, + action_description: meta.description.clone(), + is_error: false, + execution_id: None, + }), + ) + .into_response()) +} /// Dispatch an MCP-runtime action and build its response. #[allow(clippy::too_many_arguments)] @@ -37,6 +99,44 @@ pub(super) async fn dispatch( audit_body_mode: audit_capture::AuditResponseBodyMode, deliver_url: bool, ) -> Result { + // Gateway-served actions never reach the upstream. An `x-overslash-upload` + // block says the byte route is plain HTTP on this origin, not a tool, so + // there is nothing to call — `tools/call` with this name would answer + // "unknown tool". The mirror of this test lives on the replay path in + // `stored_call::run_mcp`; both are needed, because a `risk: write` upload + // reaches the replay path first for any agent whose calls are gated. + if let Some(spec) = meta.upload.as_ref() { + let minted = proxy_upload::intercept_mint( + state, + ext, + proxy_upload::Mint { + org_id: auth.org_id, + identity_id, + service_instance_id: meta.instance_id, + service_key: req.service.as_deref(), + action_key: req.action.as_deref(), + mcp_url: &mcp_target.url, + mcp_auth: &mcp_target.auth, + spec, + arguments: &mcp_target.arguments, + }, + ) + .await?; + return mint_response( + state, + ext, + scope, + minted, + req, + meta, + auth, + identity_id, + ip, + call_tags, + ) + .await; + } + let mut result = match mcp_caller::invoke( state, scope, @@ -165,6 +265,24 @@ pub(super) async fn dispatch( ) .await; + // Record what the tool said about the bytes it referenced, on *any* + // successful call rather than only a deferred one. The template's guidance + // is to call a download tool without `deliver` when the raw reference is + // what you want to forward, so gating this on the mint would leave the + // common path unrecorded and every later approval quoting a bare hash. + if !is_error && let Some(spec) = meta.download.as_ref() { + deferred::record_downloaded( + state, + ext, + auth.org_id, + meta.instance_id, + req.service.as_deref(), + spec, + &result, + ) + .await; + } + // Deferred delivery. See `deferred::swap_in_mcp_download` for why a // failed tool result is never minted from. if deliver_url && !is_error { diff --git a/crates/overslash-api/src/routes/actions/deferred.rs b/crates/overslash-api/src/routes/actions/deferred.rs index 97848d5e..9226e807 100644 --- a/crates/overslash-api/src/routes/actions/deferred.rs +++ b/crates/overslash-api/src/routes/actions/deferred.rs @@ -95,7 +95,7 @@ async fn mint_mcp_download( action_key: Option<&str>, filter_timeout: std::time::Duration, ) -> Result { - use overslash_core::types::{DisclosureField, DownloadAuth, McpAuth}; + use overslash_core::types::{DownloadAuth, McpAuth}; let Some(spec) = spec else { return Err(AppError::BadRequest(format!( @@ -105,54 +105,27 @@ async fn mint_mcp_download( ))); }; - // The MCP envelope, as jq sees it: {runtime, tool, structured, content, - // is_error}. Same input shape the `disclose` filters address. - let envelope: serde_json::Value = - serde_json::from_str(&result.body).unwrap_or(serde_json::Value::Null); - - // Reuse the disclosure runner rather than a second jq harness: it already - // owns the timeout, the input-size ceiling, and spawn_blocking. Labels are - // the join key because it drops zero-yield filters, so positions shift. - let mut fields = vec![DisclosureField { - label: "url".into(), - filter: spec.url.clone(), - max_chars: None, - primary: false, - }]; - for (label, filter) in [ - ("mime", spec.mime.as_ref()), - ("size", spec.size.as_ref()), - ("filename", spec.filename.as_ref()), - ] { - if let Some(f) = filter { - fields.push(DisclosureField { - label: label.into(), - filter: f.clone(), - max_chars: None, - primary: false, - }); - } - } - - let disclosed = - crate::services::disclosure::run_disclosures(&fields, &envelope, filter_timeout) - .await - .map_err(|e| AppError::BadGateway(format!("download filters failed: {e}")))?; + let picked = evaluate_download(spec, result, filter_timeout) + .await + .map_err(AppError::BadGateway)?; let pick = |label: &str| -> Option { - disclosed - .iter() - .find(|d| d.label == label) - .and_then(|d| d.value.clone()) + match label { + "mime" => picked.mime.clone(), + "size" => picked.size.clone(), + "filename" => picked.filename.clone(), + _ => None, + } }; - let Some(raw_url) = pick("url").filter(|s| !s.trim().is_empty()) else { + let Some(raw_url) = picked.url.clone() else { return Err(AppError::BadGateway(format!( "tool `{}` returned no value for the declared download url filter (`{}`)", mcp_target.tool, spec.url ))); }; - let url = resolve_download_url(&mcp_target.url, raw_url.trim())?; + let url = + crate::services::deferred_download::resolve_same_origin(&mcp_target.url, raw_url.trim())?; let secret_name = match (&mcp_target.auth, spec.auth) { (_, DownloadAuth::None) => None, @@ -179,14 +152,14 @@ async fn mint_mcp_download( crate::services::deferred_download::Mint { org_id, identity_id, - // MCP resolution doesn't thread the instance id through - // `ResolvedMeta` (it's an HTTP-replay concern), and the token - // doesn't need it — `request` already names everything the fetch - // re-resolves. + // The token doesn't need it — `request` already names everything + // the fetch re-resolves — and leaving it unset keeps this row + // describing the fetch rather than the binding it came from. service_instance_id: None, service_key, action_key, request: Some(crate::services::deferred_download::bearer_request( + "GET", url, secret_name, )), @@ -200,33 +173,6 @@ async fn mint_mcp_download( .await } -/// Resolve a download location from a tool result against the MCP server's own -/// URL, rejecting anything that would send the credential elsewhere. -/// -/// Accepts a path (`/media/abc`) or an absolute URL on the same origin. -/// Everything else — a different host, a different scheme, a non-URL — is an -/// error rather than a best-effort fetch. -fn resolve_download_url(mcp_url: &str, raw: &str) -> Result { - let base = url::Url::parse(mcp_url) - .map_err(|e| AppError::Internal(format!("mcp instance url is not a url: {e}")))?; - - let joined = base.join(raw).map_err(|e| { - AppError::BadGateway(format!("download url `{raw}` is not resolvable: {e}")) - })?; - - let same_origin = joined.scheme() == base.scheme() - && joined.host_str() == base.host_str() - && joined.port_or_known_default() == base.port_or_known_default(); - if !same_origin { - return Err(AppError::BadGateway(format!( - "download url `{raw}` points outside the MCP server's origin ({}); \ - refusing to send this service's credential to another host", - base.origin().ascii_serialization() - ))); - } - Ok(joined.to_string()) -} - /// The call-handler context an HTTP-runtime mint reads from. /// /// Borrows the resolved objects rather than exploding them into scalars: the @@ -387,83 +333,127 @@ pub(super) async fn mint_http_descriptor( Ok(descriptor) } -#[cfg(test)] -mod download_url_tests { - use super::resolve_download_url; - - const MCP: &str = "https://wa.example.com:8443/mcp"; - - /// The shape the shipped WhatsApp template uses: the tool hands back a - /// path, which resolves against the server that served the tool call. - #[test] - fn a_relative_path_resolves_against_the_mcp_server() { - let out = resolve_download_url(MCP, "/media/abc123").unwrap(); - assert_eq!(out, "https://wa.example.com:8443/media/abc123"); - } - - #[test] - fn an_absolute_same_origin_url_is_accepted_verbatim() { - let out = - resolve_download_url(MCP, "https://wa.example.com:8443/media/abc123?x=1").unwrap(); - assert_eq!(out, "https://wa.example.com:8443/media/abc123?x=1"); - } +/// What a `x-overslash-download` block's filters yielded, evaluated once. +/// +/// Extracted so the ledger and the mint read the *same* answer rather than +/// running the filters twice and hoping jq is deterministic across two +/// envelopes it was handed separately. +pub(super) struct PickedDownload { + pub url: Option, + pub mime: Option, + pub size: Option, + pub filename: Option, + pub sha256: Option, +} - /// The control that matters. The URL comes from the MCP server's own - /// response and the deferred fetch attaches that instance's credential — - /// so a hostile or compromised server naming another host must not be able - /// to have the gateway deliver the bearer to it. - #[test] - fn a_different_host_is_refused() { - let err = resolve_download_url(MCP, "http://169.254.169.254/latest/meta-data/") - .expect_err("off-origin must be refused"); - assert!( - format!("{err:?}").contains("outside the MCP server's origin"), - "{err:?}" - ); - } +/// Evaluate a download declaration against a completed tool result. +/// +/// The MCP envelope, as jq sees it, is `{runtime, tool, structured, content, +/// is_error}` — the same input shape `disclose` filters address. The disclosure +/// runner is reused rather than a second jq harness because it already owns the +/// timeout, the input-size ceiling and the `spawn_blocking`. +pub(super) async fn evaluate_download( + spec: &overslash_core::types::DownloadSpec, + result: &overslash_core::types::ActionResult, + filter_timeout: std::time::Duration, +) -> Result { + use overslash_core::types::DisclosureField; - #[test] - fn a_different_scheme_on_the_same_host_is_refused() { - // Downgrading https→http on the same host would put the credential on - // the wire in plaintext, so origin comparison includes the scheme. - resolve_download_url(MCP, "http://wa.example.com:8443/media/abc") - .expect_err("scheme change must be refused"); - } + let envelope: serde_json::Value = + serde_json::from_str(&result.body).unwrap_or(serde_json::Value::Null); - #[test] - fn a_different_port_on_the_same_host_is_refused() { - resolve_download_url(MCP, "https://wa.example.com:9999/media/abc") - .expect_err("port change must be refused"); + let mut fields = vec![DisclosureField { + label: "url".into(), + filter: spec.url.clone(), + max_chars: None, + primary: false, + }]; + for (label, filter) in [ + ("mime", spec.mime.as_ref()), + ("size", spec.size.as_ref()), + ("filename", spec.filename.as_ref()), + ("sha256", spec.sha256.as_ref()), + ] { + if let Some(f) = filter { + fields.push(DisclosureField { + label: label.into(), + filter: f.clone(), + max_chars: None, + primary: false, + }); + } } - /// Default ports compare equal to their explicit form — otherwise a server - /// configured as `https://host/mcp` returning `https://host:443/…` would - /// be rejected for no reason. - #[test] - fn an_implicit_default_port_matches_its_explicit_form() { - resolve_download_url( - "https://wa.example.com/mcp", - "https://wa.example.com:443/media/abc", - ) - .expect("443 is https's default port"); - } + let disclosed = + crate::services::disclosure::run_disclosures(&fields, &envelope, filter_timeout) + .await + .map_err(|e| format!("download filters failed: {e}"))?; + // Joined on label because the runner drops zero-yield filters, so positions + // shift. + let pick = |label: &str| -> Option { + disclosed + .iter() + .find(|d| d.label == label) + .and_then(|d| d.value.clone()) + .map(|s| s.trim().to_string()) + .filter(|s| !s.is_empty()) + }; - #[test] - fn a_protocol_relative_url_cannot_smuggle_a_new_host() { - // `//evil.com/x` inherits the scheme but not the host — the classic - // way past a naive "starts with http" check. - resolve_download_url(MCP, "//evil.com/media/abc") - .expect_err("protocol-relative host swap must be refused"); - } + Ok(PickedDownload { + url: pick("url"), + mime: pick("mime"), + size: pick("size"), + filename: pick("filename"), + sha256: pick("sha256"), + }) +} - /// The guarantee is "never leaves the origin", not "rejects anything odd". - /// A non-URL string joins as a relative path and stays on the MCP host, so - /// the worst case is a 404 upstream — and the control characters that could - /// otherwise smuggle a second request get percent-encoded on the way. - #[test] - fn a_non_url_string_stays_on_the_origin_and_is_escaped() { - let out = resolve_download_url(MCP, "not a url at all\n").unwrap(); - assert!(out.starts_with("https://wa.example.com:8443/"), "{out}"); - assert!(!out.contains('\n') && !out.contains(' '), "{out}"); +/// Record what a download tool said about the bytes it referenced. +/// +/// Deliberately *not* gated on `deliver: "url"`. The template's own guidance +/// tells agents to call a download tool without `deliver` when they want the +/// raw reference to forward — which is the common case — so gating the write on +/// the mint would leave exactly the paths that produce references to later +/// approvals unrecorded, and every one of those approvals showing a bare hash. +/// +/// Best-effort throughout: the call succeeded, and failing it because a +/// descriptive row would not write trades a working call for a tidier ledger. +pub(super) async fn record_downloaded( + state: &AppState, + ext: &axum::http::Extensions, + org_id: uuid::Uuid, + instance_id: Option, + service_key: Option<&str>, + spec: &overslash_core::types::DownloadSpec, + result: &overslash_core::types::ActionResult, +) { + let filter_timeout = std::time::Duration::from_millis(state.config.filter_timeout_ms); + let picked = match evaluate_download(spec, result, filter_timeout).await { + Ok(p) => p, + Err(e) => { + tracing::warn!(error = %e, "media ledger: download filters failed"); + return; + } + }; + let Some(media_path) = picked.url else { + return; + }; + if let Err(e) = overslash_db::repos::media_descriptor::record( + state.db(ext), + overslash_db::repos::media_descriptor::NewMediaDescriptor { + org_id, + service_instance_id: instance_id, + service_key, + media_path: &media_path, + sha256: picked.sha256.as_deref(), + mime: picked.mime.as_deref(), + size_bytes: picked.size.as_deref().and_then(|s| s.parse::().ok()), + filename: picked.filename.as_deref(), + source: overslash_db::repos::media_descriptor::MediaSource::Download, + }, + ) + .await + { + tracing::warn!(error = %e, "media ledger: download descriptor not recorded"); } } diff --git a/crates/overslash-api/src/routes/actions/dto.rs b/crates/overslash-api/src/routes/actions/dto.rs index 9b2a240b..c464c184 100644 --- a/crates/overslash-api/src/routes/actions/dto.rs +++ b/crates/overslash-api/src/routes/actions/dto.rs @@ -342,6 +342,12 @@ pub(super) struct ResolvedMeta { /// it's how a tool result says "the bytes are over there". HTTP actions /// are their own download and leave this `None`. pub(super) download: Option, + /// The action's `x-overslash-upload` block, when it declares one. + /// + /// Its presence is what makes an action gateway-served: the dispatch sites + /// mint a capability instead of forwarding a tool call, so the upstream + /// never sees a tool by this name. + pub(super) upload: Option, /// Whether this call authenticates via OAuth, mirroring /// `ResolvedAuth::oauth_injected`. /// diff --git a/crates/overslash-api/src/routes/actions/flags.rs b/crates/overslash-api/src/routes/actions/flags.rs index 50de5013..84fb1ac4 100644 --- a/crates/overslash-api/src/routes/actions/flags.rs +++ b/crates/overslash-api/src/routes/actions/flags.rs @@ -178,9 +178,15 @@ pub(super) fn validate_resolved( .actions .get(action_key) .is_some_and(|a| a.response_type.as_deref() == Some("binary")); + let gateway_upload = resolved + .svc + .actions + .get(action_key) + .is_some_and(|a| a.upload.is_some()); let blockers = wait_mode::Blockers { platform_runtime, binary_response, + gateway_upload, ..Default::default() }; @@ -214,5 +220,18 @@ pub(super) fn validate_resolved( ))); } + // The mirror of the `deliver: "url"` refusal, and for the same reason: an + // upload action's whole output is a capability with a lifetime, so + // deferring the call spends that lifetime in a queue. A caller who wants + // the URL later should ask for it later. + if gateway_upload { + return Err(AppError::BadRequest(format!( + "action '{}' on service '{}' mints an upload capability; execution: \"{m}\" \ + is not supported — the upload URL would start expiring while the call \ + waited in the queue. Call it synchronously.", + action_key, resolved.svc.key + ))); + } + Ok(blockers) } diff --git a/crates/overslash-api/src/routes/actions/replay_payload.rs b/crates/overslash-api/src/routes/actions/replay_payload.rs index 6985b5f6..3a28a2c6 100644 --- a/crates/overslash-api/src/routes/actions/replay_payload.rs +++ b/crates/overslash-api/src/routes/actions/replay_payload.rs @@ -50,6 +50,11 @@ pub(super) fn build( tool: target.tool.clone(), arguments: target.arguments.clone(), pagination: stored_pagination(meta, req), + // Resolved once, here, because replay cannot resolve it again. + upload: meta.upload.clone(), + service_instance_id: meta.instance_id, + service_key: meta.service_scope.as_ref().map(|s| s.service_key.clone()), + action_key: req.action.clone(), }) .ok(); } @@ -117,6 +122,7 @@ mod tests { redact: Vec::new(), oauth_injected: false, download: None, + upload: None, params: Default::default(), resolved: Default::default(), canonical: Default::default(), diff --git a/crates/overslash-api/src/routes/actions/resolve.rs b/crates/overslash-api/src/routes/actions/resolve.rs index f8add26d..b351f4c8 100644 --- a/crates/overslash-api/src/routes/actions/resolve.rs +++ b/crates/overslash-api/src/routes/actions/resolve.rs @@ -203,6 +203,7 @@ pub(super) async fn resolve_request( action_handoff_after_ms: None, service_timeout_ms: svc.default_timeout_ms, download: None, + upload: None, params: HashMap::new(), resolved: HashMap::new(), canonical: HashMap::new(), @@ -378,6 +379,23 @@ pub(super) async fn resolve_request( ) .await; + // Ledger-backed resolvers, folded in after the runtime fork so one + // lookup path serves both. Later insert wins, but the two sets are + // disjoint by construction: `has_one_target` makes a param declare + // exactly one of `get`, `tool` or `source`. + let mut resolved = resolved; + resolved.display.extend( + crate::services::param_resolver::resolve_ledger_params( + state.db(ext), + scope.org_id(), + instance.as_ref().map(|i| i.id), + action, + &req.params, + ) + .await + .display, + ); + // Interpolate `{param}` placeholders in the action description // using the caller's supplied params, preferring a resolved // display name. Mirrors the HTTP path so approvals and audit rows @@ -420,6 +438,7 @@ pub(super) async fn resolve_request( action_pagination: action.pagination.clone(), action_handoff_after_ms: action.handoff_after_ms, download: action.download.clone(), + upload: action.upload.clone(), params: req.params.clone(), resolved: resolved.display, canonical: resolved.canonical, @@ -431,7 +450,13 @@ pub(super) async fn resolve_request( arguments, }), platform_target: None, - instance_id: None, + // Threaded on the MCP fork too, now that something reads it + // here: the media ledger scopes references to the instance + // that stores them, because a content address is only + // meaningful on one host. Replay still resolves its own + // credential from the payload, so this changes nothing + // there. + instance_id: instance.as_ref().map(|i| i.id), binding: BindingFacts::new(instance.as_ref(), &svc, mcp_principal), }, )); @@ -485,6 +510,7 @@ pub(super) async fn resolve_request( // Platform actions dispatch in-process; nothing is dialed. oauth_injected: false, download: None, + upload: None, params: HashMap::new(), resolved: HashMap::new(), canonical: HashMap::new(), @@ -839,6 +865,23 @@ pub(super) async fn resolve_request( ) .await; + // Ledger-backed resolvers, folded in after the runtime fork so one + // lookup path serves both. Later insert wins, but the two sets are + // disjoint by construction: `has_one_target` makes a param declare + // exactly one of `get`, `tool` or `source`. + let mut resolved = resolved; + resolved.display.extend( + crate::services::param_resolver::resolve_ledger_params( + state.db(ext), + scope.org_id(), + instance.as_ref().map(|i| i.id), + action, + &req.params, + ) + .await + .display, + ); + // The approval title and audit row use the short `summary` (falling // back to `description` when an action authors only the long form) — // the agent-facing `description` is free to run to a paragraph, which @@ -883,6 +926,7 @@ pub(super) async fn resolve_request( action_pagination: action.pagination.clone(), action_handoff_after_ms: action.handoff_after_ms, download: None, + upload: None, params: req.params.clone(), resolved: resolved.display, canonical: resolved.canonical, diff --git a/crates/overslash-api/src/routes/actions/tags.rs b/crates/overslash-api/src/routes/actions/tags.rs index 82cc44a8..7d6bf6f5 100644 --- a/crates/overslash-api/src/routes/actions/tags.rs +++ b/crates/overslash-api/src/routes/actions/tags.rs @@ -192,6 +192,7 @@ mod tests { redact: Vec::new(), oauth_injected: false, download: None, + upload: None, params: Default::default(), resolved: Default::default(), canonical: Default::default(), diff --git a/crates/overslash-api/src/routes/approvals/dto.rs b/crates/overslash-api/src/routes/approvals/dto.rs index fa6bd93f..06fa422b 100644 --- a/crates/overslash-api/src/routes/approvals/dto.rs +++ b/crates/overslash-api/src/routes/approvals/dto.rs @@ -138,6 +138,7 @@ mod risk_tests { disabled: false, request_body: None, download: None, + upload: None, }, ); let mut registry = ServiceRegistry::default(); diff --git a/crates/overslash-api/src/routes/mod.rs b/crates/overslash-api/src/routes/mod.rs index e9d25afa..cccdfae6 100644 --- a/crates/overslash-api/src/routes/mod.rs +++ b/crates/overslash-api/src/routes/mod.rs @@ -38,6 +38,7 @@ pub mod services; pub mod skill_md; pub mod templates; pub mod unsubscribe; +pub mod uploads; pub mod util; pub mod version; pub mod webhooks; diff --git a/crates/overslash-api/src/routes/uploads.rs b/crates/overslash-api/src/routes/uploads.rs new file mode 100644 index 00000000..c30540db --- /dev/null +++ b/crates/overslash-api/src/routes/uploads.rs @@ -0,0 +1,738 @@ +//! `POST|PUT /v1/uploads/{token}` — redeem an upload capability. +//! +//! Mounted **outside** the auth and rate-limit layers, like +//! [`super::downloads`]: the pushing process is deliberately not the caller. It +//! is `curl` in a sandbox, or a browser — something that holds none of the +//! caller's credentials and cannot be handed any. The token in the URL is the +//! sole authority, which is why it is 256 bits of randomness, stored only as a +//! hash, short-lived, and good for exactly one successful push. +//! +//! The authorization decision was made and audited when the action call minted +//! the token; see [`crate::services::proxy_upload`]. What happens here is the +//! second half: re-resolve the upstream credential from the vault as it stands +//! right now, stream the body through it, and record what came back. +//! +//! # Nothing the redeemer sends decides anything +//! +//! The filename, the target route, the byte ceiling and the declared content +//! all come off the token. The redeemer contributes bytes and, at most, a +//! `Content-Type` hint. That is not defensive tidiness: a redeemer who could +//! choose the filename could get a reviewer's approval for `notes.txt` and push +//! `payroll.xlsx` under it. +//! +//! Failure modes are deliberately indistinguishable. Unknown token, expired +//! token, already-consumed token and deleted identity all return the same bare +//! `404`. + +use axum::{ + Router, + extract::{Path, State}, + http::{HeaderMap, StatusCode}, + response::{IntoResponse, Response}, + routing::post, +}; +use overslash_core::types::{ActionRequest, DisclosureField, UploadResultSpec}; +use overslash_db::repos::audit::AuditEntry; +use overslash_db::repos::upload_token::{self, UploadTokenRow}; +use overslash_db::scopes::OrgScope; + +use crate::{ + AppState, + extractors::{ClientIp, ReqExt}, + services::{deferred_download, proxy_upload}, +}; + +/// Per-IP ceiling on redemption attempts. +/// +/// Far tighter than the download side's 120/60, and the asymmetry is the point: +/// a download token is multi-use precisely so a large transfer can resume, so +/// its ceiling has to accommodate a legitimate retry storm. An upload token is +/// good for one successful push and cannot be resumed, so repeated attempts +/// against one IP are not a transfer pattern. +const UPLOAD_IP_MAX: u32 = 10; +const UPLOAD_IP_WINDOW_SECS: u32 = 60; + +/// Cap on the upstream's *response*. It is a small JSON descriptor; anything +/// approaching this is a misconfigured target rather than a large answer. +const MAX_UPSTREAM_RESPONSE_BYTES: usize = 64 * 1024; + +/// Headers copied from the inbound request to the upstream. Exactly one, and +/// the shortness is deliberate. +/// +/// On the response side the gateway relays an upstream it authenticated to. On +/// this side it relays an *anonymous* client into a host holding the org's +/// credential, so the default is to forward nothing. `content-length` is +/// dropped because the body is re-framed as it streams; `authorization` because +/// the redeemer's credential, if it has one, has no business on a connection +/// where ours is the one that belongs. +const FORWARDED_UPLOAD_HEADERS: [&str; 1] = ["content-type"]; + +/// Content types that mean "I did not state one". A byte route that sniffs its +/// input treats these as a gap, so passing them through would replace a sniff +/// with a wrong answer — the difference between a photo sent as a photo and a +/// photo sent as a document. +const GENERIC_CONTENT_TYPES: [&str; 3] = [ + "application/octet-stream", + "application/x-www-form-urlencoded", + "binary/octet-stream", +]; + +pub fn router() -> Router { + Router::new().route("/v1/uploads/{token}", post(redeem).put(redeem)) +} + +/// `body` must stay last: it is the only `FromRequest` extractor in the list, +/// and every `FromRequestParts` one has to precede it. +/// +/// Taking `axum::body::Body` rather than `Bytes` is also what keeps the body +/// unbuffered — the limited-body machinery lives inside the extractors that +/// collect, and this one does not collect. +async fn redeem( + State(state): State, + ReqExt(ext): ReqExt, + client_ip: ClientIp, + Path(token): Path, + headers: HeaderMap, + body: axum::body::Body, +) -> Response { + // Anonymous, so the global rate-limit middleware (which keys on the API-key + // prefix) skips it entirely. Throttle here on the shared store, the same + // way the download and magic-link endpoints do. + let ip = client_ip.0.as_deref().unwrap_or("unknown"); + let rl = state + .rate_limiter(&ext) + .check_and_increment( + &format!("up:redeem:ip:{ip}"), + UPLOAD_IP_MAX, + UPLOAD_IP_WINDOW_SECS, + ) + .await; + if !rl.allowed { + let retry_after = rl + .reset_at + .saturating_sub(crate::services::rate_limit::now_unix()); + return crate::error::AppError::RateLimited { + limit: rl.limit, + reset_at: rl.reset_at, + retry_after, + } + .into_response(); + } + + let not_found = || (StatusCode::NOT_FOUND, "unknown or expired token").into_response(); + + // `claim` matches on hash, unexpired and unconsumed in one statement, so + // from here an expired token, an unknown one and one whose single push + // already happened are the same answer. + let row = + match upload_token::claim(state.db(&ext), &deferred_download::hash_token(&token)).await { + Ok(Some(r)) => r, + Ok(None) => return not_found(), + Err(e) => { + tracing::error!(error = %e, "upload: token lookup failed"); + return (StatusCode::INTERNAL_SERVER_ERROR, "upload failed").into_response(); + } + }; + + let scope = OrgScope::new(row.org_id, state.db_pool(&ext)); + + // The permission decision was made and audited at mint time; this catches + // the identity being deleted in between, so outstanding tokens die with + // their principal rather than outliving it. + match scope.get_identity(row.identity_id).await { + Ok(Some(_)) => {} + Ok(None) => return not_found(), + Err(e) => { + tracing::error!(error = %e, "upload: identity lookup failed"); + return (StatusCode::INTERNAL_SERVER_ERROR, "upload failed").into_response(); + } + } + + match push(&state, &ext, &scope, &row, headers, body).await { + Ok(outcome) => { + log_upload(&scope, &row, ip, &outcome).await; + outcome.into_response() + } + Err(outcome) => { + log_upload(&scope, &row, ip, &outcome).await; + outcome.into_response() + } + } +} + +/// What a redemption did, so the audit row and the response describe the same +/// thing rather than being derived twice. +enum Pushed { + /// The upstream accepted the bytes and named what it stored. + Accepted { + status: u16, + descriptor: proxy_upload::UploadedDescriptor, + measured_bytes: u64, + measured_sha256: String, + }, + /// The bytes were pushed but do not match what was approved. The upstream's + /// reference is deliberately *not* carried here — see [`Pushed::refused`]. + Mismatch { + detail: String, + measured_bytes: u64, + measured_sha256: Option, + }, + /// The transfer never completed: too large, stalled, or the upstream + /// refused it. + Failed { status: StatusCode, detail: String }, + /// The upstream answered, but not with success. + Upstream { status: u16, detail: String }, +} + +impl Pushed { + /// A mismatch answers 422 and, crucially, without the stored reference. + /// + /// The hash is only known once the last byte has gone, so the bytes are + /// already upstream — the check detects, it cannot prevent. Withholding the + /// reference is what makes the detection worth having: nothing downstream + /// can name bytes it was never told the name of, so no send can act on + /// them. They linger upstream as an unreferenced orphan, which is a fact + /// worth stating rather than papering over. + fn refused(detail: String, measured_bytes: u64, measured_sha256: Option) -> Self { + Pushed::Mismatch { + detail, + measured_bytes, + measured_sha256, + } + } +} + +impl IntoResponse for Pushed { + fn into_response(self) -> Response { + match self { + Pushed::Accepted { + status, descriptor, .. + } => { + let body = serde_json::json!({ + "media_path": descriptor.media_path, + "sha256": descriptor.sha256, + "mime": descriptor.mime, + "size": descriptor.size_bytes, + "filename": descriptor.filename, + }); + ( + StatusCode::from_u16(status).unwrap_or(StatusCode::OK), + axum::Json(body), + ) + .into_response() + } + Pushed::Mismatch { detail, .. } => { + (StatusCode::UNPROCESSABLE_ENTITY, detail).into_response() + } + Pushed::Failed { status, detail } => (status, detail).into_response(), + Pushed::Upstream { status, detail } => ( + StatusCode::from_u16(status).unwrap_or(StatusCode::BAD_GATEWAY), + detail, + ) + .into_response(), + } + } +} + +/// Stream the body upstream and reconcile what came back against what was +/// approved. +/// +/// `Err` carries a [`Pushed`] too: both arms audit, and splitting the type +/// would mean two shapes to keep in step for no gain. +async fn push( + state: &AppState, + ext: &axum::http::Extensions, + scope: &OrgScope, + row: &UploadTokenRow, + headers: HeaderMap, + body: axum::body::Body, +) -> Result { + let request: ActionRequest = serde_json::from_value(row.request.clone()).map_err(|e| { + // Only reachable if a row was written by an incompatible build. + tracing::error!(token_id = %row.id, error = %e, "upload: stored request unreadable"); + Pushed::Failed { + status: StatusCode::INTERNAL_SERVER_ERROR, + detail: "upload failed".into(), + } + })?; + + let max_bytes = row.max_bytes.max(0) as u64; + // A stated length over the cap is refused before a byte moves. The + // mid-stream meter is still the real enforcement — a chunked body states + // no length, and a caller is free to state one that is untrue. + if let Some(len) = headers + .get(axum::http::header::CONTENT_LENGTH) + .and_then(|v| v.to_str().ok()) + .and_then(|v| v.parse::().ok()) + && len > max_bytes + { + return Err(Pushed::Failed { + status: StatusCode::PAYLOAD_TOO_LARGE, + detail: format!("upload of {len} bytes exceeds the {max_bytes}-byte limit"), + }); + } + + let resolved = + deferred_download::resolve_for_replay(state, scope, row.service_key.as_deref(), &request) + .await + .map_err(|e| Pushed::Failed { + status: StatusCode::BAD_GATEWAY, + detail: format!("upload credential resolution failed: {e}"), + })?; + + let mut out_headers = resolved.headers.clone(); + for name in FORWARDED_UPLOAD_HEADERS { + if let Some(value) = headers.get(name).and_then(|v| v.to_str().ok()) { + let v = value.trim(); + let generic = GENERIC_CONTENT_TYPES.iter().any(|g| { + v.split(';') + .next() + .unwrap_or(v) + .trim() + .eq_ignore_ascii_case(g) + }); + if !v.is_empty() && !generic { + out_headers.insert(name.to_string(), v.to_string()); + } + } + } + // Fall back to what the caller declared at mint time. Still nothing if it + // declared nothing — a byte route that sniffs would rather be told nothing + // than told something untrue. + if !out_headers + .keys() + .any(|k| k.eq_ignore_ascii_case("content-type")) + && let Some(mime) = row.declared_mime.as_deref() + { + out_headers.insert("content-type".to_string(), mime.to_string()); + } + + let url = with_filename( + &resolved.url, + row.filename_param.as_deref(), + row.declared_filename.as_deref(), + ); + + let (metered, meter) = proxy_upload::metered_body( + body, + proxy_upload::Limit { + max_bytes, + declared_bytes: row.declared_size_bytes.map(|n| n.max(0) as u64), + idle: std::time::Duration::from_millis(state.config.call_stream_idle_timeout_ms), + }, + ); + + let response = crate::services::http_caller::call_streaming_upload( + &state.http_client, + &request.method, + &url, + &out_headers, + metered, + ) + .await + .map_err(|e| { + // The transfer aborted, and only the meter knows why: the abort reaches + // us as a generic reqwest failure, so ask the meter rather than reading + // the error's text. Over the ceiling is the caller's problem (413); + // anything else is the upstream's (502). + let status = if meter.exceeded() { + StatusCode::PAYLOAD_TOO_LARGE + } else { + StatusCode::BAD_GATEWAY + }; + Pushed::Failed { + status, + detail: format!("upload failed: {e}"), + } + })?; + + let status = response.status().as_u16(); + let upstream_body = read_capped(response).await.map_err(|e| Pushed::Failed { + status: StatusCode::BAD_GATEWAY, + detail: format!("upload response unreadable: {e}"), + })?; + + if !(200..300).contains(&status) { + return Err(Pushed::Upstream { + status, + detail: truncate(&upstream_body, 512), + }); + } + + // `None` means the body was never read to completion — the upstream + // answered before consuming it. Fail closed: reporting a verification that + // did not happen is worse than not verifying, because it is believed. + let Some(measured) = meter.finish() else { + return Err(Pushed::refused( + "upload could not be verified: the upstream answered before the body was \ + fully sent, so the gateway never measured what it stored" + .into(), + 0, + None, + )); + }; + let measured_hex = measured.hex(); + + if let Some(declared) = row.declared_size_bytes + && measured.bytes != declared.max(0) as u64 + { + return Err(Pushed::refused( + format!( + "upload size mismatch: declared {declared} bytes, received {}", + measured.bytes + ), + measured.bytes, + Some(measured_hex), + )); + } + if let Some(declared) = row.declared_sha256.as_deref() + && !declared.eq_ignore_ascii_case(&measured_hex) + { + return Err(Pushed::refused( + format!("upload content mismatch: declared sha256 {declared}, received {measured_hex}"), + measured.bytes, + Some(measured_hex), + )); + } + + let spec = upload_result_spec(row); + let mut descriptor = read_descriptor( + &upstream_body, + spec.as_ref(), + std::time::Duration::from_millis(state.config.filter_timeout_ms), + ) + .await + .map_err(|detail| Pushed::Upstream { + status: StatusCode::BAD_GATEWAY.as_u16(), + detail, + })?; + + // Cross-check the upstream's own digest against ours. A disagreement means + // one of the two is describing different bytes, and guessing which would be + // exactly the wrong instinct: record neither as authoritative, refuse. + if let Some(upstream_hash) = descriptor.sha256.as_deref() + && !upstream_hash.eq_ignore_ascii_case(&measured_hex) + { + return Err(Pushed::refused( + format!( + "upstream stored a different object than was sent: it reports sha256 \ + {upstream_hash}, the gateway measured {measured_hex}" + ), + measured.bytes, + Some(measured_hex), + )); + } + + // Fill the gaps the upstream left from what was declared, never overwrite + // what it stated: it is the authority on what it stored. + if descriptor.sha256.is_none() { + descriptor.sha256 = Some(measured_hex.clone()); + } + if descriptor.size_bytes.is_none() { + descriptor.size_bytes = Some(measured.bytes as i64); + } + if descriptor.filename.is_none() { + descriptor.filename = row.declared_filename.clone(); + } + if descriptor.mime.is_none() { + descriptor.mime = row.declared_mime.clone(); + } + + // Best-effort, and deliberately not fatal on either. The push succeeded; + // failing it because a bookkeeping row would not write would trade a + // working upload for a tidier ledger. + if upload_token::complete( + state.db(ext), + row.id, + overslash_db::repos::upload_token::StoredDescriptor { + media_path: &descriptor.media_path, + sha256: descriptor.sha256.as_deref(), + size_bytes: descriptor.size_bytes, + mime: descriptor.mime.as_deref(), + filename: descriptor.filename.as_deref(), + }, + ) + .await + .is_err() + { + tracing::warn!(token_id = %row.id, "upload: completion not recorded"); + } + proxy_upload::record_uploaded( + state.db(ext), + row.org_id, + row.service_instance_id, + row.service_key.as_deref(), + &descriptor, + ) + .await; + + Ok(Pushed::Accepted { + status, + descriptor, + measured_bytes: measured.bytes, + measured_sha256: measured_hex, + }) +} + +/// The `result` jq block, recovered from the token's own row. +/// +/// Returns `None` when the template declared none, in which case the response +/// is read as the conventional flat descriptor. Kept as a function so the +/// "where does the spec come from at redemption time" question has one answer. +fn upload_result_spec(row: &UploadTokenRow) -> Option { + let raw = row.result_spec.as_ref()?; + match serde_json::from_value(raw.clone()) { + Ok(spec) => Some(spec), + Err(e) => { + // Only reachable if a row was written by an incompatible build. + // Falling back to the flat reader beats failing a push whose bytes + // already landed. + tracing::warn!(token_id = %row.id, error = %e, "upload: stored result spec unreadable"); + None + } + } +} + +/// Read the upstream's descriptor out of its response body. +/// +/// With a `result` block, through jq — reusing the disclosure runner rather +/// than a second jq harness, exactly as the download mint does, so the timeout, +/// the input ceiling and the `spawn_blocking` are not re-implemented. Without +/// one, through the conventional flat keys, so a target that already answers +/// `{media_path, sha256, mime, size, filename}` needs no declaration at all. +async fn read_descriptor( + body: &str, + spec: Option<&UploadResultSpec>, + filter_timeout: std::time::Duration, +) -> Result { + let value: serde_json::Value = serde_json::from_str(body) + .map_err(|e| format!("upload target did not answer with JSON: {e}"))?; + + let Some(spec) = spec else { + let get = |k: &str| { + value + .get(k) + .and_then(serde_json::Value::as_str) + .map(str::to_string) + }; + let media_path = get("media_path") + .filter(|s| !s.trim().is_empty()) + .ok_or_else(|| "upload target returned no media_path".to_string())?; + return Ok(proxy_upload::UploadedDescriptor { + media_path, + sha256: get("sha256"), + mime: get("mime"), + size_bytes: value.get("size").and_then(serde_json::Value::as_i64), + filename: get("filename"), + }); + }; + + let mut fields = vec![DisclosureField { + label: "media_path".into(), + filter: spec.media_path.clone(), + max_chars: None, + primary: false, + }]; + for (label, filter) in [ + ("sha256", spec.sha256.as_ref()), + ("mime", spec.mime.as_ref()), + ("size", spec.size.as_ref()), + ("filename", spec.filename.as_ref()), + ] { + if let Some(f) = filter { + fields.push(DisclosureField { + label: label.into(), + filter: f.clone(), + max_chars: None, + primary: false, + }); + } + } + let disclosed = crate::services::disclosure::run_disclosures(&fields, &value, filter_timeout) + .await + .map_err(|e| format!("upload result filters failed: {e}"))?; + // Joined on label because the runner drops zero-yield filters, so positions + // shift. Same reason the download mint joins on label. + let pick = |label: &str| -> Option { + disclosed + .iter() + .find(|d| d.label == label) + .and_then(|d| d.value.clone()) + .filter(|s| !s.trim().is_empty()) + }; + let media_path = pick("media_path") + .ok_or_else(|| "upload target returned no value for the declared media_path".to_string())?; + Ok(proxy_upload::UploadedDescriptor { + media_path, + sha256: pick("sha256"), + mime: pick("mime"), + size_bytes: pick("size").and_then(|s| s.trim().parse::().ok()), + filename: pick("filename"), + }) +} + +/// Append the token's declared filename under the parameter name the template +/// said the byte route takes it in. +/// +/// Both halves come off the token, and both have to be present: a route that +/// declares no `filename_param` takes none, so appending a guessed `filename=` +/// would put an unrecognized parameter on someone else's API. Percent-encoded +/// via `query_pairs_mut` rather than formatted in, because the value is +/// caller-supplied and a filename containing `&` would otherwise become a +/// second parameter. +fn with_filename(url: &str, param: Option<&str>, filename: Option<&str>) -> String { + let (Some(param), Some(name)) = ( + param.map(str::trim).filter(|s| !s.is_empty()), + filename.map(str::trim).filter(|s| !s.is_empty()), + ) else { + return url.to_string(); + }; + match url::Url::parse(url) { + Ok(mut u) => { + u.query_pairs_mut().append_pair(param, name); + u.to_string() + } + // Unreachable — the URL was validated at mint. Sending the file + // unnamed beats failing a push that would otherwise work. + Err(_) => url.to_string(), + } +} + +/// Read a bounded response body. The answer is a small JSON descriptor; +/// anything near the cap is a misconfigured target rather than a big answer. +async fn read_capped(response: reqwest::Response) -> Result { + use futures_util::StreamExt as _; + let mut collected: Vec = Vec::new(); + let mut stream = response.bytes_stream(); + while let Some(chunk) = stream.next().await { + let chunk = chunk.map_err(|e| e.to_string())?; + collected.extend_from_slice(&chunk); + if collected.len() > MAX_UPSTREAM_RESPONSE_BYTES { + return Err(format!( + "upload target answered with more than {MAX_UPSTREAM_RESPONSE_BYTES} bytes" + )); + } + } + Ok(String::from_utf8_lossy(&collected).into_owned()) +} + +/// Crop to `max` bytes without splitting a codepoint. +fn truncate(s: &str, max: usize) -> String { + if s.len() <= max { + return s.to_string(); + } + s[..s.floor_char_boundary(max)].to_string() +} + +/// Audit the redemption. Sibling of `action.downloaded`: like that row the body +/// is never buffered so it cannot be captured, and like it the row exists so a +/// deferred push is not a hole in the trail between "agent was allowed to +/// upload" and "bytes entered the service". +/// +/// Declared *and* measured are both recorded, because a divergence between them +/// is the whole signal — the row is the only place it survives. +async fn log_upload(scope: &OrgScope, row: &UploadTokenRow, ip: &str, outcome: &Pushed) { + let (status, is_error, detail, bytes, sha) = match outcome { + Pushed::Accepted { + status, + measured_bytes, + measured_sha256, + .. + } => ( + Some(*status), + false, + None, + Some(*measured_bytes), + Some(measured_sha256.clone()), + ), + Pushed::Mismatch { + detail, + measured_bytes, + measured_sha256, + } => ( + Some(422), + true, + Some(detail.clone()), + Some(*measured_bytes), + measured_sha256.clone(), + ), + Pushed::Failed { status, detail } => ( + Some(status.as_u16()), + true, + Some(detail.clone()), + None, + None, + ), + Pushed::Upstream { status, detail } => { + (Some(*status), true, Some(detail.clone()), None, None) + } + }; + let stored_path = match outcome { + Pushed::Accepted { descriptor, .. } => Some(descriptor.media_path.clone()), + _ => None, + }; + let _ = scope + .clone() + .log_audit(AuditEntry { + org_id: row.org_id, + identity_id: Some(row.identity_id), + action: "action.uploaded", + resource_type: row.service_key.as_deref(), + resource_id: None, + detail: serde_json::json!({ + "runtime": "upload", + "service": row.service_key, + "action": row.action_key, + "status_code": status, + "is_error": is_error, + "error": detail, + "declared_sha256": row.declared_sha256, + "declared_size_bytes": row.declared_size_bytes, + "declared_filename": row.declared_filename, + "declared_mime": row.declared_mime, + "measured_sha256": sha, + "measured_size_bytes": bytes, + "stored_media_path": stored_path, + "request": { "skipped": "streamed" }, + }), + description: Some("Upload capability redeemed"), + ip_address: Some(ip), + }) + .await; +} + +#[cfg(test)] +mod with_filename_tests { + use super::with_filename; + + const URL: &str = "https://wa.example.com/media"; + + #[test] + fn uses_the_parameter_name_the_template_declared() { + let out = with_filename(URL, Some("name"), Some("clip.mp4")); + assert_eq!(out, "https://wa.example.com/media?name=clip.mp4"); + } + + /// A route that takes no filename parameter must not be handed a guessed + /// one — the declaration is what says the parameter exists at all. + #[test] + fn appends_nothing_when_the_template_declares_no_parameter() { + assert_eq!(with_filename(URL, None, Some("clip.mp4")), URL); + } + + #[test] + fn appends_nothing_when_no_filename_was_declared() { + assert_eq!(with_filename(URL, Some("filename"), None), URL); + } + + /// The value is caller-supplied, so it is encoded rather than formatted in: + /// an `&` would otherwise start a second parameter on the byte route. + #[test] + fn a_filename_cannot_smuggle_a_second_parameter() { + let out = with_filename(URL, Some("filename"), Some("a&evil=1.txt")); + assert_eq!( + out, + "https://wa.example.com/media?filename=a%26evil%3D1.txt" + ); + } +} diff --git a/crates/overslash-api/src/services/action_caller.rs b/crates/overslash-api/src/services/action_caller.rs index 39aeac3b..3a636ed2 100644 --- a/crates/overslash-api/src/services/action_caller.rs +++ b/crates/overslash-api/src/services/action_caller.rs @@ -145,6 +145,26 @@ pub struct StoredMcpCall { /// and a replayed tool call is as entitled to a `next` as an inline one. #[serde(default, skip_serializing_if = "Option::is_none")] pub pagination: Option, + /// The action's upload declaration, when it has one. + /// + /// Carried on the payload for the same reason `pagination` is: replay + /// resolves nothing. It holds a URL and a tool name, not an action key, so + /// it cannot look the declaration back up — and an upload action that + /// arrived here without its spec would be dispatched as a tool call the + /// upstream has never heard of. A `risk: write` upload reaches this path + /// *first* for any gated agent, so the omission would not be an edge case. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub upload: Option, + /// Which instance stores what an upload pushes. Needed to scope the media + /// ledger row, which replay cannot otherwise derive. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub service_instance_id: Option, + /// Carried for the ledger and audit rows. `serde(default)` throughout so + /// rows written before these fields existed still parse. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub service_key: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub action_key: Option, } /// Replay payload for a platform-runtime approval. Stored on diff --git a/crates/overslash-api/src/services/deferred_download.rs b/crates/overslash-api/src/services/deferred_download.rs index 48bc74ae..fe323748 100644 --- a/crates/overslash-api/src/services/deferred_download.rs +++ b/crates/overslash-api/src/services/deferred_download.rs @@ -121,12 +121,7 @@ pub async fn mint( ext: &axum::http::Extensions, m: Mint<'_>, ) -> Result { - // 32 random bytes → URL-safe token; only its SHA-256 is stored. Same - // construction as magic-link tokens. - let mut buf = [0u8; 32]; - rand::rng().fill(&mut buf); - let raw_token = base64::Engine::encode(&base64::engine::general_purpose::URL_SAFE_NO_PAD, buf); - let token_hash = Sha256::digest(raw_token.as_bytes()).to_vec(); + let (raw_token, token_hash) = new_token(); let request = m .request @@ -187,6 +182,21 @@ pub fn hash_token(raw: &str) -> Vec { Sha256::digest(raw.as_bytes()).to_vec() } +/// Mint a fresh capability token: the raw value to put in a URL, and the hash +/// to put in a row. +/// +/// 32 random bytes, URL-safe base64, only the SHA-256 stored — the same +/// construction as magic-link tokens. Shared by both byte directions so +/// "the raw token exists only in the URL" is one fact rather than a convention +/// two call sites happen to observe. +pub fn new_token() -> (String, Vec) { + let mut buf = [0u8; 32]; + rand::rng().fill(&mut buf); + let raw = base64::Engine::encode(&base64::engine::general_purpose::URL_SAFE_NO_PAD, buf); + let hash = hash_token(&raw); + (raw, hash) +} + /// Header names that carry a credential rather than describing the request. /// /// Not exhaustive, and can't be — an upstream is free to read a secret out of @@ -238,9 +248,14 @@ pub fn reject_inline_credentials(request: &ActionRequest) -> Result<(), AppError /// ref whose `name` *is* the vault secret name, with the prefix carried /// literally. No new resolution path, no second place for credential handling /// to drift. -pub fn bearer_request(url: String, secret_name: Option<&str>) -> ActionRequest { +/// +/// `method` is a parameter rather than a constant because the byte routes come +/// in both directions: a download is a GET, an upload a POST or PUT at the same +/// origin behind the same bearer. `body` stays `None` either way — an upload's +/// bytes are not in the row, they arrive when the capability is redeemed. +pub fn bearer_request(method: &str, url: String, secret_name: Option<&str>) -> ActionRequest { ActionRequest { - method: "GET".into(), + method: method.into(), url, headers: HashMap::new(), body: None, @@ -273,16 +288,7 @@ pub async fn open_upstream( service_key: Option<&str>, request: &ActionRequest, ) -> Result { - let secret_values = crate::services::action_caller::resolve_credential_values( - state, - scope, - service_key, - request, - ) - .await?; - let (resolved_url, resolved_headers) = - inject_secrets(request, &secret_values).map_err(|e| AppError::BadRequest(e.to_string()))?; - let resolved_url = state.config.apply_base_overrides(&resolved_url); + let Resolved { url, headers } = resolve_for_replay(state, scope, service_key, request).await?; // The deployment default, not a D56-resolved budget: a token redemption // has no caller-supplied `timeout_ms` and no action key to read the @@ -294,8 +300,8 @@ pub async fn open_upstream( http_caller::call_streaming( &state.http_client, &request.method, - &resolved_url, - &resolved_headers, + &url, + &headers, request.body.as_deref(), std::time::Duration::from_millis(state.config.call_timeout_ms), ) @@ -303,6 +309,81 @@ pub async fn open_upstream( .map_err(|e| AppError::BadGateway(format!("download upstream request failed: {e}"))) } +/// A stored request with its credentials resolved as they stand right now. +pub struct Resolved { + pub url: String, + pub headers: HashMap, +} + +/// Re-resolve a stored request's credentials against the current vault. +/// +/// Split out of [`open_upstream`] because the two byte directions share this +/// half and nothing else: a download dials and streams the response, an upload +/// dials and streams the *request*. Keeping one credential path means a +/// rotated secret is picked up and a deleted one fails closed identically in +/// both directions — the alternative is two resolutions that agree until +/// someone changes one of them. +pub async fn resolve_for_replay( + state: &AppState, + scope: &OrgScope, + service_key: Option<&str>, + request: &ActionRequest, +) -> Result { + let secret_values = crate::services::action_caller::resolve_credential_values( + state, + scope, + service_key, + request, + ) + .await?; + let (url, headers) = + inject_secrets(request, &secret_values).map_err(|e| AppError::BadRequest(e.to_string()))?; + Ok(Resolved { + url: state.config.apply_base_overrides(&url), + headers, + }) +} + +/// Resolve a byte location against the origin that served the tool call, +/// refusing anything that would send the credential elsewhere. +/// +/// Accepts a path (`/media/abc`) or an absolute URL on the same origin. +/// Everything else — a different host, a different scheme, a non-URL — is an +/// error rather than a best-effort request. +/// +/// # Why the origin is pinned +/// +/// Both directions attach the service instance's credential to whatever this +/// returns. On a download the untrusted input is the MCP server's own response, +/// so an unconstrained URL lets a hostile or compromised server name +/// `http://169.254.169.254/…` and have the gateway deliver the bearer to it. On +/// an upload the untrusted input is the *template*, which is a lower-trust +/// surface once orgs author their own and a higher-consequence one, since a +/// redemption sends the caller's bytes as well as the credential. Requiring the +/// route to live on the host we just talked to closes both, and costs nothing: +/// "the bytes are on that host" is the actual contract. +pub fn resolve_same_origin(base_url: &str, raw: &str) -> Result { + let base = url::Url::parse(base_url).map_err(|e| { + AppError::BadGateway(format!( + "service instance url `{base_url}` is not a URL: {e}" + )) + })?; + let joined = base + .join(raw) + .map_err(|e| AppError::BadGateway(format!("byte location `{raw}` is not a URL: {e}")))?; + let same_origin = joined.scheme() == base.scheme() + && joined.host_str() == base.host_str() + && joined.port_or_known_default() == base.port_or_known_default(); + if !same_origin { + return Err(AppError::BadGateway(format!( + "byte location `{raw}` points outside the MCP server's origin ({}); \ + refusing to send this service's credential to another host", + base.origin().ascii_serialization() + ))); + } + Ok(joined.to_string()) +} + /// Pipe an upstream response straight through to the caller: raw bytes, the /// upstream status, and the [`FORWARDED_HEADERS`] allowlist. /// @@ -332,3 +413,86 @@ pub fn stream_through(upstream: reqwest::Response, idle: std::time::Duration) -> .body(body) .expect("status + allowlisted headers always build a valid response") } + +/// Moved here with `resolve_same_origin` itself, unchanged: passing after the +/// move is the evidence the upload path inherited exactly the download path's +/// refusals rather than a re-derived approximation of them. +#[cfg(test)] +mod same_origin_tests { + use super::resolve_same_origin; + + const MCP: &str = "https://wa.example.com:8443/mcp"; + + /// The shape the shipped WhatsApp template uses: the tool hands back a + /// path, which resolves against the server that served the tool call. + #[test] + fn a_relative_path_resolves_against_the_mcp_server() { + let out = resolve_same_origin(MCP, "/media/abc123").unwrap(); + assert_eq!(out, "https://wa.example.com:8443/media/abc123"); + } + + #[test] + fn an_absolute_same_origin_url_is_accepted_verbatim() { + let out = resolve_same_origin(MCP, "https://wa.example.com:8443/media/abc123?x=1").unwrap(); + assert_eq!(out, "https://wa.example.com:8443/media/abc123?x=1"); + } + + /// The control that matters. The URL comes from the MCP server's own + /// response and the deferred fetch attaches that instance's credential — + /// so a hostile or compromised server naming another host must not be able + /// to have the gateway deliver the bearer to it. + #[test] + fn a_different_host_is_refused() { + let err = resolve_same_origin(MCP, "http://169.254.169.254/latest/meta-data/") + .expect_err("off-origin must be refused"); + assert!( + format!("{err:?}").contains("outside the MCP server's origin"), + "{err:?}" + ); + } + + #[test] + fn a_different_scheme_on_the_same_host_is_refused() { + // Downgrading https→http on the same host would put the credential on + // the wire in plaintext, so origin comparison includes the scheme. + resolve_same_origin(MCP, "http://wa.example.com:8443/media/abc") + .expect_err("scheme change must be refused"); + } + + #[test] + fn a_different_port_on_the_same_host_is_refused() { + resolve_same_origin(MCP, "https://wa.example.com:9999/media/abc") + .expect_err("port change must be refused"); + } + + /// Default ports compare equal to their explicit form — otherwise a server + /// configured as `https://host/mcp` returning `https://host:443/…` would + /// be rejected for no reason. + #[test] + fn an_implicit_default_port_matches_its_explicit_form() { + resolve_same_origin( + "https://wa.example.com/mcp", + "https://wa.example.com:443/media/abc", + ) + .expect("443 is https's default port"); + } + + #[test] + fn a_protocol_relative_url_cannot_smuggle_a_new_host() { + // `//evil.com/x` inherits the scheme but not the host — the classic + // way past a naive "starts with http" check. + resolve_same_origin(MCP, "//evil.com/media/abc") + .expect_err("protocol-relative host swap must be refused"); + } + + /// The guarantee is "never leaves the origin", not "rejects anything odd". + /// A non-URL string joins as a relative path and stays on the MCP host, so + /// the worst case is a 404 upstream — and the control characters that could + /// otherwise smuggle a second request get percent-encoded on the way. + #[test] + fn a_non_url_string_stays_on_the_origin_and_is_escaped() { + let out = resolve_same_origin(MCP, "not a url at all\n").unwrap(); + assert!(out.starts_with("https://wa.example.com:8443/"), "{out}"); + assert!(!out.contains('\n') && !out.contains(' '), "{out}"); + } +} diff --git a/crates/overslash-api/src/services/http_caller.rs b/crates/overslash-api/src/services/http_caller.rs index 0cae2cf9..726c133b 100644 --- a/crates/overslash-api/src/services/http_caller.rs +++ b/crates/overslash-api/src/services/http_caller.rs @@ -54,6 +54,19 @@ pub enum CallError { Timeout { timeout_ms: u64 }, } +/// What goes on the wire as the request body. +/// +/// An enum rather than an `Option<&str>` alongside an out-of-band +/// `reqwest::Body` because the *default content type* differs per arm, and that +/// default is a contract rather than a convenience: JSON for a text body, and +/// nothing at all for a stream, since a byte route that sniffs its input treats +/// a stated `application/json` as a claim rather than a gap. +enum OutgoingBody<'a> { + None, + Text(&'a str), + Stream(reqwest::Body), +} + /// Build a reqwest request from the given parameters. /// /// `total_timeout` is `Some` only for the buffered path — see the module docs @@ -63,7 +76,7 @@ fn build_request( method: &str, url: &str, headers: &HashMap, - body: Option<&str>, + body: OutgoingBody<'_>, total_timeout: Option, ) -> reqwest::RequestBuilder { let method = method @@ -75,14 +88,23 @@ fn build_request( builder = builder.header(k.as_str(), v.as_str()); } - if let Some(body) = body { - if !headers - .keys() - .any(|k| k.eq_ignore_ascii_case("content-type")) - { - builder = builder.header("Content-Type", "application/json"); + match body { + OutgoingBody::None => {} + OutgoingBody::Text(body) => { + if !headers + .keys() + .any(|k| k.eq_ignore_ascii_case("content-type")) + { + builder = builder.header("Content-Type", "application/json"); + } + builder = builder.body(body.to_string()); + } + // No content-type default. The caller decides — and on the upload path + // it deliberately sends none when it has none, so the upstream sniffs + // the bytes instead of being told something untrue about them. + OutgoingBody::Stream(body) => { + builder = builder.body(body); } - builder = builder.body(body.to_string()); } if let Some(t) = total_timeout { @@ -109,10 +131,17 @@ pub async fn call( let start = Instant::now(); let timeout_ms = timeout.as_millis() as u64; - let response = build_request(client, method, url, headers, body, Some(timeout)) - .send() - .await - .map_err(|e| map_reqwest_timeout(e, timeout_ms))?; + let response = build_request( + client, + method, + url, + headers, + to_outgoing(body), + Some(timeout), + ) + .send() + .await + .map_err(|e| map_reqwest_timeout(e, timeout_ms))?; let status_code = response.status().as_u16(); // Fold rather than collect: a `HashMap` built straight from the iterator @@ -197,7 +226,7 @@ pub async fn call_streaming( let timeout_ms = timeout.as_millis() as u64; match tokio::time::timeout( timeout, - build_request(client, method, url, headers, body, None).send(), + build_request(client, method, url, headers, to_outgoing(body), None).send(), ) .await { @@ -206,6 +235,53 @@ pub async fn call_streaming( } } +/// The `Option<&str>` both public buffered/streamed entry points still take, +/// as the enum the builder now speaks. +fn to_outgoing(body: Option<&str>) -> OutgoingBody<'_> { + match body { + Some(b) => OutgoingBody::Text(b), + None => OutgoingBody::None, + } +} + +/// Send a **streamed request body** and return the upstream response. +/// +/// The inbound mirror of [`call_streaming`], and the only path that can carry a +/// body too large to hold: every other caller here materializes an owned +/// `String`, which is fine for a JSON payload and impossible for a hundred +/// megabytes of file. +/// +/// # Why there is no timeout parameter +/// +/// [`call_streaming`] bounds its header phase with a `tokio` timeout because on +/// a GET, `send()` resolves as soon as the response headers arrive. On an +/// upload it does not: `send()` waits for the last byte of the *request* body, +/// so a deadline here is a cap on total transfer duration, and a legitimate +/// large push over a slow link dies at it. That is the trap the module docs +/// describe for response streaming, inverted, so the same answer applies — +/// liveness is bounded per chunk, not in total. The caller meters the body (see +/// [`crate::services::proxy_upload::metered_body`]) and that meter carries both +/// the idle guard and the byte ceiling. +pub async fn call_streaming_upload( + client: &reqwest::Client, + method: &str, + url: &str, + headers: &HashMap, + body: reqwest::Body, +) -> Result { + build_request( + client, + method, + url, + headers, + OutgoingBody::Stream(body), + None, + ) + .send() + .await + .map_err(CallError::Request) +} + /// Wrap a streamed response body so a *stall* is fatal but slowness is not. /// /// Each chunk gets its own `idle` budget. A transfer that keeps delivering diff --git a/crates/overslash-api/src/services/mod.rs b/crates/overslash-api/src/services/mod.rs index 3b2a1113..f38e9330 100644 --- a/crates/overslash-api/src/services/mod.rs +++ b/crates/overslash-api/src/services/mod.rs @@ -46,6 +46,7 @@ pub mod platform_secrets; pub mod platform_services; pub mod platform_templates; pub mod principals; +pub mod proxy_upload; pub mod rate_limit; pub mod resolve_cache; pub mod response_filter; diff --git a/crates/overslash-api/src/services/param_resolver.rs b/crates/overslash-api/src/services/param_resolver.rs index da70bbb8..67991e18 100644 --- a/crates/overslash-api/src/services/param_resolver.rs +++ b/crates/overslash-api/src/services/param_resolver.rs @@ -87,6 +87,81 @@ fn project( (display, canonical) } +/// Resolve display names for params that declare `source:` — answered from the +/// gateway's own records rather than from the upstream. +/// +/// Runtime-agnostic on purpose, and called once after both runtime forks rather +/// than inside either: the ledger is a database read, so it shares nothing with +/// the HTTP fan-out's client or the MCP fork's pinned-client preamble. Folding +/// it into one of them would tie a lookup that needs neither to whichever +/// runtime happened to host it. +/// +/// Not cached. The ledger *is* the cache — it is local, org-scoped, and +/// authoritative — so a second layer in front of it would add staleness and +/// nothing else. +pub async fn resolve_ledger_params( + pool: &sqlx::PgPool, + org_id: uuid::Uuid, + service_instance_id: Option, + action: &ServiceAction, + params: &HashMap, +) -> ResolvedParams { + use overslash_core::types::ResolveSource; + + let mut out = ResolvedParams::default(); + for (name, param) in &action.params { + let Some(resolver) = param.resolve.as_ref() else { + continue; + }; + if resolver.source != Some(ResolveSource::Media) { + continue; + } + let Some(reference) = params + .get(name) + .and_then(serde_json::Value::as_str) + .map(str::trim) + .filter(|s| !s.is_empty()) + else { + continue; + }; + let row = match overslash_db::repos::media_descriptor::find( + pool, + org_id, + service_instance_id, + reference, + ) + .await + { + Ok(Some(row)) => row, + // A reference the gateway never handled is simply absent, and the + // disclose filter falls back to the raw argument. That fallback is + // lossless — the reviewer still sees exactly the string the call + // will send — so a miss is never worth a warning. + Ok(None) => continue, + Err(e) => { + tracing::warn!(error = %e, "media ledger lookup failed"); + continue; + } + }; + // The same projection every other resolver goes through, over a body + // shaped like the descriptor a template already knows how to address. + let body = serde_json::json!({ + "media_path": row.media_path, + "sha256": row.sha256, + "mime": row.mime, + "size": row.size_bytes, + "filename": row.filename, + "source": row.source, + }); + // `scope` is refused on a `source` resolver at parse time, so the + // canonical half is always `None` here — a ledger lookup describes + // bytes, it does not name a principal to scope a grant to. + let (display, _canonical) = project(resolver, &body); + out.insert_resolution(name, display.as_deref(), None); + } + out +} + /// Resolve display names for HTTP-runtime action params that declare `get:`. /// /// Makes concurrent GET requests to the same service host using the already- diff --git a/crates/overslash-api/src/services/platform_services/mod.rs b/crates/overslash-api/src/services/platform_services/mod.rs index 22e2cded..016c4052 100644 --- a/crates/overslash-api/src/services/platform_services/mod.rs +++ b/crates/overslash-api/src/services/platform_services/mod.rs @@ -189,6 +189,7 @@ mod test_fixtures { disabled: false, request_body: None, download: None, + upload: None, }, ); } diff --git a/crates/overslash-api/src/services/proxy_upload.rs b/crates/overslash-api/src/services/proxy_upload.rs new file mode 100644 index 00000000..6ac744ef --- /dev/null +++ b/crates/overslash-api/src/services/proxy_upload.rs @@ -0,0 +1,463 @@ +//! Proxy uploads: minting a capability to push bytes, and metering them on the +//! way through. +//! +//! The inbound half of the byte path. [`deferred_download`] hands a caller a +//! URL to *fetch* from; this hands a caller a URL to *push* to, and the +//! redemption streams those bytes into the service with the credential +//! re-resolved from the vault at that moment. Same trust model, opposite +//! direction: the process holding the token is deliberately not the caller and +//! holds none of the caller's credentials. +//! +//! # The organizing invariant +//! +//! **Everything the reviewer approved is fixed at mint time; the anonymous +//! redemption leg contributes only bytes.** The filename, the target route, the +//! ceiling and the declared content all come off the token, never off the +//! request that redeems it. That is what makes the approval mean something: a +//! redeemer who could choose the filename could get a reviewer's yes to +//! `notes.txt` and push `payroll.xlsx`. +//! +//! # What verification can and cannot do +//! +//! A declared size is *prevented* from being exceeded — the meter cuts the +//! transfer mid-stream, so the upstream never sees the overage. A declared +//! hash can only be *detected*, because a hash is not known until the last +//! byte, which is after the upstream already has them. On a mismatch the +//! redemption fails and the upstream's reference is never handed back, so +//! nothing downstream can name those bytes; the bytes themselves may linger +//! upstream as an unreferenced orphan. Saying this plainly matters more than +//! the check reading as airtight, because an operator who believes a mismatch +//! is impossible will not think about the orphan. +//! +//! [`deferred_download`]: crate::services::deferred_download + +use std::collections::HashMap; +use std::sync::{Arc, Mutex}; +use std::time::Duration; + +use futures_util::StreamExt; +use overslash_core::types::{ActionResult, DownloadAuth, McpAuth, UploadSpec}; +use overslash_db::repos::upload_token::{self, NewUploadToken}; +use serde_json::json; +use sha2::{Digest, Sha256}; +use uuid::Uuid; + +use crate::{AppState, error::AppError, services::deferred_download}; + +/// What the caller gets back in place of a place to put bytes. +#[derive(Debug, serde::Serialize)] +pub struct UploadDescriptor { + /// Absolute URL to push to. Carries the raw token in its path. + pub upload_url: String, + /// The verb the URL accepts. + pub method: &'static str, + /// RFC 3339. After this the URL 404s. + pub expires_at: String, + /// Hard ceiling for this push, already clamped to the deployment limit. + pub max_bytes: u64, + /// A ready-to-run invocation. Present because the whole point of this + /// shape is that the bytes never enter an agent's context — so the useful + /// thing to hand back is the command that moves them without doing so. + pub hint: String, +} + +// --------------------------------------------------------------------------- +// Minting +// --------------------------------------------------------------------------- + +/// Everything a mint reads. A struct rather than an eleven-argument function, +/// for the reason the download side gives: adjacent `Option<&str>`s swap +/// silently at a call site with no type error to catch it. +pub(crate) struct Mint<'a> { + pub org_id: Uuid, + pub identity_id: Uuid, + pub service_instance_id: Option, + pub service_key: Option<&'a str>, + pub action_key: Option<&'a str>, + /// The resolved MCP server URL. The byte route resolves against this + /// origin and no other. + pub mcp_url: &'a str, + pub mcp_auth: &'a McpAuth, + pub spec: &'a UploadSpec, + /// The call's arguments, which is where the caller states what it intends + /// to push. + pub arguments: &'a serde_json::Value, +} + +/// Mint a capability for pushing bytes, instead of dispatching a tool call. +/// +/// Called only once a dispatch site has found an upload block, which is why it +/// takes `spec` rather than an `Option` — the two sites branch, this mints. +/// +/// There are two such sites and missing either is silent: an ungated call goes +/// through the inline executor, while a *gated* one — the first call any +/// permission-checked agent makes — is replayed from a stored payload after +/// approval. An interception present only on the inline path would work +/// perfectly until the moment an approval was involved. +pub(crate) async fn intercept_mint( + state: &AppState, + ext: &axum::http::Extensions, + m: Mint<'_>, +) -> Result { + let started = std::time::Instant::now(); + let spec = m.spec; + + let url = deferred_download::resolve_same_origin(m.mcp_url, &spec.path)?; + + // Same credential rules as a deferred download, deliberately reusing its + // vocabulary: an OAuth bearer is minted live from the caller's connection + // and is not persistable, so a redemption could not re-present it. + let secret_name = match (m.mcp_auth, spec.auth) { + (_, DownloadAuth::None) => None, + (McpAuth::Bearer { secret_name }, DownloadAuth::Inherit) => secret_name.as_deref(), + (McpAuth::None, DownloadAuth::Inherit) => None, + (McpAuth::OAuth { .. }, DownloadAuth::Inherit) => { + return Err(AppError::BadRequest( + "uploads are not supported for OAuth-authenticated MCP services yet; \ + the deferred push cannot re-mint an OAuth bearer" + .into(), + )); + } + }; + + let declared = Declared::read(m.arguments)?; + // A template may lower the deployment ceiling, never raise it. Both are + // real limits — the deployment's bounds what this process will move, the + // template's bounds what the upstream will accept. + let max_bytes = spec + .max_bytes + .unwrap_or(state.config.upload_max_bytes) + .min(state.config.upload_max_bytes); + if let Some(size) = declared.size_bytes + && size as u64 > max_bytes + { + return Err(AppError::BadRequest(format!( + "declared size_bytes ({size}) exceeds the {max_bytes}-byte limit for this upload" + ))); + } + + let request = deferred_download::bearer_request(spec.method.as_str(), url, secret_name); + deferred_download::reject_inline_credentials(&request)?; + + let (raw_token, token_hash) = deferred_download::new_token(); + + let row = upload_token::create( + state.db(ext), + NewUploadToken { + token_hash: &token_hash, + org_id: m.org_id, + identity_id: m.identity_id, + service_instance_id: m.service_instance_id, + service_key: m.service_key, + action_key: m.action_key, + request: serde_json::to_value(&request) + .map_err(|e| AppError::Internal(format!("upload request not serializable: {e}")))?, + credential_ref: json!({}), + declared_sha256: declared.sha256.as_deref(), + declared_size_bytes: declared.size_bytes, + declared_mime: declared.mime.as_deref(), + declared_filename: declared.filename.as_deref(), + max_bytes: max_bytes as i64, + filename_param: spec.filename_param.as_deref(), + // Carried on the row because redemption cannot resolve it: it + // holds a token, not an action key. Same constraint that puts + // `pagination` on an approval's replay payload. + result_spec: spec + .result + .as_ref() + .and_then(|r| serde_json::to_value(r).ok()), + ttl_secs: state.config.upload_token_ttl_secs, + }, + ) + .await + .map_err(|e| AppError::Internal(format!("could not mint upload token: {e}")))?; + + let base = state.config.public_url.trim_end_matches('/'); + let upload_url = format!("{base}/v1/uploads/{raw_token}"); + let expires_at = row + .expires_at + .format(&time::format_description::well_known::Rfc3339) + .unwrap_or_default(); + + let content_type = declared + .mime + .clone() + .unwrap_or_else(|| "application/octet-stream".to_string()); + let descriptor = UploadDescriptor { + method: spec.method.as_str(), + hint: format!( + "curl -sSf -X {} --data-binary @FILE -H 'Content-Type: {}' '{}'", + spec.method.as_str(), + content_type, + upload_url + ), + upload_url, + expires_at, + max_bytes, + }; + + Ok(ActionResult { + status_code: 200, + headers: HashMap::from([("content-type".to_string(), "application/json".to_string())]), + body: serde_json::to_string(&descriptor).unwrap_or_default(), + duration_ms: started.elapsed().as_millis() as u64, + filtered_body: None, + }) +} + +/// What the caller said it was about to push. +/// +/// Every field is optional, and the asymmetry in how they are treated is the +/// point: absent means "unverified", while present means "enforced". A caller +/// that declares nothing gets an approval reading "some bytes, to be chosen +/// later", which is a weaker thing to approve and should look like one. +struct Declared { + sha256: Option, + size_bytes: Option, + mime: Option, + filename: Option, +} + +impl Declared { + fn read(arguments: &serde_json::Value) -> Result { + let get = |k: &str| { + arguments + .get(k) + .and_then(serde_json::Value::as_str) + .map(str::trim) + .filter(|s| !s.is_empty()) + .map(str::to_string) + }; + // Normalized to lowercase so a caller writing an uppercase digest is + // not told its own file's hash mismatched. + let sha256 = get("sha256").map(|s| s.to_ascii_lowercase()); + if let Some(h) = &sha256 + && (h.len() != 64 || !h.bytes().all(|b| b.is_ascii_hexdigit())) + { + return Err(AppError::BadRequest( + "sha256 must be 64 hexadecimal characters".into(), + )); + } + let size_bytes = match arguments.get("size_bytes") { + None | Some(serde_json::Value::Null) => None, + Some(v) => match v.as_i64().filter(|n| *n > 0) { + Some(n) => Some(n), + None => { + return Err(AppError::BadRequest( + "size_bytes must be a positive integer".into(), + )); + } + }, + }; + Ok(Declared { + sha256, + size_bytes, + mime: get("mime"), + filename: get("filename"), + }) + } +} + +// --------------------------------------------------------------------------- +// Metering +// --------------------------------------------------------------------------- + +/// What a redemption is allowed to accept, all of it fixed at mint time. +pub struct Limit { + /// Hard ceiling for this token, already clamped to the deployment's. + pub max_bytes: u64, + /// Declared at mint. When set the transfer must be *exactly* this long: + /// over is cut mid-stream, under is caught at end of stream. + pub declared_bytes: Option, + /// Per-chunk stall budget. Bounds liveness without bounding duration, so a + /// slow but progressing transfer is not punished for being large. + pub idle: Duration, +} + +/// What a completed pass over the body measured. +#[derive(Debug, Clone, Copy)] +pub struct Measured { + pub bytes: u64, + pub sha256: [u8; 32], +} + +impl Measured { + /// Lowercase hex, the spelling every upstream states its digests in. + pub fn hex(&self) -> String { + self.sha256.iter().fold(String::new(), |mut acc, b| { + use std::fmt::Write as _; + let _ = write!(acc, "{b:02x}"); + acc + }) + } +} + +#[derive(Default)] +struct Tally { + bytes: u64, + hasher: Sha256, + finished: bool, + exceeded: bool, +} + +/// Handle onto a live meter. +pub struct MeterHandle(Arc>); + +impl MeterHandle { + /// Whether the transfer was cut for going past its ceiling. + /// + /// Read from the meter rather than matched against the transport error's + /// text: the abort reaches the caller as a generic reqwest failure with our + /// message buried an unknown number of `source()` hops down, and a status + /// code that depends on string matching is one dependency-bump away from + /// silently becoming a 502. + pub fn exceeded(&self) -> bool { + self.0.lock().map(|t| t.exceeded).unwrap_or(false) + } + + /// What the body measured, or `None` if it was never read to completion. + /// + /// The `None` case is not a formality: an upstream can answer before + /// consuming the request body (an early error, a 100-continue it declines), + /// and the tally at that moment describes a prefix. Callers must treat + /// `None` as "unverified" and fail closed — reporting a verification that + /// did not happen is worse than not verifying, because it is believed. + pub fn finish(&self) -> Option { + let t = self.0.lock().ok()?; + if !t.finished { + return None; + } + Some(Measured { + bytes: t.bytes, + sha256: t.hasher.clone().finalize().into(), + }) + } +} + +/// Why a metered body stopped short. +/// +/// Named rather than folded into one `io::Error` because the two cases get +/// different statuses — over-limit is the caller's fault (413), a stall is +/// nobody's (504) — and by the time the error surfaces the transfer is gone. +const ERR_TOO_LARGE: &str = "upload exceeds the byte limit for this token"; +const ERR_STALLED: &str = "upload stalled"; + +/// Wrap a request body so it can be handed to reqwest without buffering: +/// count it, hash it, and cut the transfer the moment it exceeds what was +/// declared. +/// +/// Cutting mid-stream is what makes the size limit *preventive* rather than +/// merely detective. Yielding an error aborts reqwest's request body and tears +/// the connection, so a content-addressed upstream that reads to completion +/// before committing never stores the overage. +/// +/// Deliberately the same `unfold` shape as +/// [`crate::services::http_caller::idle_guarded_stream`], so the inbound and +/// outbound guards read alike. +pub fn metered_body(body: axum::body::Body, limit: Limit) -> (reqwest::Body, MeterHandle) { + let tally = Arc::new(Mutex::new(Tally::default())); + let shared = tally.clone(); + // The tighter of the two bounds. A declared size is exact, so anything + // past it is already a mismatch and there is no reason to keep paying for + // bytes that cannot be accepted. + let ceiling = limit + .declared_bytes + .map_or(limit.max_bytes, |d| d.min(limit.max_bytes)); + + let stream = + futures_util::stream::unfold(Some(Box::pin(body.into_data_stream())), move |state| { + let shared = shared.clone(); + async move { + let mut inner = state?; + match tokio::time::timeout(limit.idle, inner.next()).await { + Ok(Some(Ok(chunk))) => { + // The guard is never held across an await, so this + // future stays `Send`. + { + let mut t = match shared.lock() { + Ok(t) => t, + Err(_) => { + return Some(( + Err(std::io::Error::other("upload meter poisoned")), + None, + )); + } + }; + t.bytes += chunk.len() as u64; + if t.bytes > ceiling { + t.exceeded = true; + return Some(( + Err(std::io::Error::new( + std::io::ErrorKind::InvalidData, + ERR_TOO_LARGE, + )), + None, + )); + } + t.hasher.update(&chunk); + } + Some((Ok(chunk), Some(inner))) + } + Ok(Some(Err(e))) => Some((Err(std::io::Error::other(e)), None)), + Ok(None) => { + if let Ok(mut t) = shared.lock() { + t.finished = true; + } + None + } + Err(_elapsed) => Some(( + Err(std::io::Error::new( + std::io::ErrorKind::TimedOut, + ERR_STALLED, + )), + None, + )), + } + } + }); + + (reqwest::Body::wrap_stream(stream), MeterHandle(tally)) +} + +/// Record what an upload stored, so a later approval referencing these bytes +/// can describe them. +/// +/// Best-effort and deliberately not fatal: the push succeeded, and failing the +/// redemption because a descriptive row could not be written would trade a +/// working upload for a prettier approval. +pub(crate) async fn record_uploaded( + pool: &sqlx::PgPool, + org_id: Uuid, + service_instance_id: Option, + service_key: Option<&str>, + d: &UploadedDescriptor, +) { + if let Err(e) = overslash_db::repos::media_descriptor::record( + pool, + overslash_db::repos::media_descriptor::NewMediaDescriptor { + org_id, + service_instance_id, + service_key, + media_path: &d.media_path, + sha256: d.sha256.as_deref(), + mime: d.mime.as_deref(), + size_bytes: d.size_bytes, + filename: d.filename.as_deref(), + source: overslash_db::repos::media_descriptor::MediaSource::Upload, + }, + ) + .await + { + tracing::warn!(error = %e, "upload: media descriptor not recorded"); + } +} + +/// What the upstream said it stored, read through the template's `result` jq. +#[derive(Debug, Default, Clone)] +pub struct UploadedDescriptor { + pub media_path: String, + pub sha256: Option, + pub mime: Option, + pub size_bytes: Option, + pub filename: Option, +} diff --git a/crates/overslash-api/src/services/stored_call.rs b/crates/overslash-api/src/services/stored_call.rs index 97dd7a62..97748b47 100644 --- a/crates/overslash-api/src/services/stored_call.rs +++ b/crates/overslash-api/src/services/stored_call.rs @@ -245,6 +245,57 @@ async fn run_http(ctx: StoredCallCtx<'_>, stored: StoredCallRequest) -> StoredOu } async fn run_mcp(ctx: StoredCallCtx<'_>, call: StoredMcpCall) -> StoredOutcome { + // Gateway-served actions never reach the upstream — the twin of the test + // in `call_mcp::dispatch`, and the one that actually matters first: an + // upload is `risk: write`, so for any gated agent the *first* call lands + // here rather than inline. Without this the approval would be granted and + // the replay would answer "unknown tool", which is a failure that only + // appears once a human has said yes. + // + // The spec rides on the payload because replay resolves nothing: it holds + // a URL and a tool name, not an action key to look a declaration back up + // with. See `StoredMcpCall::upload`. + if let Some(spec) = call.upload.as_ref() { + return match crate::services::proxy_upload::intercept_mint( + ctx.state, + ctx.ext, + crate::services::proxy_upload::Mint { + org_id: ctx.org_id, + identity_id: ctx.identity_id, + service_instance_id: call.service_instance_id, + service_key: call.service_key.as_deref(), + action_key: call.action_key.as_deref(), + mcp_url: &call.url, + mcp_auth: &call.auth, + spec, + arguments: &call.arguments, + }, + ) + .await + { + Ok(result) => { + let value = serde_json::to_value(&result) + .unwrap_or_else(|_| serde_json::json!({"note": "result not serializable"})); + StoredOutcome::Executed { + summary: serde_json::json!({ + "runtime": "mcp", + "tool": call.tool, + "cause": "upload_requested", + "response": { "skipped": "capability" }, + }), + result: value, + typed: result, + upstream_errored: false, + is_error: false, + } + } + Err(e) => StoredOutcome::Failed { + message: format!("upload mint failed: {e}"), + error: Some(Box::new(e)), + }, + }; + } + // Re-resolve OAuth fresh, for the same reason as the HTTP path: the stored // payload is credential-free (only the provider survives in `call.auth`), // so the token — which may have expired while the call waited — is minted diff --git a/crates/overslash-api/src/services/wait_mode.rs b/crates/overslash-api/src/services/wait_mode.rs index 615e1fe2..aa6012f9 100644 --- a/crates/overslash-api/src/services/wait_mode.rs +++ b/crates/overslash-api/src/services/wait_mode.rs @@ -66,6 +66,11 @@ pub struct Blockers { /// `response_type: "binary"` — the bytes would be mangled by /// `String::from_utf8_lossy` on their way into the execution row. pub binary_response: bool, + /// `x-overslash-upload` — the action mints a capability rather than + /// calling anything, and the token would start expiring while the call sat + /// in the queue. Same reasoning as `deliver_url`, which mints the outbound + /// half of the same mechanism. + pub gateway_upload: bool, /// `ASYNC_EXECUTION_ENABLED` is off, so nothing drains the queue. A /// caller-named mode is refused for this at the top of the handler; the /// template rung just never gets adopted. @@ -93,6 +98,8 @@ impl Blockers { Some("platform_runtime") } else if self.binary_response { Some("binary_response") + } else if self.gateway_upload { + Some("gateway_upload") } else { None } @@ -232,6 +239,7 @@ mod tests { return_url: false, platform_runtime: false, binary_response: false, + gateway_upload: false, async_disabled: false, }; diff --git a/crates/overslash-api/tests/api.rs b/crates/overslash-api/tests/api.rs index 839bb706..89f745e0 100644 --- a/crates/overslash-api/tests/api.rs +++ b/crates/overslash-api/tests/api.rs @@ -148,6 +148,7 @@ mod template_validation; mod templates; mod templates_import; mod trial; +mod uploads; mod upstream_metrics; mod version; mod wait_mode; diff --git a/crates/overslash-api/tests/common/mod.rs b/crates/overslash-api/tests/common/mod.rs index ad50a65d..e46ad07b 100644 --- a/crates/overslash-api/tests/common/mod.rs +++ b/crates/overslash-api/tests/common/mod.rs @@ -663,6 +663,8 @@ where audit_response_body_max_bytes: 65_536, filter_timeout_ms: 2000, download_token_ttl_secs: 900, + upload_token_ttl_secs: 900, + upload_max_bytes: 100 * 1024 * 1024, call_result_max_bytes: 1024 * 1024, dashboard_url: "/".into(), dashboard_origin: "*localhost*".into(), @@ -755,6 +757,7 @@ where .merge(overslash_api::routes::permissions::router()) .merge(overslash_api::routes::actions::router()) .merge(overslash_api::routes::downloads::router()) + .merge(overslash_api::routes::uploads::router()) .merge(overslash_api::routes::icons::router()) .merge(overslash_api::routes::actions::validate_router()) .merge(overslash_api::routes::approvals::router()) @@ -879,6 +882,8 @@ pub async fn start_api_with_dev_auth(pool: PgPool) -> (String, Client) { audit_response_body_max_bytes: 65_536, filter_timeout_ms: 2000, download_token_ttl_secs: 900, + upload_token_ttl_secs: 900, + upload_max_bytes: 100 * 1024 * 1024, call_result_max_bytes: 1024 * 1024, dashboard_url: "/".into(), dashboard_origin: "*localhost*".into(), @@ -962,6 +967,7 @@ pub async fn start_api_with_dev_auth(pool: PgPool) -> (String, Client) { .merge(overslash_api::routes::permissions::router()) .merge(overslash_api::routes::actions::router()) .merge(overslash_api::routes::downloads::router()) + .merge(overslash_api::routes::uploads::router()) .merge(overslash_api::routes::actions::validate_router()) .merge(overslash_api::routes::approvals::router()) .merge(overslash_api::routes::executions::router()) @@ -1040,6 +1046,8 @@ pub async fn start_api_with_auth_providers( audit_response_body_max_bytes: 65_536, filter_timeout_ms: 2000, download_token_ttl_secs: 900, + upload_token_ttl_secs: 900, + upload_max_bytes: 100 * 1024 * 1024, call_result_max_bytes: 1024 * 1024, dashboard_url: "/".into(), dashboard_origin: "*localhost*".into(), @@ -1126,6 +1134,7 @@ pub async fn start_api_with_auth_providers( .merge(overslash_api::routes::permissions::router()) .merge(overslash_api::routes::actions::router()) .merge(overslash_api::routes::downloads::router()) + .merge(overslash_api::routes::uploads::router()) .merge(overslash_api::routes::actions::validate_router()) .merge(overslash_api::routes::approvals::router()) .merge(overslash_api::routes::executions::router()) @@ -1626,6 +1635,8 @@ where audit_response_body_max_bytes: 65_536, filter_timeout_ms: 2000, download_token_ttl_secs: 900, + upload_token_ttl_secs: 900, + upload_max_bytes: 100 * 1024 * 1024, call_result_max_bytes: 1024 * 1024, dashboard_url: "/".into(), dashboard_origin: "*localhost*".into(), @@ -1710,6 +1721,7 @@ where .merge(overslash_api::routes::permissions::router()) .merge(overslash_api::routes::actions::router()) .merge(overslash_api::routes::downloads::router()) + .merge(overslash_api::routes::uploads::router()) .merge(overslash_api::routes::actions::validate_router()) .merge(overslash_api::routes::approvals::router()) .merge(overslash_api::routes::executions::router()) @@ -1794,6 +1806,8 @@ pub async fn start_api_for_search(pool: PgPool) -> (String, Client) { audit_response_body_max_bytes: 65_536, filter_timeout_ms: 2000, download_token_ttl_secs: 900, + upload_token_ttl_secs: 900, + upload_max_bytes: 100 * 1024 * 1024, call_result_max_bytes: 1024 * 1024, dashboard_url: "/".into(), dashboard_origin: "*localhost*".into(), @@ -1880,6 +1894,7 @@ pub async fn start_api_for_search(pool: PgPool) -> (String, Client) { .merge(overslash_api::routes::search::router()) .merge(overslash_api::routes::actions::router()) .merge(overslash_api::routes::downloads::router()) + .merge(overslash_api::routes::uploads::router()) .merge(overslash_api::routes::actions::validate_router()) .merge(overslash_api::routes::mcp::router()) .merge(overslash_api::routes::auth::router()) @@ -1933,6 +1948,8 @@ pub async fn start_api_with_body_limit(pool: PgPool, max_bytes: usize) -> (Socke audit_response_body_max_bytes: 65_536, filter_timeout_ms: 2000, download_token_ttl_secs: 900, + upload_token_ttl_secs: 900, + upload_max_bytes: 100 * 1024 * 1024, call_result_max_bytes: 1024 * 1024, dashboard_url: "/".into(), dashboard_origin: "*localhost*".into(), @@ -2016,6 +2033,7 @@ pub async fn start_api_with_body_limit(pool: PgPool, max_bytes: usize) -> (Socke .merge(overslash_api::routes::permissions::router()) .merge(overslash_api::routes::actions::router()) .merge(overslash_api::routes::downloads::router()) + .merge(overslash_api::routes::uploads::router()) .merge(overslash_api::routes::actions::validate_router()) .merge(overslash_api::routes::approvals::router()) .merge(overslash_api::routes::executions::router()) @@ -2264,6 +2282,8 @@ pub async fn make_app_state(pool: PgPool) -> overslash_api::AppState { audit_response_body_max_bytes: 65_536, filter_timeout_ms: 2000, download_token_ttl_secs: 900, + upload_token_ttl_secs: 900, + upload_max_bytes: 100 * 1024 * 1024, dashboard_url: "/".into(), dashboard_origin: "*localhost*".into(), mcp_extra_origins: String::new(), diff --git a/crates/overslash-api/tests/common/shared_router.rs b/crates/overslash-api/tests/common/shared_router.rs index 6bc64d9d..d000585b 100644 --- a/crates/overslash-api/tests/common/shared_router.rs +++ b/crates/overslash-api/tests/common/shared_router.rs @@ -322,6 +322,8 @@ fn shared_config(addr: SocketAddr) -> overslash_api::config::Config { audit_response_body_max_bytes: 65_536, filter_timeout_ms: 2000, download_token_ttl_secs: 900, + upload_token_ttl_secs: 900, + upload_max_bytes: 100 * 1024 * 1024, call_result_max_bytes: 1024 * 1024, dashboard_url: "/".into(), dashboard_origin: "*localhost*".into(), diff --git a/crates/overslash-api/tests/integration.rs b/crates/overslash-api/tests/integration.rs index 9c06b12a..e3662417 100644 --- a/crates/overslash-api/tests/integration.rs +++ b/crates/overslash-api/tests/integration.rs @@ -56,6 +56,8 @@ async fn start_api(pool: PgPool) -> (SocketAddr, Client) { audit_response_body_max_bytes: 65_536, filter_timeout_ms: 2000, download_token_ttl_secs: 900, + upload_token_ttl_secs: 900, + upload_max_bytes: 100 * 1024 * 1024, dashboard_url: "/".into(), dashboard_origin: "*localhost*".into(), mcp_extra_origins: String::new(), @@ -1804,6 +1806,8 @@ async fn test_service_registry_api() { audit_response_body_max_bytes: 65_536, filter_timeout_ms: 2000, download_token_ttl_secs: 900, + upload_token_ttl_secs: 900, + upload_max_bytes: 100 * 1024 * 1024, dashboard_url: "/".into(), dashboard_origin: "*localhost*".into(), mcp_extra_origins: String::new(), @@ -2855,6 +2859,8 @@ async fn start_api_with_registry( audit_response_body_max_bytes: 65_536, filter_timeout_ms: 2000, download_token_ttl_secs: 900, + upload_token_ttl_secs: 900, + upload_max_bytes: 100 * 1024 * 1024, dashboard_url: "/".into(), dashboard_origin: "*localhost*".into(), mcp_extra_origins: String::new(), diff --git a/crates/overslash-api/tests/mcp_elicitation.rs b/crates/overslash-api/tests/mcp_elicitation.rs index be5dc4b6..75cedd34 100644 --- a/crates/overslash-api/tests/mcp_elicitation.rs +++ b/crates/overslash-api/tests/mcp_elicitation.rs @@ -1180,6 +1180,8 @@ async fn build_state_for_session(fx: &McpFixture) -> overslash_api::AppState { audit_response_body_max_bytes: 65_536, filter_timeout_ms: 2000, download_token_ttl_secs: 900, + upload_token_ttl_secs: 900, + upload_max_bytes: 100 * 1024 * 1024, call_result_max_bytes: 1024 * 1024, dashboard_url: "/".into(), dashboard_origin: "*localhost*".into(), diff --git a/crates/overslash-api/tests/rate_limits.rs b/crates/overslash-api/tests/rate_limits.rs index 72353159..7e830c93 100644 --- a/crates/overslash-api/tests/rate_limits.rs +++ b/crates/overslash-api/tests/rate_limits.rs @@ -367,6 +367,8 @@ async fn test_resolve_user_budget_falls_back_to_org_default() { audit_response_body_max_bytes: 65_536, filter_timeout_ms: 2000, download_token_ttl_secs: 900, + upload_token_ttl_secs: 900, + upload_max_bytes: 100 * 1024 * 1024, call_result_max_bytes: 1024 * 1024, dashboard_url: "/".into(), dashboard_origin: "*localhost*".into(), @@ -641,6 +643,8 @@ async fn test_resolve_user_budget_per_user_override_wins() { audit_response_body_max_bytes: 65_536, filter_timeout_ms: 2000, download_token_ttl_secs: 900, + upload_token_ttl_secs: 900, + upload_max_bytes: 100 * 1024 * 1024, call_result_max_bytes: 1024 * 1024, dashboard_url: "/".into(), dashboard_origin: "*localhost*".into(), @@ -733,6 +737,8 @@ async fn make_app_state(pool: PgPool) -> overslash_api::AppState { audit_response_body_max_bytes: 65_536, filter_timeout_ms: 2000, download_token_ttl_secs: 900, + upload_token_ttl_secs: 900, + upload_max_bytes: 100 * 1024 * 1024, call_result_max_bytes: 1024 * 1024, dashboard_url: "/".into(), dashboard_origin: "*localhost*".into(), @@ -1149,6 +1155,8 @@ async fn test_cache_invalidation_user_budget() { audit_response_body_max_bytes: 65_536, filter_timeout_ms: 2000, download_token_ttl_secs: 900, + upload_token_ttl_secs: 900, + upload_max_bytes: 100 * 1024 * 1024, call_result_max_bytes: 1024 * 1024, dashboard_url: "/".into(), dashboard_origin: "*localhost*".into(), @@ -1338,6 +1346,8 @@ async fn test_cache_invalidation_org_flushes_all() { audit_response_body_max_bytes: 65_536, filter_timeout_ms: 2000, download_token_ttl_secs: 900, + upload_token_ttl_secs: 900, + upload_max_bytes: 100 * 1024 * 1024, call_result_max_bytes: 1024 * 1024, dashboard_url: "/".into(), dashboard_origin: "*localhost*".into(), diff --git a/crates/overslash-api/tests/trial.rs b/crates/overslash-api/tests/trial.rs index 6c6c97d2..14ed05d3 100644 --- a/crates/overslash-api/tests/trial.rs +++ b/crates/overslash-api/tests/trial.rs @@ -142,6 +142,8 @@ async fn make_app_state(pool: PgPool) -> overslash_api::AppState { audit_response_body_max_bytes: 65_536, filter_timeout_ms: 2000, download_token_ttl_secs: 900, + upload_token_ttl_secs: 900, + upload_max_bytes: 100 * 1024 * 1024, call_result_max_bytes: 1024 * 1024, dashboard_url: "/".into(), dashboard_origin: "*localhost*".into(), diff --git a/crates/overslash-api/tests/uploads.rs b/crates/overslash-api/tests/uploads.rs new file mode 100644 index 00000000..239fecb0 --- /dev/null +++ b/crates/overslash-api/tests/uploads.rs @@ -0,0 +1,941 @@ +//! Integration tests for proxy uploads — minting a capability to push bytes, +//! redeeming it, and the media ledger that lets a later approval describe what +//! was pushed. +//! +//! The inbound mirror of `downloads.rs`, and the tests are shaped around the +//! places this design can fail *silently* rather than around the happy path: +//! +//! * the gated path, where a `risk: write` upload is replayed from a stored +//! payload after approval and must mint rather than dispatch a tool call; +//! * single-use redemption, where a second push under one authorization is +//! the whole thing the token shape exists to prevent; +//! * content binding, where a mismatch must withhold the reference rather +//! than merely log; +//! * the ledger, where a descriptor seen *without* `deliver: "url"` is the +//! common case and the easy one to miss. + +use crate::common; + +use std::net::SocketAddr; +use std::sync::{Arc, Mutex}; + +use axum::{ + Json, Router, + extract::State, + http::{HeaderMap, StatusCode}, + response::IntoResponse, + routing::post, +}; +use reqwest::Client; +use serde_json::{Value, json}; +use tokio::net::TcpListener; + +// ── A stub that plays both halves, as the real container does ─────────── +// +// `/mcp` for the tool calls and `POST /media` for the bytes, behind the same +// bearer. The gateway must present that bearer on the byte route having +// re-resolved it from the vault, on a request the original caller never +// authenticated. + +const UPLOAD_SHA: &str = "b94d27b9934d3e08a52e52d7da7dabfac484efe37a5380ee9088f7ace2efcde9"; +const UPLOAD_BODY: &[u8] = b"hello world"; + +#[derive(Default)] +struct StubInner { + /// Every `POST /media` the stub saw: (content_type, filename, body). + uploads: Vec<(Option, Option, Vec)>, + /// Tool names dispatched over `/mcp`. A gateway-served action must never + /// appear here. + tool_calls: Vec, +} + +#[derive(Clone, Default)] +struct Stub { + inner: Arc>, +} + +async fn media_upload( + State(stub): State, + headers: HeaderMap, + axum::extract::RawQuery(query): axum::extract::RawQuery, + body: axum::body::Bytes, +) -> axum::response::Response { + if headers + .get("authorization") + .and_then(|v| v.to_str().ok()) + .is_none_or(|v| v != "Bearer stub-token") + { + return (StatusCode::UNAUTHORIZED, "unauthorized").into_response(); + } + let filename = query.as_deref().and_then(|q| { + url::form_urlencoded::parse(q.as_bytes()) + .find(|(k, _)| k == "stored_as") + .map(|(_, v)| v.to_string()) + }); + let content_type = headers + .get("content-type") + .and_then(|v| v.to_str().ok()) + .map(str::to_string); + + let digest = { + use sha2::{Digest, Sha256}; + let d: [u8; 32] = Sha256::digest(&body).into(); + d.iter().map(|b| format!("{b:02x}")).collect::() + }; + stub.inner.lock().unwrap().uploads.push(( + content_type.clone(), + filename.clone(), + body.to_vec(), + )); + + // Content-addressed, exactly as the real container is. + ( + StatusCode::CREATED, + [("location", format!("/media/{digest}"))], + Json(json!({ + "media_path": format!("/media/{digest}"), + "mime": content_type.unwrap_or_else(|| "application/octet-stream".into()), + "size": body.len(), + "filename": filename.unwrap_or_else(|| format!("{}.bin", &digest[..12])), + "sha256": digest, + })), + ) + .into_response() +} + +async fn mcp_handler(State(stub): State, Json(req): Json) -> Json { + let id = req.get("id").cloned().unwrap_or(Value::Null); + let method = req.get("method").and_then(Value::as_str).unwrap_or(""); + let result = match method { + "initialize" => json!({ + "protocolVersion": "2025-06-18", + "serverInfo": { "name": "stub-upload", "version": "0" }, + "capabilities": {} + }), + "tools/call" => { + let params = req.get("params").cloned().unwrap_or(Value::Null); + let name = params + .get("name") + .and_then(Value::as_str) + .unwrap_or("") + .to_string(); + stub.inner.lock().unwrap().tool_calls.push(name.clone()); + match name.as_str() { + "download_media" => json!({ + "content": [{ "type": "text", "text": "downloaded" }], + "structuredContent": { + "media_path": format!("/media/{UPLOAD_SHA}"), + "mime": "application/pdf", + "size": 12345, + "filename": "report.pdf", + "sha256": UPLOAD_SHA, + }, + "isError": false + }), + _ => json!({ + "content": [{ "type": "text", "text": "sent" }], + "structuredContent": { "ok": true }, + "isError": false + }), + } + } + _ => json!({}), + }; + Json(json!({ "jsonrpc": "2.0", "id": id, "result": result })) +} + +async fn start_stub() -> (SocketAddr, Arc>) { + common::allow_loopback_ssrf(); + let stub = Stub::default(); + let inner = stub.inner.clone(); + let app = Router::new() + .route("/mcp", post(mcp_handler)) + .route("/media", post(media_upload)) + .with_state(stub); + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = listener.local_addr().unwrap(); + tokio::spawn(async move { + axum::serve(listener, app).await.unwrap(); + }); + (addr, inner) +} + +// ── Template fixture ──────────────────────────────────────────────────── + +/// Mirrors the shipped `services/whatsapp.yaml` entries this feature adds: +/// `upload_media` with its `upload:` block, `download_media` with a `sha256` +/// filter, and a `send_file` whose `media_path` resolves through the ledger. +fn template_yaml(key: &str, url: &str, secret_name: &str, upload_path: &str) -> String { + format!( + r#"openapi: "3.1.0" +info: + title: Upload Stub + x-overslash-key: {key} +x-overslash-runtime: mcp +paths: {{}} +x-overslash-mcp: + url: {url} + auth: {{ kind: bearer, secret_name: {secret_name} }} + autodiscover: false + tools: + - name: upload_media + risk: write + description: 'Mint a URL for pushing a file' + upload: + path: {upload_path} + method: POST + # Deliberately not the conventional `filename`: a fixture that reuses + # it cannot tell a template-driven parameter name from a hardcoded one, + # which is exactly how this shipped inert once. + filename_param: stored_as + auth: inherit + max_bytes: 1024 + result: + media_path: .media_path + sha256: .sha256 + mime: .mime + size: .size + filename: .filename + input_schema: + type: object + properties: + filename: {{ type: string }} + mime: {{ type: string }} + size_bytes: {{ type: integer }} + sha256: {{ type: string }} + required: [] + disclose: + - label: "File" + primary: true + filter: '.arguments.filename // "unnamed upload"' + - label: "SHA-256" + filter: ".arguments.sha256 // empty" + - name: download_media + risk: read + description: 'Download media from {{chat_jid}}' + download: + url: .structured.media_path + mime: .structured.mime + size: .structured.size + filename: .structured.filename + sha256: .structured.sha256 + auth: inherit + input_schema: + type: object + properties: + chat_jid: {{ type: string }} + required: [chat_jid] + - name: send_file + risk: write + description: 'Send a file to {{recipient}}' + input_schema: + type: object + properties: + recipient: {{ type: string }} + media_path: + type: string + resolve: + source: media + display: '{{filename}}[ ({{mime}}, {{size}} bytes)]' + required: [recipient, media_path] + disclose: + - label: "File" + primary: true + filter: ".resolved.media_path // .arguments.media_path" +"# + ) +} + +struct Fx { + base: String, + client: Client, + agent_key: String, + admin_key: String, + stub: Arc>, +} + +async fn setup(pool: sqlx::PgPool) -> Fx { + setup_with(pool, "/media", &["upstub:**"], |_| {}).await +} + +/// Grants only the byte-moving actions, leaving `send_file` uncovered so it +/// bubbles an approval — a *gap* in the chain, not a deny, since a deny is +/// refused outright and never produces the disclosure a reviewer would read. +async fn setup_for_disclosure(pool: sqlx::PgPool) -> Fx { + setup_with( + pool, + "/media", + &["upstub:upload_media:**", "upstub:download_media:**"], + |_| {}, + ) + .await +} + +/// An empty `grants` leaves the agent with nothing, so its first call is gated +/// and goes down the replay path instead of the inline one. +async fn setup_with(pool: sqlx::PgPool, upload_path: &str, grants: &[&str], customize: F) -> Fx +where + F: FnOnce(&mut overslash_api::config::Config), +{ + let (stub_addr, stub) = start_stub().await; + let stub_url = format!("http://{stub_addr}/mcp"); + let path = upload_path.replace("{PORT}", &stub_addr.port().to_string()); + + let (api_addr, client) = common::start_api_with(pool, customize).await; + let base = format!("http://{api_addr}"); + let (_org, agent_ident, agent_key, admin_key) = + common::bootstrap_org_identity(&base, &client).await; + + let yaml = template_yaml("upstub", &stub_url, "stub_token", &path); + let resp = client + .post(format!("{base}/v1/templates")) + .header(auth(&admin_key).0, auth(&admin_key).1) + .json(&json!({ "openapi": yaml, "user_level": false })) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 200, "template: {:?}", resp.text().await); + + client + .put(format!("{base}/v1/secrets/stub_token")) + .header(auth(&admin_key).0, auth(&admin_key).1) + .json(&json!({ "value": "stub-token" })) + .send() + .await + .unwrap(); + + for pattern in grants { + client + .post(format!("{base}/v1/permissions")) + .header(auth(&admin_key).0, auth(&admin_key).1) + .json(&json!({ + "identity_id": agent_ident, + "action_pattern": pattern, + "effect": "allow", + })) + .send() + .await + .unwrap(); + } + + let resp = client + .post(format!("{base}/v1/services")) + .header(auth(&agent_key).0, auth(&agent_key).1) + .json(&json!({ "name": "upstub", "template_key": "upstub" })) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 200, "service: {:?}", resp.text().await); + + Fx { + base, + client, + agent_key, + admin_key, + stub, + } +} + +fn auth(key: &str) -> (&'static str, String) { + ("Authorization", format!("Bearer {key}")) +} + +/// Call `upload_media` and return the parsed descriptor. +async fn mint(fx: &Fx, params: Value) -> Value { + let resp = fx + .client + .post(format!("{}/v1/actions/call", fx.base)) + .header(auth(&fx.agent_key).0, auth(&fx.agent_key).1) + .json(&json!({ "service": "upstub", "action": "upload_media", "params": params })) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 200, "mint: {:?}", resp.text().await); + let body: Value = resp.json().await.unwrap(); + serde_json::from_str(body["result"]["body"].as_str().expect("body string")).unwrap() +} + +// ── Minting ───────────────────────────────────────────────────────────── + +#[tokio::test] +async fn mint_returns_a_url_and_never_touches_the_container() { + let pool = common::test_pool().await; + let fx = setup(pool).await; + + let d = mint( + &fx, + json!({ "filename": "hello.txt", "mime": "text/plain" }), + ) + .await; + + let url = d["upload_url"].as_str().expect("upload_url"); + assert!(url.starts_with(&fx.base), "{url}"); + assert_eq!(d["method"], "POST"); + // The template's own ceiling, not the deployment's — a template may lower + // the limit, and the caller is told the number that will actually bind. + assert_eq!(d["max_bytes"], 1024); + assert!(d["hint"].as_str().unwrap().contains("--data-binary")); + + // Nothing about the container reaches the caller. Handing over its URL or + // its bearer is precisely what this mechanism exists to avoid. + let text = d.to_string(); + assert!(!text.contains("stub-token"), "credential leaked: {text}"); + assert!(!text.contains("/mcp"), "instance url leaked: {text}"); + + // A gateway-served action is never forwarded: `tools/call` with this name + // would answer "unknown tool". + let inner = fx.stub.lock().unwrap(); + assert!(inner.tool_calls.is_empty(), "{:?}", inner.tool_calls); + assert!(inner.uploads.is_empty(), "nothing pushed yet"); +} + +#[tokio::test] +async fn declared_size_over_the_template_ceiling_is_refused_at_mint() { + let pool = common::test_pool().await; + let fx = setup(pool).await; + + // Refused before a token exists rather than at redemption: the caller + // already knows the size, so there is no reason to hand back a capability + // that cannot succeed. + let resp = fx + .client + .post(format!("{}/v1/actions/call", fx.base)) + .header(auth(&fx.agent_key).0, auth(&fx.agent_key).1) + .json(&json!({ + "service": "upstub", "action": "upload_media", + "params": { "size_bytes": 999_999 }, + })) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 400); + assert!(resp.text().await.unwrap().contains("exceeds")); +} + +#[tokio::test] +async fn an_off_origin_upload_path_is_refused() { + let pool = common::test_pool().await; + // A path that resolves to another host would send this instance's + // credential — and the caller's bytes — somewhere it does not belong. + let fx = setup_with( + pool, + "https://evil.example.com/media", + &["upstub:**"], + |_| {}, + ) + .await; + + let resp = fx + .client + .post(format!("{}/v1/actions/call", fx.base)) + .header(auth(&fx.agent_key).0, auth(&fx.agent_key).1) + .json(&json!({ "service": "upstub", "action": "upload_media", "params": {} })) + .send() + .await + .unwrap(); + assert!(resp.status().is_client_error() || resp.status().is_server_error()); + let text = resp.text().await.unwrap(); + assert!(text.contains("outside the MCP server's origin"), "{text}"); +} + +#[tokio::test] +async fn async_execution_is_refused_for_an_upload_mint() { + let pool = common::test_pool().await; + // Async has to be *enabled*, or the top-of-handler "async is disabled" + // refusal fires first and hides the rejection under test. + let fx = setup_with(pool, "/media", &["upstub:**"], |cfg| { + cfg.async_execution.enabled = true; + }) + .await; + + // The capability would spend its lifetime in a queue. Same reasoning the + // deferred-download flags already carry. + let resp = fx + .client + .post(format!("{}/v1/actions/call", fx.base)) + .header(auth(&fx.agent_key).0, auth(&fx.agent_key).1) + .json(&json!({ + "service": "upstub", "action": "upload_media", + "params": {}, "execution": "async", + })) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 400); + assert!(resp.text().await.unwrap().contains("upload capability")); +} + +// ── Redemption ────────────────────────────────────────────────────────── + +#[tokio::test] +async fn redemption_streams_bytes_through_and_returns_the_stored_reference() { + let pool = common::test_pool().await; + let fx = setup(pool).await; + + let d = mint( + &fx, + json!({ "filename": "hello.txt", "mime": "text/plain" }), + ) + .await; + let url = d["upload_url"].as_str().unwrap(); + + // The pushing client sends no credential of its own — the token in the URL + // is the whole authority. + let resp = fx + .client + .post(url) + .header("content-type", "text/plain") + .body(UPLOAD_BODY) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 201, "push: {:?}", resp.text().await); + let stored: Value = resp.json().await.unwrap(); + assert_eq!(stored["media_path"], format!("/media/{UPLOAD_SHA}")); + assert_eq!(stored["sha256"], UPLOAD_SHA); + + let inner = fx.stub.lock().unwrap(); + assert_eq!(inner.uploads.len(), 1); + let (ct, filename, body) = &inner.uploads[0]; + assert_eq!(body, UPLOAD_BODY); + assert_eq!(ct.as_deref(), Some("text/plain")); + // Two claims at once: the filename comes off the *token*, never off the + // request that redeemed it — a redeemer who could choose it could get + // approval for one name and push another under it — and it arrives under + // the parameter name the *template* declared (`stored_as`), not a + // hardcoded `filename`. + assert_eq!(filename.as_deref(), Some("hello.txt")); +} + +#[tokio::test] +async fn a_token_is_good_for_exactly_one_push() { + let pool = common::test_pool().await; + let fx = setup(pool).await; + + let d = mint(&fx, json!({ "filename": "once.txt" })).await; + let url = d["upload_url"].as_str().unwrap(); + + let first = fx.client.post(url).body(UPLOAD_BODY).send().await.unwrap(); + assert_eq!(first.status(), 201); + + // The difference from a download token, which is deliberately multi-use so + // a dropped transfer can resume. Two pushes under one authorization would + // mean "what the reviewer approved" had no answer. + let second = fx + .client + .post(url) + .body(b"different".as_ref()) + .send() + .await + .unwrap(); + assert_eq!(second.status(), 404); + assert_eq!(second.text().await.unwrap(), "unknown or expired token"); + + assert_eq!( + fx.stub.lock().unwrap().uploads.len(), + 1, + "the second redemption must not reach the container" + ); +} + +#[tokio::test] +async fn unknown_expired_and_consumed_tokens_are_indistinguishable() { + let pool = common::test_pool().await; + let fx = setup(pool.clone()).await; + + // Unknown. + let unknown = fx + .client + .post(format!("{}/v1/uploads/{}", fx.base, "a".repeat(43))) + .body(UPLOAD_BODY) + .send() + .await + .unwrap(); + + // Consumed. + let d = mint(&fx, json!({})).await; + let consumed_url = d["upload_url"].as_str().unwrap().to_string(); + fx.client + .post(&consumed_url) + .body(UPLOAD_BODY) + .send() + .await + .unwrap(); + let consumed = fx + .client + .post(&consumed_url) + .body(UPLOAD_BODY) + .send() + .await + .unwrap(); + + // Expired. + let d = mint(&fx, json!({})).await; + let expired_url = d["upload_url"].as_str().unwrap().to_string(); + sqlx::query!("UPDATE upload_tokens SET expires_at = now() - interval '1 hour'") + .execute(&pool) + .await + .unwrap(); + let expired = fx + .client + .post(&expired_url) + .body(UPLOAD_BODY) + .send() + .await + .unwrap(); + + // A distinguishable "expired" would confirm to someone probing that a + // given token string was once real. + for (label, resp) in [ + ("unknown", unknown), + ("consumed", consumed), + ("expired", expired), + ] { + assert_eq!(resp.status(), 404, "{label}"); + assert_eq!( + resp.text().await.unwrap(), + "unknown or expired token", + "{label}" + ); + } +} + +#[tokio::test] +async fn deleting_the_secret_after_mint_fails_the_push_closed() { + let pool = common::test_pool().await; + let fx = setup(pool).await; + + let d = mint(&fx, json!({})).await; + let url = d["upload_url"].as_str().unwrap(); + + // The credential is re-resolved at redemption rather than persisted, so + // revoking it invalidates outstanding capabilities with no sweep. + let del = fx + .client + .delete(format!("{}/v1/secrets/stub_token", fx.base)) + .header(auth(&fx.admin_key).0, auth(&fx.admin_key).1) + .send() + .await + .unwrap(); + assert!(del.status().is_success(), "delete: {:?}", del.text().await); + + let resp = fx.client.post(url).body(UPLOAD_BODY).send().await.unwrap(); + assert!( + resp.status().is_client_error() || resp.status().is_server_error(), + "push should fail closed, got {}", + resp.status() + ); + assert!( + fx.stub.lock().unwrap().uploads.is_empty(), + "no bytes should reach the container without a credential" + ); +} + +// ── Content binding ───────────────────────────────────────────────────── + +#[tokio::test] +async fn a_sha256_mismatch_is_refused_and_withholds_the_reference() { + let pool = common::test_pool().await; + let fx = setup(pool).await; + + let d = mint(&fx, json!({ "sha256": "0".repeat(64) })).await; + let url = d["upload_url"].as_str().unwrap(); + + let resp = fx.client.post(url).body(UPLOAD_BODY).send().await.unwrap(); + assert_eq!(resp.status(), 422); + let text = resp.text().await.unwrap(); + assert!(text.contains("content mismatch"), "{text}"); + + // The bytes did reach the container — a hash is only known after the last + // one, so the check detects rather than prevents. Withholding the + // reference is what makes detecting it worth anything: nothing downstream + // can name bytes it was never told the name of. + assert!( + !text.contains("/media/"), + "the stored reference must not be handed back: {text}" + ); +} + +#[tokio::test] +async fn a_declared_size_is_enforced_mid_stream() { + let pool = common::test_pool().await; + let fx = setup(pool).await; + + let d = mint(&fx, json!({ "size_bytes": 4 })).await; + let url = d["upload_url"].as_str().unwrap(); + + // Unlike the hash, an over-length transfer is *prevented*: the meter cuts + // it, so the container never commits the overage. + let resp = fx + .client + .post(url) + .body(vec![b'x'; 512]) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 413); + assert!( + fx.stub.lock().unwrap().uploads.is_empty(), + "an oversized push must not land" + ); +} + +#[tokio::test] +async fn a_stated_content_length_over_the_cap_is_refused_up_front() { + let pool = common::test_pool().await; + let fx = setup(pool).await; + + let d = mint(&fx, json!({})).await; + let url = d["upload_url"].as_str().unwrap(); + + // The template's ceiling is 1024. A stated length past it is answered + // before a byte moves; the mid-stream meter is still the real enforcement, + // because a chunked body states no length and a caller may state an untrue + // one. + let resp = fx + .client + .post(url) + .body(vec![b'x'; 4096]) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 413); + assert!(fx.stub.lock().unwrap().uploads.is_empty()); +} + +// ── The gated path ────────────────────────────────────────────────────── + +#[tokio::test] +async fn an_approved_upload_mints_on_replay_instead_of_calling_a_tool() { + let pool = common::test_pool().await; + let fx = setup_with(pool, "/media", &[], |_| {}).await; + + // `upload_media` is risk: write, so a gated agent's *first* call is + // replayed from a stored payload after approval. Replay resolves nothing — + // it holds a URL and a tool name, not an action key — so without the spec + // riding on the payload this dispatches `tools/call {name: upload_media}` + // and the container answers "unknown tool". That failure only appears + // after a human has said yes, which is why it has its own test. + let resp = fx + .client + .post(format!("{}/v1/actions/call", fx.base)) + .header(auth(&fx.agent_key).0, auth(&fx.agent_key).1) + .json(&json!({ + "service": "upstub", "action": "upload_media", + "params": { "filename": "gated.txt" }, + })) + .send() + .await + .unwrap(); + assert_eq!( + resp.status(), + 202, + "expected an approval: {:?}", + resp.text().await + ); + let body: Value = resp.json().await.unwrap(); + let approval_id = body["approval_id"].as_str().expect("approval_id"); + + // The approval describes what was declared — that is all there is to + // describe, since the bytes have not been offered yet. + let approval: Value = fx + .client + .get(format!("{}/v1/approvals/{approval_id}", fx.base)) + .header(auth(&fx.admin_key).0, auth(&fx.admin_key).1) + .send() + .await + .unwrap() + .json() + .await + .unwrap(); + let file_row = approval["disclosed_fields"] + .as_array() + .expect("disclosed_fields") + .iter() + .find(|f| f["label"] == "File") + .expect("File row"); + assert_eq!(file_row["value"], "gated.txt"); + + let resp = fx + .client + .post(format!("{}/v1/approvals/{approval_id}/resolve", fx.base)) + .header(auth(&fx.admin_key).0, auth(&fx.admin_key).1) + .json(&json!({ "resolution": "allow" })) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 200); + + let resp = fx + .client + .post(format!("{}/v1/approvals/{approval_id}/call", fx.base)) + .header(auth(&fx.agent_key).0, auth(&fx.agent_key).1) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 200, "replay: {:?}", resp.text().await); + let body: Value = resp.json().await.unwrap(); + assert_eq!(body["execution"]["status"], "executed"); + + // The replay minted rather than dispatched. + assert!( + fx.stub.lock().unwrap().tool_calls.is_empty(), + "replay must not forward a gateway-served action: {:?}", + fx.stub.lock().unwrap().tool_calls + ); + + let result = body["execution"]["result"].clone(); + let text = result.to_string(); + assert!(text.contains("/v1/uploads/"), "replay result: {text}"); +} + +// ── Audit ─────────────────────────────────────────────────────────────── + +#[tokio::test] +async fn a_redemption_is_audited_with_declared_and_measured() { + let pool = common::test_pool().await; + let fx = setup(pool).await; + + let d = mint( + &fx, + json!({ "filename": "audited.txt", "sha256": UPLOAD_SHA }), + ) + .await; + let url = d["upload_url"].as_str().unwrap(); + let resp = fx.client.post(url).body(UPLOAD_BODY).send().await.unwrap(); + assert_eq!(resp.status(), 201); + + let audit: Value = fx + .client + .get(format!("{}/v1/audit?limit=50", fx.base)) + .header(auth(&fx.admin_key).0, auth(&fx.admin_key).1) + .send() + .await + .unwrap() + .json() + .await + .unwrap(); + let entries = audit["entries"] + .as_array() + .or_else(|| audit.as_array()) + .expect("audit entries"); + let row = entries + .iter() + .find(|e| e["action"] == "action.uploaded") + .unwrap_or_else(|| panic!("no action.uploaded row: {audit}")); + + // Both halves, because a divergence between them is the entire signal and + // this row is the only place it survives. + assert_eq!(row["detail"]["declared_sha256"], UPLOAD_SHA); + assert_eq!(row["detail"]["measured_sha256"], UPLOAD_SHA); + assert_eq!(row["detail"]["measured_size_bytes"], UPLOAD_BODY.len()); + assert_eq!(row["detail"]["is_error"], false); + assert_eq!( + row["detail"]["stored_media_path"], + format!("/media/{UPLOAD_SHA}") + ); + // The body streams through and is never buffered, so it cannot be captured. + assert_eq!(row["detail"]["request"]["skipped"], "streamed"); +} + +// ── The media ledger ──────────────────────────────────────────────────── + +#[tokio::test] +async fn an_uploaded_reference_is_described_in_a_later_approval() { + let pool = common::test_pool().await; + let fx = setup_for_disclosure(pool).await; + + let d = mint( + &fx, + json!({ "filename": "invoice.pdf", "mime": "application/pdf" }), + ) + .await; + let push = fx + .client + .post(d["upload_url"].as_str().unwrap()) + .header("content-type", "application/pdf") + .body(UPLOAD_BODY) + .send() + .await + .unwrap(); + assert_eq!(push.status(), 201); + let stored: Value = push.json().await.unwrap(); + let media_path = stored["media_path"].as_str().unwrap().to_string(); + + // Send it. The approval's primary row is what a reviewer actually reads, + // and a bare content hash tells them nothing about what they are + // approving. + let disclosed = disclosed_file_row(&fx, &media_path).await; + assert_eq!( + disclosed, + format!("invoice.pdf (application/pdf, {} bytes)", UPLOAD_BODY.len()), + "the File row should describe the bytes, not hash them" + ); +} + +#[tokio::test] +async fn a_downloaded_reference_is_described_even_without_deliver_url() { + let pool = common::test_pool().await; + let fx = setup_for_disclosure(pool).await; + + // The common forwarding path: `download_media` *without* `deliver: "url"` + // returns the raw reference so it can be re-sent. Recording only on the + // deferred path would leave exactly this case unenriched — which is most + // of them. + let resp = fx + .client + .post(format!("{}/v1/actions/call", fx.base)) + .header(auth(&fx.agent_key).0, auth(&fx.agent_key).1) + .json(&json!({ + "service": "upstub", "action": "download_media", + "params": { "chat_jid": "34600@s.whatsapp.net" }, + })) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 200); + + let disclosed = disclosed_file_row(&fx, &format!("/media/{UPLOAD_SHA}")).await; + assert_eq!(disclosed, "report.pdf (application/pdf, 12345 bytes)"); +} + +#[tokio::test] +async fn a_reference_the_gateway_never_saw_falls_back_to_the_raw_path() { + let pool = common::test_pool().await; + let fx = setup_for_disclosure(pool).await; + + // Bytes pushed to the container out of band were never seen here. The + // fallback is lossless — the reviewer still sees exactly the string the + // call will send — so a miss is less helpful, never misleading. + let unseen = "/media/deadbeef"; + let disclosed = disclosed_file_row(&fx, unseen).await; + assert_eq!(disclosed, unseen); +} + +/// Call `send_file` and read back the approval's primary "File" row. +/// +/// Goes through a real gated call rather than poking the resolver directly: +/// the whole claim under test is that the enrichment reaches the surface a +/// human reads. +async fn disclosed_file_row(fx: &Fx, media_path: &str) -> String { + let resp = fx + .client + .post(format!("{}/v1/actions/call", fx.base)) + .header(auth(&fx.agent_key).0, auth(&fx.agent_key).1) + .json(&json!({ + "service": "upstub", "action": "send_file", + "params": { "recipient": "34600@s.whatsapp.net", "media_path": media_path }, + })) + .send() + .await + .unwrap(); + let body: Value = resp.json().await.unwrap(); + body["disclosed_fields"] + .as_array() + .unwrap_or_else(|| panic!("no disclosed_fields: {body}")) + .iter() + .find(|f| f["label"] == "File") + .and_then(|f| f["value"].as_str()) + .unwrap_or_else(|| panic!("no File row: {body}")) + .to_string() +} diff --git a/crates/overslash-api/tests/whatsapp.rs b/crates/overslash-api/tests/whatsapp.rs index 47a55d71..ce26b17c 100644 --- a/crates/overslash-api/tests/whatsapp.rs +++ b/crates/overslash-api/tests/whatsapp.rs @@ -1631,6 +1631,20 @@ const INTENTIONALLY_NOT_EXPOSED: &[&str] = &[ "ping", ]; +/// Actions the gateway *serves itself* rather than forwarding. +/// +/// They are in the template so they carry a permission key, a risk class and a +/// disclose block; they are not in tools/list because the container has no such +/// tool. Direction 1 skips them — that assertion means "nothing we forward can +/// 404", and nothing here is forwarded. Direction 2 is unaffected: these names +/// are not in the container catalog to begin with. +const GATEWAY_SYNTHESIZED: &[&str] = &[ + // `POST /media` is plain HTTP on the container's own origin behind the same + // bearer, not a JSON-RPC tool. The entry exists so pushing bytes is + // permission-checked and approvable. + "upload_media", +]; + #[tokio::test] async fn shipped_template_matches_the_container_catalog() { let pool = common::test_pool().await; @@ -1670,6 +1684,9 @@ async fn shipped_template_matches_the_container_catalog() { // that moved or was renamed upstream answers -32603 "Unknown tool" at // call time, which is a 502 the caller can do nothing about. for name in &exposed { + if GATEWAY_SYNTHESIZED.contains(name) { + continue; + } assert!( CONTAINER_CATALOG_V0_7_0.contains(name), "template exposes `{name}`, which whatsapp-mcp-docker v0.7.0 does not serve — \ diff --git a/crates/overslash-core/src/instance_config.rs b/crates/overslash-core/src/instance_config.rs index 24dd71f7..28b10ba2 100644 --- a/crates/overslash-core/src/instance_config.rs +++ b/crates/overslash-core/src/instance_config.rs @@ -170,6 +170,7 @@ mod tests { disabled: false, request_body: None, download: None, + upload: None, }; for (name, instance_config) in params { action diff --git a/crates/overslash-core/src/openapi/alias.rs b/crates/overslash-core/src/openapi/alias.rs index 2447f307..ba200283 100644 --- a/crates/overslash-core/src/openapi/alias.rs +++ b/crates/overslash-core/src/openapi/alias.rs @@ -58,6 +58,10 @@ pub(super) const MCP_TOOL_ALIASES: &[Alias] = &[ alias: "download", canonical: "x-overslash-download", }, + Alias { + alias: "upload", + canonical: "x-overslash-upload", + }, Alias { alias: "timeout_ms", canonical: "x-overslash-timeout_ms", diff --git a/crates/overslash-core/src/openapi/ext.rs b/crates/overslash-core/src/openapi/ext.rs index fa91c73f..4c789a53 100644 --- a/crates/overslash-core/src/openapi/ext.rs +++ b/crates/overslash-core/src/openapi/ext.rs @@ -68,6 +68,7 @@ pub enum Ext { WaitMode, HandoffAfterMs, Download, + Upload, Pagination, // Parameters, body properties, tool properties, platform-action params Resolve, @@ -108,6 +109,7 @@ impl Ext { Ext::WaitMode => "x-overslash-wait-mode", Ext::HandoffAfterMs => "x-overslash-handoff_after_ms", Ext::Download => "x-overslash-download", + Ext::Upload => "x-overslash-upload", Ext::Pagination => "x-overslash-pagination", Ext::Resolve => "x-overslash-resolve", Ext::Aliases => "x-overslash-aliases", @@ -162,6 +164,7 @@ pub(super) const ALL: &[Ext] = &[ Ext::WaitMode, Ext::HandoffAfterMs, Ext::Download, + Ext::Upload, Ext::Pagination, Ext::Resolve, Ext::Aliases, @@ -337,6 +340,14 @@ pub(super) const READS: &[(Ext, &[Pos])] = &[ // is its own download, since `deliver: "url"` mints a token from the // resolved request (see the comment at actions.rs:160). (Ext::Download, &[Pos::McpTool, Pos::McpToolDiscovered]), + // mcp.rs. Same positions as Download, for a sharper reason: the block names + // a *route on the MCP origin*, which exists only because an MCP instance + // URL does. `McpToolDiscovered` is required even though a discovered entry + // never authors one — `lower_mcp_tool` is shared with + // `overlay_discovered_tools`, so the read genuinely happens at that + // position after the merge, and omitting it fires the `debug_assert!` in + // `get` from the overlay path. + (Ext::Upload, &[Pos::McpTool, Pos::McpToolDiscovered]), // actions.rs · mcp.rs. Wherever an action is authored, minus the platform // runtime: a platform action answers from this process, so there is no // upstream page to be on. `NextStyle::Link` is refused at `Pos::McpTool` @@ -457,7 +468,7 @@ mod tests { fn every_variant_is_in_all() { // `ALL` drives name resolution and did-you-mean suggestions, so a // variant missing from it is invisible to the lint. - assert_eq!(ALL.len(), 31, "ALL has drifted from the enum"); + assert_eq!(ALL.len(), 32, "ALL has drifted from the enum"); let mut keys: Vec<&str> = ALL.iter().map(|e| e.key()).collect(); keys.sort_unstable(); let before = keys.len(); @@ -504,6 +515,10 @@ mod tests { // normalizes at a position that then ignores it. assert!(!reads_at(Ext::Download, Pos::Operation)); assert!(reads_at(Ext::Download, Pos::McpTool)); + // Upload rides the same MCP-only positions as Download: the route it + // names only exists because an MCP instance URL does. + assert!(!reads_at(Ext::Upload, Pos::Operation)); + assert!(reads_at(Ext::Upload, Pos::McpTool)); assert!(!reads_at( Ext::Template, Pos::SecurityScheme(SchemeKind::Http) diff --git a/crates/overslash-core/src/openapi/extract/mcp.rs b/crates/overslash-core/src/openapi/extract/mcp.rs index 0dab488d..2064e5ec 100644 --- a/crates/overslash-core/src/openapi/extract/mcp.rs +++ b/crates/overslash-core/src/openapi/extract/mcp.rs @@ -14,7 +14,7 @@ use super::super::ext::{self, Ext, Pos}; use super::{ parse_aliases, parse_disclose, parse_download, parse_instance_config, parse_pagination, parse_redact, parse_resolver, parse_scope_params, parse_sql_policy, parse_timeout_ms, - parse_wait_mode, + parse_upload, parse_wait_mode, }; // ── x-overslash-mcp → McpSpec + ServiceActions ─────────────────────── @@ -216,7 +216,14 @@ pub(crate) fn extract_mcp_actions( } } None => { - if autodiscover { + // A gateway-served action is authored precisely *because* + // the upstream has no such tool, so its absence from a + // resync is the expected state rather than drift. Warning + // on it would make this diagnostic permanently, wrongly + // noisy for the one template that uses it — and a warning + // that is always wrong is one operators learn to ignore. + let gateway_served = obj.contains_key(Ext::Upload.key()); + if autodiscover && !gateway_served { warnings.push(ValidationIssue::new( "mcp_tool_not_discovered", format!( @@ -324,6 +331,20 @@ fn lower_mcp_tool( let disclose = parse_disclose(ext::get(obj, Pos::McpTool, Ext::Disclose), &base, errors); let redact = parse_redact(ext::get(obj, Pos::McpTool, Ext::Redact), &base, errors); let download = parse_download(ext::get(obj, Pos::McpTool, Ext::Download), &base, errors); + let upload = parse_upload(ext::get(obj, Pos::McpTool, Ext::Upload), &base, errors); + // A tool is one or the other. A tool that were both would be intercepted + // *and* dispatched, and there is no sensible order for that: the + // interception is what makes the action not reach the upstream at all. + if download.is_some() && upload.is_some() { + errors.push(ValidationIssue::new( + "upload_malformed", + "a tool declares either x-overslash-download or x-overslash-upload, not both: \ + an upload action is served by the gateway and never reaches the upstream tool, \ + so there is no result for a download block to point at", + base.clone(), + )); + return None; + } let timeout_ms = parse_timeout_ms( ext::get(obj, Pos::McpTool, Ext::TimeoutMs), Ext::TimeoutMs.key(), @@ -390,6 +411,7 @@ fn lower_mcp_tool( output_schema, disabled, download, + upload, // Everything else defaults — notably `request_body`, since MCP tool // calls are framed by the MCP client (which sets its own JSON-RPC // content type) and never routed through `resolve`. diff --git a/crates/overslash-core/src/openapi/extract/mod.rs b/crates/overslash-core/src/openapi/extract/mod.rs index 7ee2577d..f3b8628c 100644 --- a/crates/overslash-core/src/openapi/extract/mod.rs +++ b/crates/overslash-core/src/openapi/extract/mod.rs @@ -21,7 +21,7 @@ use serde_json::{Map, Value}; use crate::template_validation::ValidationIssue; use crate::types::{ DisclosureField, DownloadAuth, DownloadSpec, ExecutionMode, NextSpec, NextStyle, PageSize, - PaginationSpec, ScopeParams, + PaginationSpec, ScopeParams, UploadMethod, UploadResultSpec, UploadSpec, }; use super::ext::{self, Ext, Pos}; @@ -226,10 +226,138 @@ fn parse_download( mime: pick("mime"), size: pick("size"), filename: pick("filename"), + sha256: pick("sha256"), auth, }) } +/// Parse `x-overslash-upload` — the declaration that an action mints a +/// capability to push bytes at a route on the MCP origin, rather than calling a +/// tool at all. +/// +/// `path` and `result.media_path` are mandatory, for the same reason +/// `parse_download` insists on `url`: an upload with nowhere to put the bytes, +/// or one that cannot hand back the reference the send tools take, is not a +/// weaker upload but nothing at all. Everything else is optional metadata, +/// dropped individually when blank so one typo doesn't take the block down. +fn parse_upload( + v: Option<&Value>, + base: &str, + issues: &mut Vec, +) -> Option { + let v = v?; + let p = format!("{base}.x-overslash-upload"); + let Some(obj) = v.as_object() else { + issues.push(ValidationIssue::new( + "upload_malformed", + "x-overslash-upload must be an object with `path` and optional \ + {method, filename_param, auth, max_bytes, result}", + p, + )); + return None; + }; + let path = match obj.get("path").and_then(Value::as_str) { + Some(s) if !s.trim().is_empty() => s.trim().to_string(), + _ => { + issues.push(ValidationIssue::new( + "upload_malformed", + "`path` must be a non-empty path or same-origin URL", + format!("{p}.path"), + )); + return None; + } + }; + let method = match obj.get("method").and_then(Value::as_str) { + None => UploadMethod::default(), + Some(m) if m.eq_ignore_ascii_case("post") => UploadMethod::Post, + Some(m) if m.eq_ignore_ascii_case("put") => UploadMethod::Put, + Some(other) => { + issues.push(ValidationIssue::new( + "upload_malformed", + format!("`method` must be `POST` or `PUT` (got {other:?})"), + format!("{p}.method"), + )); + return None; + } + }; + let auth = match obj.get("auth").and_then(Value::as_str) { + None | Some("inherit") => DownloadAuth::Inherit, + Some("none") => DownloadAuth::None, + Some(other) => { + issues.push(ValidationIssue::new( + "upload_malformed", + format!("`auth` must be `inherit` or `none` (got {other:?})"), + format!("{p}.auth"), + )); + return None; + } + }; + // A non-positive or non-integer ceiling is refused rather than clamped: it + // reads as a deliberate limit, and silently substituting the deployment + // default would be the opposite of what the author wrote. + let max_bytes = match obj.get("max_bytes") { + None | Some(Value::Null) => None, + Some(v) => match v.as_u64().filter(|n| *n > 0) { + Some(n) => Some(n), + None => { + issues.push(ValidationIssue::new( + "upload_malformed", + "`max_bytes` must be a positive integer", + format!("{p}.max_bytes"), + )); + return None; + } + }, + }; + let result = match obj.get("result") { + None | Some(Value::Null) => None, + Some(r) => { + let Some(robj) = r.as_object() else { + issues.push(ValidationIssue::new( + "upload_malformed", + "`result` must be an object of jq expressions", + format!("{p}.result"), + )); + return None; + }; + let rpick = |key: &str| { + robj.get(key) + .and_then(Value::as_str) + .filter(|s| !s.trim().is_empty()) + .map(str::to_string) + }; + let Some(media_path) = rpick("media_path") else { + issues.push(ValidationIssue::new( + "upload_malformed", + "`result.media_path` must be a non-empty jq expression string", + format!("{p}.result.media_path"), + )); + return None; + }; + Some(UploadResultSpec { + media_path, + sha256: rpick("sha256"), + mime: rpick("mime"), + size: rpick("size"), + filename: rpick("filename"), + }) + } + }; + let filename_param = obj + .get("filename_param") + .and_then(Value::as_str) + .filter(|s| !s.trim().is_empty()) + .map(str::to_string); + Some(UploadSpec { + path, + method, + filename_param, + auth, + max_bytes, + result, + }) +} + /// Read an `x-overslash-timeout_ms` (or `x-overslash-default_timeout_ms`) off an /// operation, MCP tool, or `info` object. /// diff --git a/crates/overslash-core/src/openapi/extract/params.rs b/crates/overslash-core/src/openapi/extract/params.rs index ed95fdca..eddcaa3a 100644 --- a/crates/overslash-core/src/openapi/extract/params.rs +++ b/crates/overslash-core/src/openapi/extract/params.rs @@ -5,7 +5,7 @@ use std::collections::HashMap; use serde_json::{Map, Value}; use crate::template_validation::ValidationIssue; -use crate::types::{ActionParam, ParamLocation, ParamResolver, RequestBodySpec}; +use crate::types::{ActionParam, ParamLocation, ParamResolver, RequestBodySpec, ResolveSource}; use super::super::ext::{self, Ext, Pos}; use super::{parse_aliases, parse_instance_config, parse_sql_policy}; @@ -229,13 +229,48 @@ pub(super) fn parse_resolver( }, }; + // `source` names a target the gateway answers itself. Unknown values are + // reported rather than dropped: a typo'd source would otherwise leave a + // resolver with no target at all, which reads downstream as "the author + // forgot one" instead of "the author misspelled one". + let source = match obj.get("source").and_then(Value::as_str) { + None => None, + Some("media") => Some(ResolveSource::Media), + Some(other) => { + issues.push(ValidationIssue::new( + "invalid_resolver_source", + format!("`source` must be `media` (got {other:?})"), + format!("{base}.x-overslash-resolve.source"), + )); + None + } + }; + + // `scope` canonicalizes the *permission key*, and a ledger-resolved value + // describes bytes rather than naming a principal. Canonicalizing on it + // would collapse every file sharing a filename onto one grant — a silent + // authorization widening, so the combination is refused rather than + // half-honored. + let scope = text("scope"); + if source.is_some() && scope.is_some() { + issues.push(ValidationIssue::new( + "invalid_resolver_source", + "`scope` cannot be combined with a `source` resolver: scope renames the \ + permission a call is checked against, and a ledger lookup describes bytes \ + rather than identifying a principal to scope to", + format!("{base}.x-overslash-resolve.scope"), + )); + return None; + } + Some(ParamResolver { + source, get: text("get"), tool: text("tool"), args, pick: text("pick"), display: text("display"), - scope: text("scope"), + scope, cache_ttl, }) } diff --git a/crates/overslash-core/src/search/mod.rs b/crates/overslash-core/src/search/mod.rs index 2b8386f1..663503d3 100644 --- a/crates/overslash-core/src/search/mod.rs +++ b/crates/overslash-core/src/search/mod.rs @@ -225,6 +225,7 @@ mod tests { disabled: false, request_body: None, download: None, + upload: None, } } diff --git a/crates/overslash-core/src/service_layer/mod.rs b/crates/overslash-core/src/service_layer/mod.rs index 115fdafd..dbf121ab 100644 --- a/crates/overslash-core/src/service_layer/mod.rs +++ b/crates/overslash-core/src/service_layer/mod.rs @@ -178,6 +178,7 @@ pub(crate) mod fixtures { disabled: false, request_body: None, download: None, + upload: None, } } diff --git a/crates/overslash-core/src/template_validation/core/action/tests.rs b/crates/overslash-core/src/template_validation/core/action/tests.rs index 1b065140..46c3d497 100644 --- a/crates/overslash-core/src/template_validation/core/action/tests.rs +++ b/crates/overslash-core/src/template_validation/core/action/tests.rs @@ -563,6 +563,7 @@ fn platform_namespace_action_allowed() { disabled: false, request_body: None, download: None, + upload: None, }, ); let r = run(&d); diff --git a/crates/overslash-core/src/template_validation/core/mod.rs b/crates/overslash-core/src/template_validation/core/mod.rs index 75b3f8d8..1e982d7e 100644 --- a/crates/overslash-core/src/template_validation/core/mod.rs +++ b/crates/overslash-core/src/template_validation/core/mod.rs @@ -137,6 +137,7 @@ mod tests { disabled: false, request_body: None, download: None, + upload: None, }, ); m @@ -233,6 +234,7 @@ mod tests { disabled: false, request_body: None, download: None, + upload: None, }, ); ServiceDefinition { diff --git a/crates/overslash-core/src/template_validation/core/resolver.rs b/crates/overslash-core/src/template_validation/core/resolver.rs index 95e8731b..b980d90b 100644 --- a/crates/overslash-core/src/template_validation/core/resolver.rs +++ b/crates/overslash-core/src/template_validation/core/resolver.rs @@ -33,10 +33,19 @@ pub(super) fn check_resolver( if !resolver.has_one_target() { issues.err( "invalid_resolver_target", - "resolver must declare exactly one of `get` (HTTP) or `tool` (MCP)", + "resolver must declare exactly one of `get` (HTTP), `tool` (MCP), or \ + `source` (answered from the gateway's own records)", format!("{base}.resolve"), ); } + if resolver.source.is_some() && !resolver.args.is_empty() { + issues.err( + "invalid_resolver_target", + "resolver.args applies to `tool` resolvers only; a `source` resolver is \ + answered from what the gateway already recorded about this param's value", + format!("{base}.resolve.args"), + ); + } if resolver.get.is_some() && !resolver.args.is_empty() { issues.err( "invalid_resolver_target", @@ -646,6 +655,7 @@ mod tests { disabled: false, request_body: None, download: None, + upload: None, } } } diff --git a/crates/overslash-core/src/types/service/action.rs b/crates/overslash-core/src/types/service/action.rs index bf33eb82..30a0fe87 100644 --- a/crates/overslash-core/src/types/service/action.rs +++ b/crates/overslash-core/src/types/service/action.rs @@ -173,6 +173,18 @@ pub struct ServiceAction { /// field of that descriptor is the object and what it looks like. #[serde(default, skip_serializing_if = "Option::is_none")] pub download: Option, + /// `x-overslash-upload`: this action does not call a tool at all. It mints + /// a capability to push bytes at a plain-HTTP route on the MCP server's own + /// origin, and the gateway intercepts it before it would ever reach + /// `tools/call`. + /// + /// The inbound mirror of [`download`](Self::download), and deliberately not + /// the same shape. A download block is jq over a result that already + /// exists; at upload-mint time nothing has run, so there is no envelope to + /// address — the route half is static and only + /// [`result`](UploadSpec::result) is jq. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub upload: Option, } /// How to turn an MCP tool result into a downloadable object. @@ -201,6 +213,13 @@ pub struct DownloadSpec { /// jq expression yielding a suggested filename. #[serde(default, skip_serializing_if = "Option::is_none")] pub filename: Option, + /// jq expression yielding the object's content hash, when the upstream + /// states one. Not used to fetch anything — it is what lets the gateway + /// record the descriptor in its media ledger, so a later approval that + /// merely *references* these bytes can say what they are instead of + /// showing a reviewer a bare hash. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub sha256: Option, /// Which credential the deferred fetch presents. See [`DownloadAuth`]. #[serde(default)] pub auth: DownloadAuth, @@ -221,6 +240,99 @@ pub enum DownloadAuth { None, } +/// How to mint a capability for pushing bytes at an MCP server's byte route. +/// +/// The declaration exists because `POST /media` is *not* a tool: bytes never +/// ride a JSON-RPC call in either direction, so the inbound half is plain HTTP +/// on the same origin and behind the same credential as `/mcp`. An action +/// carrying this block is served by the gateway itself — it is in the template +/// so that pushing bytes is a permission-checked, approvable, disclosed action +/// like any other, not because the upstream serves a tool by that name. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct UploadSpec { + /// Path (`/media`) or same-origin absolute URL. Resolved against the + /// service instance's own URL by the same joiner the download path uses, + /// and for a sharper reason: on download the untrusted input is the MCP + /// server's *response*, while here it is the *template* — a lower-trust + /// surface once orgs author their own, and a higher-consequence one, since + /// a redemption sends the caller's bytes **and** the instance credential. + pub path: String, + /// Which verb the byte route takes. Parsed rather than free text so a typo + /// is a template error instead of a 405 at redemption. + #[serde(default)] + pub method: UploadMethod, + /// Query parameter carrying the declared filename, when the target takes + /// one. The value always comes from the token, never from whoever redeems + /// it — see the module docs on `routes::uploads`. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub filename_param: Option, + /// Which credential the redemption presents. Reuses [`DownloadAuth`] + /// verbatim: same two meanings, and — the point — the same one place for + /// the `McpAuth::OAuth` refusal to live, rather than two enums for it to + /// drift between. + #[serde(default)] + pub auth: DownloadAuth, + /// Advisory ceiling surfaced to the caller on the minted descriptor. + /// Always clamped by the deployment's `upload_max_bytes`; a template can + /// lower the limit, never raise it. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub max_bytes: Option, + /// jq filters over the byte route's JSON response. This is what makes the + /// block portable across services and what feeds the media ledger. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub result: Option, +} + +/// How to read a byte route's response, as jq over its JSON body. +/// +/// Only [`media_path`](Self::media_path) is required — it is the reference the +/// send tools take, so an upload that cannot produce one has accomplished +/// nothing the caller can use. The rest is what the gateway records so a later +/// approval can describe these bytes. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct UploadResultSpec { + /// jq expression yielding the stored object's reference. + pub media_path: String, + /// jq expression yielding the content hash the upstream computed. + /// + /// Cross-checked against what the gateway measured on the way through, so + /// a disagreement between the two is caught rather than recorded. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub sha256: Option, + /// jq expression yielding the stored MIME type. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub mime: Option, + /// jq expression yielding the stored byte length. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub size: Option, + /// jq expression yielding the stored filename. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub filename: Option, +} + +/// The verb a byte route accepts. +/// +/// Two variants and no `Other`: an upload is a create-or-replace of one object, +/// and a template naming anything else is describing a route this mechanism +/// cannot serve. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "UPPERCASE")] +pub enum UploadMethod { + #[default] + Post, + Put, +} + +impl UploadMethod { + /// The wire spelling, for handing to the HTTP caller. + pub fn as_str(self) -> &'static str { + match self { + UploadMethod::Post => "POST", + UploadMethod::Put => "PUT", + } + } +} + impl ServiceAction { /// The template to interpolate for human-facing surfaces (approval title, /// audit description). Prefers the short [`summary`](Self::summary) and @@ -231,6 +343,20 @@ impl ServiceAction { } } +/// A resolver target that is answered from the gateway's own records instead of +/// from the upstream. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "lowercase")] +pub enum ResolveSource { + /// The media ledger: what the gateway recorded about bytes it moved, keyed + /// by the reference the param carries. Enriches an approval that would + /// otherwise ask a reviewer to sign off on a bare content hash. + /// + /// Best-effort by construction — a reference the gateway never handled is + /// simply absent, and the disclose filter falls back to the raw argument. + Media, +} + /// An operation's declared `requestBody`, reduced to what routing needs: which /// media type to send it as, and whether the upstream demands it. /// @@ -301,6 +427,13 @@ pub struct DisclosureField { /// With it both collapse to `recipient=+34600123456`. #[derive(Debug, Clone, Default, Serialize, Deserialize)] pub struct ParamResolver { + /// Resolve against something the gateway already knows rather than the + /// upstream. Mutually exclusive with `get` and `tool`, and the reason it + /// exists: some params reference bytes, and asking the upstream to describe + /// them would mean a network round-trip and a credential use inside the + /// approval path, for a value the gateway already saw go past. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub source: Option, /// HTTP runtime: GET endpoint path with `{param}` placeholders, e.g. /// `/calendar/v3/calendars/{calendarId}`. Mutually exclusive with `tool`. #[serde(default, skip_serializing_if = "Option::is_none")] @@ -352,9 +485,20 @@ impl ParamResolver { } } - /// Whether exactly one runtime target is declared. + /// Whether exactly one target is declared. + /// + /// Three now, so this counts rather than comparing two bools — `a != b` + /// silently reads "both set" as valid the moment a third arrives. pub fn has_one_target(&self) -> bool { - self.get.is_some() != self.tool.is_some() + [ + self.get.is_some(), + self.tool.is_some(), + self.source.is_some(), + ] + .iter() + .filter(|set| **set) + .count() + == 1 } } diff --git a/crates/overslash-core/src/types/service/definition.rs b/crates/overslash-core/src/types/service/definition.rs index 093b18ea..3e2ec18b 100644 --- a/crates/overslash-core/src/types/service/definition.rs +++ b/crates/overslash-core/src/types/service/definition.rs @@ -441,6 +441,7 @@ mod tests { disabled: false, request_body: None, download: None, + upload: None, }, ); let svc = ServiceDefinition { @@ -510,6 +511,7 @@ mod tests { disabled: false, request_body: None, download: None, + upload: None, }; let j = serde_json::to_value(&a).unwrap(); assert!(j.get("disabled").is_none()); diff --git a/crates/overslash-db/migrations/116_upload_tokens.down.sql b/crates/overslash-db/migrations/116_upload_tokens.down.sql new file mode 100644 index 00000000..00d96529 --- /dev/null +++ b/crates/overslash-db/migrations/116_upload_tokens.down.sql @@ -0,0 +1,2 @@ +DROP TABLE media_descriptors; +DROP TABLE upload_tokens; diff --git a/crates/overslash-db/migrations/116_upload_tokens.up.sql b/crates/overslash-db/migrations/116_upload_tokens.up.sql new file mode 100644 index 00000000..af23d1c3 --- /dev/null +++ b/crates/overslash-db/migrations/116_upload_tokens.up.sql @@ -0,0 +1,142 @@ +-- Proxy uploads: capability tokens for pushing bytes *into* a service, and the +-- ledger of what those bytes turned out to be. +-- +-- The inbound mirror of download_tokens (107). An MCP service whose media moves +-- over plain HTTP — bytes never ride a JSON-RPC call in either direction — has +-- a byte route behind the same credential as its tool endpoint. Overslash could +-- fetch from it and could not push to it, so an agent could forward a file +-- someone sent it but could never send a file it made. Originating bytes was +-- out-of-band work for whoever operated the container. +-- +-- Handing the agent that host and credential instead is the one thing the vault +-- exists to prevent: these credentials are static and unscoped, so the bearer +-- that authorizes an upload authorizes every write the service has. The token +-- below is the narrower thing that did not otherwise exist. +-- +-- A row is a promise in the opposite direction from a download token: "bytes +-- matching this description may be pushed once, by whoever holds the token, +-- until expires_at". The authorization decision is still made and audited at +-- mint time by the ordinary action call; only the bytes are deferred. +CREATE TABLE upload_tokens ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + -- sha256(raw_token), as everywhere else a secret is at rest here. + token_hash BYTEA NOT NULL UNIQUE, + org_id UUID NOT NULL REFERENCES orgs(id) ON DELETE CASCADE, + -- Who the push acts as. Re-checked at redemption, so a deleted identity's + -- outstanding tokens die with it rather than outliving their principal. + identity_id UUID NOT NULL REFERENCES identities(id) ON DELETE CASCADE, + service_instance_id UUID REFERENCES service_instances(id) ON DELETE CASCADE, + service_key TEXT, + action_key TEXT, + -- The upstream request to make when the bytes arrive: {method, url, + -- headers, body}. `body` is always null — the bytes are not here, they + -- arrive at redemption — and secrets are named as SecretRef rather than + -- carried, exactly as download_tokens.request does. + request JSONB NOT NULL, + -- How to re-mint the credential at redemption, never the credential. + credential_ref JSONB NOT NULL DEFAULT '{}'::jsonb, + + -- The declared half: what the caller said it was going to push, fixed at + -- mint time and therefore the thing a reviewer actually approved. + -- + -- declared_sha256 is what makes an approval mean something. Without it the + -- approval authorizes "some bytes, to be chosen later"; with it, exactly + -- one file, because the redemption hashes the stream and refuses to hand + -- back a reference when the two disagree. + declared_sha256 TEXT, + declared_size_bytes BIGINT, + declared_mime TEXT, + declared_filename TEXT, + -- Hard ceiling for this token, already clamped to the deployment's limit. + max_bytes BIGINT NOT NULL, + -- Which query parameter the byte route takes the filename in, from the + -- template. NULL means the route takes none, and the redemption appends + -- nothing rather than guessing a name. + filename_param TEXT, + -- The template's `result` jq block, resolved at mint and carried here. + -- + -- Redemption resolves nothing: it holds a token, not an action key, so it + -- cannot look the declaration back up — the same constraint that puts + -- `timeout_ms` and `pagination` on an approval's replay payload rather than + -- re-deriving them. NULL means the target answers the conventional flat + -- descriptor and needs no filters. + result_spec JSONB, + + -- The stored half: what the upstream actually recorded. Written only on a + -- successful redemption, so a token whose consumed_at is set while these + -- are still null is a push that started and did not land — legible in the + -- table rather than inferred from its absence. + stored_media_path TEXT, + stored_sha256 TEXT, + stored_size_bytes BIGINT, + stored_mime TEXT, + stored_filename TEXT, + completed_at TIMESTAMPTZ, + + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + expires_at TIMESTAMPTZ NOT NULL, + consumed_at TIMESTAMPTZ +); + +-- Single-use, and deliberately the opposite of download_tokens' multi-use rule. +-- Redeeming a download twice re-fetches the same bytes, which is why a resumed +-- `curl -C -` is allowed to; redeeming an upload twice stores two *different* +-- payloads under one authorization, so "what the reviewer approved" would stop +-- having an answer. The claim is an UPDATE guarded on consumed_at IS NULL, so +-- concurrent redemptions cannot both win. +CREATE INDEX upload_tokens_expiry_idx ON upload_tokens (expires_at); + +COMMENT ON TABLE upload_tokens IS + 'Single-use capability tokens for pushing bytes into a service. Minted by an ' + 'action carrying x-overslash-upload, redeemed by POST /v1/uploads/{token}.'; +COMMENT ON COLUMN upload_tokens.token_hash IS + 'sha256 of the raw token; the raw value exists only in the minted URL.'; +COMMENT ON COLUMN upload_tokens.declared_sha256 IS + 'Content hash the caller declared at mint time. Verified against the stream during ' + 'redemption; a mismatch refuses the descriptor so no later call can reference the bytes.'; +COMMENT ON COLUMN upload_tokens.consumed_at IS + 'Set by the claim. A row with consumed_at but no completed_at is a push that started ' + 'and did not land.'; + +-- What the gateway knows about bytes it has moved. +-- +-- Both send tools take a media reference — a content-addressed path — so an +-- approval to send a file could only ever show a reviewer the hash. That is a +-- request to approve something unreadable. The gateway sees a full descriptor +-- at exactly two moments (a download tool's result passing through, and an +-- upload redemption completing), and this is where it keeps them so a later +-- approval can say "invoice-march.pdf (application/pdf, 240 KB)" instead. +-- +-- Scoped to the instance, not just the org: a content address is only +-- meaningful on the host that stores it, so the same hash on two instances is +-- the same bytes but not the same stored object. +CREATE TABLE media_descriptors ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + org_id UUID NOT NULL REFERENCES orgs(id) ON DELETE CASCADE, + service_instance_id UUID REFERENCES service_instances(id) ON DELETE CASCADE, + service_key TEXT, + -- The reference as the send tools take it, e.g. `/media/`. + media_path TEXT NOT NULL, + sha256 TEXT, + mime TEXT, + size_bytes BIGINT, + filename TEXT, + -- 'download' (seen passing through a tool result) or 'upload' (pushed + -- through the gateway). Kept because provenance is the useful thing to show + -- a reviewer next to a filename someone else chose. + source TEXT NOT NULL, + first_seen_at TIMESTAMPTZ NOT NULL DEFAULT now(), + last_seen_at TIMESTAMPTZ NOT NULL DEFAULT now(), + CONSTRAINT media_descriptors_source_check CHECK (source IN ('download', 'upload')) +); + +-- The lookup key, and the upsert target. NULLS NOT DISTINCT so a Mode A row +-- with no instance still collides with itself rather than accumulating +-- duplicates on every re-observation. +CREATE UNIQUE INDEX media_descriptors_ref_idx + ON media_descriptors (org_id, service_instance_id, media_path) NULLS NOT DISTINCT; + +COMMENT ON TABLE media_descriptors IS + 'What the gateway recorded about bytes it moved, so an approval that references them ' + 'can describe them instead of showing a bare content hash. Best-effort: bytes that ' + 'never passed through the gateway are simply absent.'; diff --git a/crates/overslash-db/src/repos/media_descriptor.rs b/crates/overslash-db/src/repos/media_descriptor.rs new file mode 100644 index 00000000..2aa710db --- /dev/null +++ b/crates/overslash-db/src/repos/media_descriptor.rs @@ -0,0 +1,129 @@ +//! `media_descriptors` — what the gateway knows about bytes it has moved. +//! +//! Both halves of the media path deal in *references*: a content-addressed +//! path in, the same path back out. That keeps bytes out of an agent's context, +//! but it means an approval to send a file could only ever show a reviewer a +//! hash. This table is how the gateway answers "what is `/media/<64 hex>`?" +//! without a network round-trip and a credential use inside the approval path. +//! +//! Best-effort by construction. A reference the gateway never handled — bytes +//! pushed to the service out of band — is simply absent, and the disclosure +//! falls back to the raw path. That fallback is lossless: the reviewer still +//! sees exactly the string the call will send, so a miss is "no better than +//! before", never "misleading". + +use sqlx::PgPool; +use time::OffsetDateTime; +use uuid::Uuid; + +#[derive(Debug, Clone, sqlx::FromRow)] +pub struct MediaDescriptorRow { + pub id: Uuid, + pub org_id: Uuid, + pub service_instance_id: Option, + pub service_key: Option, + pub media_path: String, + pub sha256: Option, + pub mime: Option, + pub size_bytes: Option, + pub filename: Option, + pub source: String, + pub first_seen_at: OffsetDateTime, + pub last_seen_at: OffsetDateTime, +} + +/// Where a descriptor was observed. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum MediaSource { + /// Seen passing through a tool result on its way out. + Download, + /// Pushed through the gateway on its way in. + Upload, +} + +impl MediaSource { + fn as_str(self) -> &'static str { + match self { + MediaSource::Download => "download", + MediaSource::Upload => "upload", + } + } +} + +pub struct NewMediaDescriptor<'a> { + pub org_id: Uuid, + pub service_instance_id: Option, + pub service_key: Option<&'a str>, + pub media_path: &'a str, + pub sha256: Option<&'a str>, + pub mime: Option<&'a str>, + pub size_bytes: Option, + pub filename: Option<&'a str>, + pub source: MediaSource, +} + +/// Record a descriptor, or refresh one already known. +/// +/// `COALESCE(EXCLUDED.x, existing.x)` on every metadata column: a later +/// observation that knows less must not erase what an earlier one knew. The +/// motivating case is real — a tool result may carry mime and size while an +/// upload response carries only the path, and the reviewer wants both. +/// +/// `source` is deliberately *not* coalesced but left at its original value: +/// provenance is about where these bytes entered the system, and that does not +/// change when they are seen again. +pub async fn record(pool: &PgPool, d: NewMediaDescriptor<'_>) -> Result<(), sqlx::Error> { + sqlx::query!( + "INSERT INTO media_descriptors ( + org_id, service_instance_id, service_key, media_path, + sha256, mime, size_bytes, filename, source + ) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9) + ON CONFLICT (org_id, service_instance_id, media_path) DO UPDATE + SET sha256 = COALESCE(EXCLUDED.sha256, media_descriptors.sha256), + mime = COALESCE(EXCLUDED.mime, media_descriptors.mime), + size_bytes = COALESCE(EXCLUDED.size_bytes, media_descriptors.size_bytes), + filename = COALESCE(EXCLUDED.filename, media_descriptors.filename), + service_key = COALESCE(EXCLUDED.service_key, media_descriptors.service_key), + last_seen_at = now()", + d.org_id, + d.service_instance_id, + d.service_key, + d.media_path, + d.sha256, + d.mime, + d.size_bytes, + d.filename, + d.source.as_str(), + ) + .execute(pool) + .await?; + Ok(()) +} + +/// Look up one reference, scoped to the org and the instance that stores it. +/// +/// A content address is only meaningful on the host holding the bytes, so the +/// same hash on two instances is the same bytes but not the same stored object +/// — hence the instance in the key rather than the org alone. +pub async fn find( + pool: &PgPool, + org_id: Uuid, + service_instance_id: Option, + media_path: &str, +) -> Result, sqlx::Error> { + sqlx::query_as!( + MediaDescriptorRow, + "SELECT id, org_id, service_instance_id, service_key, media_path, + sha256, mime, size_bytes, filename, source, first_seen_at, last_seen_at + FROM media_descriptors + WHERE org_id = $1 + AND service_instance_id IS NOT DISTINCT FROM $2 + AND media_path = $3", + org_id, + service_instance_id, + media_path, + ) + .fetch_optional(pool) + .await +} diff --git a/crates/overslash-db/src/repos/mod.rs b/crates/overslash-db/src/repos/mod.rs index 91a009ba..5dbee940 100644 --- a/crates/overslash-db/src/repos/mod.rs +++ b/crates/overslash-db/src/repos/mod.rs @@ -37,6 +37,7 @@ pub mod mcp_refresh_token; pub mod mcp_upstream_connection; pub mod mcp_upstream_flow; pub mod mcp_upstream_token; +pub mod media_descriptor; pub mod membership; pub mod oauth_connection_flow; pub mod oauth_mcp_client; @@ -52,6 +53,7 @@ pub mod secret_request; pub mod service_action_embedding; pub mod service_instance; pub mod service_template; +pub mod upload_token; pub mod user; pub mod webhook; pub mod webhook_digest_run; diff --git a/crates/overslash-db/src/repos/upload_token.rs b/crates/overslash-db/src/repos/upload_token.rs new file mode 100644 index 00000000..596db5e6 --- /dev/null +++ b/crates/overslash-db/src/repos/upload_token.rs @@ -0,0 +1,197 @@ +//! `upload_tokens` — single-use capability tokens for pushing bytes into a +//! service. +//! +//! The inbound mirror of [`super::download_token`], and deliberately not a +//! mirror of its *redemption* rule. A download token is multi-use so a dropped +//! transfer can resume; redeeming it twice re-fetches the same bytes. An upload +//! token redeemed twice would store two different payloads under one +//! authorization, so "what the reviewer approved" would stop having an answer. +//! [`claim`] therefore consumes. +//! +//! Only `sha256(raw_token)` is stored. See migration 116 for why a row holds a +//! credential *reference* rather than a credential, and why the declared and +//! stored halves are separate columns rather than one descriptor blob. + +use sqlx::PgPool; +use time::OffsetDateTime; +use uuid::Uuid; + +#[derive(Debug, Clone, sqlx::FromRow)] +pub struct UploadTokenRow { + pub id: Uuid, + pub token_hash: Vec, + pub org_id: Uuid, + pub identity_id: Uuid, + pub service_instance_id: Option, + pub service_key: Option, + pub action_key: Option, + /// The upstream request to make once the bytes arrive: `{method, url, + /// headers, body}`, with `body` always null — the bytes are not in the row. + pub request: serde_json::Value, + /// How to re-resolve the upstream credential at redemption time. + pub credential_ref: serde_json::Value, + /// What the caller said it would push. Fixed at mint, so this is what a + /// reviewer actually approved. + pub declared_sha256: Option, + pub declared_size_bytes: Option, + pub declared_mime: Option, + pub declared_filename: Option, + pub max_bytes: i64, + /// Which query parameter the byte route takes the filename in. `None` + /// means it takes none. + pub filename_param: Option, + /// The template's `result` jq block, resolved at mint. Redemption holds a + /// token rather than an action key, so it cannot look the declaration back + /// up — the spec has to travel with the capability. + pub result_spec: Option, + /// What the upstream recorded. Written only by [`complete`]. + pub stored_media_path: Option, + pub stored_sha256: Option, + pub stored_size_bytes: Option, + pub stored_mime: Option, + pub stored_filename: Option, + pub completed_at: Option, + pub created_at: OffsetDateTime, + pub expires_at: OffsetDateTime, + pub consumed_at: Option, +} + +/// Fields that vary per mint. A struct for the same reason +/// [`super::download_token::NewDownloadToken`] is one: the four adjacent +/// `Option<&str>` declarations are trivially swappable at a call site with no +/// type error to catch it. +pub struct NewUploadToken<'a> { + pub token_hash: &'a [u8], + pub org_id: Uuid, + pub identity_id: Uuid, + pub service_instance_id: Option, + pub service_key: Option<&'a str>, + pub action_key: Option<&'a str>, + pub request: serde_json::Value, + pub credential_ref: serde_json::Value, + pub declared_sha256: Option<&'a str>, + pub declared_size_bytes: Option, + pub declared_mime: Option<&'a str>, + pub declared_filename: Option<&'a str>, + pub max_bytes: i64, + pub filename_param: Option<&'a str>, + pub result_spec: Option, + pub ttl_secs: i64, +} + +/// What the upstream said it stored, for [`complete`]. +pub struct StoredDescriptor<'a> { + pub media_path: &'a str, + pub sha256: Option<&'a str>, + pub size_bytes: Option, + pub mime: Option<&'a str>, + pub filename: Option<&'a str>, +} + +/// Mint a token row. `expires_at = now() + ttl_secs`, from the database clock. +pub async fn create(pool: &PgPool, t: NewUploadToken<'_>) -> Result { + sqlx::query_as!( + UploadTokenRow, + "INSERT INTO upload_tokens ( + token_hash, org_id, identity_id, service_instance_id, + service_key, action_key, request, credential_ref, + declared_sha256, declared_size_bytes, declared_mime, declared_filename, + max_bytes, filename_param, result_spec, expires_at + ) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, + now() + make_interval(secs => $16)) + RETURNING id, token_hash, org_id, identity_id, service_instance_id, + service_key, action_key, request, credential_ref, + declared_sha256, declared_size_bytes, declared_mime, declared_filename, + max_bytes, filename_param, result_spec, stored_media_path, stored_sha256, + stored_size_bytes, + stored_mime, stored_filename, completed_at, + created_at, expires_at, consumed_at", + t.token_hash, + t.org_id, + t.identity_id, + t.service_instance_id, + t.service_key, + t.action_key, + t.request, + t.credential_ref, + t.declared_sha256, + t.declared_size_bytes, + t.declared_mime, + t.declared_filename, + t.max_bytes, + t.filename_param, + t.result_spec, + t.ttl_secs as f64, + ) + .fetch_one(pool) + .await +} + +/// Consume a token for one push: mark it used iff it exists, hasn't expired, +/// and hasn't already been claimed. +/// +/// Unknown, expired and already-consumed all return `None`, so the handler +/// cannot accidentally distinguish them and turn this into an oracle. The +/// guard is in the UPDATE rather than a read-then-write so two concurrent +/// redemptions cannot both win — the loser sees `None` and 404s, which is the +/// correct answer for a token whose one push already went somewhere. +/// +/// Claiming is what *starts* the push, not what completes it: a redemption that +/// then fails upstream leaves a consumed row with no `completed_at`. That is +/// deliberate. Re-arming on failure would mean a caller who can make the +/// upstream fail can re-offer bytes indefinitely against one approval. +pub async fn claim( + pool: &PgPool, + token_hash: &[u8], +) -> Result, sqlx::Error> { + sqlx::query_as!( + UploadTokenRow, + "UPDATE upload_tokens + SET consumed_at = now() + WHERE token_hash = $1 AND expires_at > now() AND consumed_at IS NULL + RETURNING id, token_hash, org_id, identity_id, service_instance_id, + service_key, action_key, request, credential_ref, + declared_sha256, declared_size_bytes, declared_mime, declared_filename, + max_bytes, filename_param, result_spec, stored_media_path, stored_sha256, + stored_size_bytes, + stored_mime, stored_filename, completed_at, + created_at, expires_at, consumed_at", + token_hash, + ) + .fetch_optional(pool) + .await +} + +/// Record what the upstream stored. Only the first call lands; a second +/// returns `None`. +pub async fn complete( + pool: &PgPool, + id: Uuid, + d: StoredDescriptor<'_>, +) -> Result, sqlx::Error> { + sqlx::query_scalar!( + "UPDATE upload_tokens + SET completed_at = now(), stored_media_path = $2, stored_sha256 = $3, + stored_size_bytes = $4, stored_mime = $5, stored_filename = $6 + WHERE id = $1 AND completed_at IS NULL + RETURNING id", + id, + d.media_path, + d.sha256, + d.size_bytes, + d.mime, + d.filename, + ) + .fetch_optional(pool) + .await +} + +/// Drop expired rows. Best-effort housekeeping — an expired token is already +/// unusable via [`claim`], so this only reclaims space. +pub async fn prune_expired(pool: &PgPool) -> Result { + let r = sqlx::query!("DELETE FROM upload_tokens WHERE expires_at < now()") + .execute(pool) + .await?; + Ok(r.rows_affected()) +} diff --git a/dashboard/src/routes/audit/AuditRow.svelte b/dashboard/src/routes/audit/AuditRow.svelte index 1e1d2e0b..bd5aecd5 100644 --- a/dashboard/src/routes/audit/AuditRow.svelte +++ b/dashboard/src/routes/audit/AuditRow.svelte @@ -230,6 +230,29 @@ // use_count well past 1 is the signal that a download URL leaked. const download = $derived(downloadSummary(entry)); + // Upload rows describe a file too, but the useful thing to show is + // different: declared vs measured. The gateway verifies what was pushed + // against what was approved, and a divergence between the two columns is + // the entire signal — this row is the only place it survives. + const upload = $derived(uploadSummary(entry)); + + function uploadSummary(e: AuditEntry) { + if (e.action !== 'action.uploaded') return null; + const d = (e.detail ?? {}) as Record; + const str = (k: string) => (typeof d[k] === 'string' ? (d[k] as string) : null); + const num = (k: string) => (typeof d[k] === 'number' ? (d[k] as number) : null); + return { + filename: str('declared_filename'), + mime: str('declared_mime'), + storedPath: str('stored_media_path'), + declaredSize: num('declared_size_bytes'), + measuredSize: num('measured_size_bytes'), + declaredSha: str('declared_sha256'), + measuredSha: str('measured_sha256'), + error: str('error') + }; + } + function downloadSummary(e: AuditEntry) { if (e.action !== 'action.downloaded') return null; const d = (e.detail ?? {}) as Record; @@ -496,7 +519,43 @@ {/if} {/if} - {#if response && !download} + {#if upload} +
+
File
+
{upload.filename ?? '—'}
+ {#if upload.mime} +
Type
+
{upload.mime}
+ {/if} + {#if upload.measuredSize !== null || upload.declaredSize !== null} +
Size
+
+ {upload.measuredSize !== null ? formatBytes(upload.measuredSize) : '—'} + {#if upload.declaredSize !== null && upload.declaredSize !== upload.measuredSize} + (declared {formatBytes(upload.declaredSize)}) + {/if} +
+ {/if} + {#if upload.measuredSha || upload.declaredSha} +
SHA-256
+
+ {upload.measuredSha ?? '—'} + {#if upload.declaredSha && upload.declaredSha !== upload.measuredSha} + (declared {upload.declaredSha}) + {/if} +
+ {/if} + {#if upload.storedPath} +
Stored as
+
{upload.storedPath}
+ {/if} + {#if upload.error} +
Refused
+
{upload.error}
+ {/if} +
+ {/if} + {#if response && !download && !upload}
response body diff --git a/dashboard/src/routes/audit/types.ts b/dashboard/src/routes/audit/types.ts index 9e159116..a1f9241e 100644 --- a/dashboard/src/routes/audit/types.ts +++ b/dashboard/src/routes/audit/types.ts @@ -136,8 +136,15 @@ export function recordedNames( /** Execution events that carry the normalized `detail.is_error` flag. * `action.downloaded` is a deferred-download redemption: the bytes left the * gateway on that request, so it's an execution for filtering purposes even - * though the originating call happened earlier (`action.deferred`). */ -const EXECUTION_ACTIONS = ['action.executed', 'action.streamed', 'action.downloaded']; + * though the originating call happened earlier (`action.deferred`). + * `action.uploaded` is its inbound twin — bytes entered the service on that + * request, and the call that authorized it happened earlier. */ +const EXECUTION_ACTIONS = [ + 'action.executed', + 'action.streamed', + 'action.downloaded', + 'action.uploaded' +]; /** Upstream-error presence for execution events. Reads the normalized * `detail.is_error` flag; falls back to `detail.status_code` for rows diff --git a/services/whatsapp.yaml b/services/whatsapp.yaml index d70fc74b..5908277d 100644 --- a/services/whatsapp.yaml +++ b/services/whatsapp.yaml @@ -11,9 +11,12 @@ # # Bytes never ride an MCP tool call in either direction. Media moves over # plain HTTP on the container's own port, behind the same bearer token as -# /mcp: `GET /media/` out (wrapped by download_media's `download` -# block below) and `POST /media` in. The inbound half is NOT reachable -# through this gateway — see the comment above send_file. +# /mcp: `GET /media/` out and `POST /media` in. Both halves are +# wrapped here — `download_media`'s `download` block turns the outbound +# reference into a capability URL, and `upload_media`'s `upload` block mints +# one for the inbound push — so an agent can move files in either direction +# without the bytes ever entering its context, and without ever being handed +# the container's URL or its AUTH_TOKEN. openapi: 3.1.0 info: title: WhatsApp @@ -515,6 +518,10 @@ x-overslash-mcp: mime: .structured.mime size: .structured.size filename: .structured.filename + # Not used to fetch anything. It is what lets the gateway record this + # descriptor in its media ledger, so a later send_file that merely + # references these bytes can say what they are. + sha256: .structured.sha256 auth: inherit disclose: - label: Chat @@ -535,9 +542,81 @@ x-overslash-mcp: description: "Stanza id of the media message, as returned by get_conversation / list_messages." required: [chat_jid, message_id] - # Sending media is the mirror image, and the asymmetry matters. Both send - # tools take a `media_path` — a pointer to bytes ALREADY stored in the - # container — and there are exactly two ways to get one: + # Not a tool the container serves. `POST /media` is plain HTTP on the same + # origin and behind the same bearer as /mcp, so this entry exists to make + # the push a permission-checked, approvable, disclosed action like any + # other; the gateway intercepts it before it would ever reach tools/call. + # + # Handing an agent the container's URL and AUTH_TOKEN instead is the one + # thing the vault exists to prevent: that token is static and unscoped, so + # it authorizes every WhatsApp write this container can make, not just an + # upload. This capability is the narrower thing that did not otherwise + # exist. + - name: upload_media + risk: write + description: >- + Get a one-shot URL for pushing a file into this container. Returns an + upload_url with the method it takes, when it expires and the byte + ceiling — send the raw bytes there and you get back a + `/media/` reference to pass to send_file or send_audio_message. The bytes never enter your context in either + direction. Declare sha256 and size_bytes when you know them: they are + verified while the bytes stream through, so a push that does not match + what was approved fails instead of being stored. + upload: + path: /media + method: POST + filename_param: filename + auth: inherit + # The container's own default (MEDIA_MAX_UPLOAD_BYTES). Stated here so + # the caller learns the limit at mint time rather than by hitting a 413 + # after sending 100 MiB. + max_bytes: 104857600 + result: + media_path: .media_path + sha256: .sha256 + mime: .mime + size: .size + filename: .filename + input_schema: + type: object + properties: + filename: + type: string + description: "Name to advertise to the recipient when this is later sent as a document." + mime: + type: string + description: "Content type of the bytes, e.g. image/jpeg. Sniffed by the container when omitted." + size_bytes: + type: integer + minimum: 1 + description: "Exact byte length. Enforced during the push: a transfer that exceeds it is cut off mid-stream." + sha256: + type: string + pattern: "^[0-9a-f]{64}$" + description: >- + Lowercase hex digest of the bytes. Binds this capability to one + specific file: the gateway hashes the stream and refuses to hand + back a media_path when the two disagree. + required: [] + # An upload approval can only show what the caller DECLARED — the bytes + # have not been offered yet. That is not a decoration: a declared sha256 + # is verified while they stream through, so what a reviewer approved is + # what gets stored. Without one, the approval genuinely authorizes "some + # bytes, to be chosen later", and should look like it. + disclose: + - label: "File" + primary: true + filter: '.arguments.filename // "unnamed upload"' + - label: "Size" + filter: ".arguments.size_bytes // empty" + - label: "Type" + filter: ".arguments.mime // empty" + - label: "SHA-256" + filter: ".arguments.sha256 // empty" + + # Sending media is the mirror image. Both send tools take a `media_path` — + # a pointer to bytes ALREADY stored in the container — and there are three + # ways to get one: # # * `download_media` called WITHOUT `deliver: "url"` returns the raw # descriptor, `media_path` included. That is the whole forwarding @@ -545,10 +624,11 @@ x-overslash-mcp: # touching an agent's context. With `deliver: "url"` the gateway # swaps `media_path` for a download_url of its own, which these tools # cannot take. - # * `POST /media` on the container, which needs the container's own URL - # and AUTH_TOKEN. Overslash never hands either to a caller, so - # ORIGINATING new bytes is out-of-band work for whoever operates the - # container — not something an agent can do through this service. + # * `upload_media` above, for originating new bytes. + # * `POST /media` on the container directly, which needs its URL and + # AUTH_TOKEN. Still out-of-band work for whoever operates the + # container, and the one case the gateway cannot describe — see the + # comment on media_path below. - name: send_file risk: write scope_param: recipient @@ -576,7 +656,19 @@ x-overslash-mcp: media_path: type: string minLength: 1 - description: "Reference to stored bytes: the `/media/` path a download_media descriptor carries. A bare `` is accepted too." + description: "Reference to stored bytes: the `/media/` path a download_media or upload_media descriptor carries. A bare `` is accepted too." + # Answered from the gateway's own ledger rather than the container, + # so an approval reads "invoice.pdf (application/pdf, 240912 bytes)" + # instead of asking a reviewer to sign off on a bare hash. + # + # Best-effort by design: bytes pushed to the container out of band + # were never seen here, so their references stay un-enriched and + # the filter below falls back to the raw path. That fallback is + # lossless — the reviewer still sees exactly what will be sent — + # so a miss is never misleading, only less helpful. + resolve: + source: media + display: '{filename}[ ({mime}, {size} bytes)]' media_type: type: string enum: [auto, image, video, audio, document, sticker] @@ -599,7 +691,7 @@ x-overslash-mcp: filter: ".arguments.recipient" - label: "File" primary: true - filter: ".arguments.media_path" + filter: ".resolved.media_path // .arguments.media_path" - label: "Type" filter: '.arguments.media_type // "auto"' - label: "Caption" @@ -635,7 +727,19 @@ x-overslash-mcp: media_path: type: string minLength: 1 - description: "Reference to stored audio: the `/media/` path a download_media descriptor carries. A bare `` is accepted too." + description: "Reference to stored audio: the `/media/` path a download_media or upload_media descriptor carries. A bare `` is accepted too." + # Answered from the gateway's own ledger rather than the container, + # so an approval reads "invoice.pdf (application/pdf, 240912 bytes)" + # instead of asking a reviewer to sign off on a bare hash. + # + # Best-effort by design: bytes pushed to the container out of band + # were never seen here, so their references stay un-enriched and + # the filter below falls back to the raw path. That fallback is + # lossless — the reviewer still sees exactly what will be sent — + # so a miss is never misleading, only less helpful. + resolve: + source: media + display: '{filename}[ ({mime}, {size} bytes)]' reply_to_id: type: string description: "Optional stanza id of the message to quote-reply to." @@ -647,7 +751,7 @@ x-overslash-mcp: filter: ".arguments.recipient" - label: "Voice note" primary: true - filter: ".arguments.media_path" + filter: ".resolved.media_path // .arguments.media_path" - label: "Reply to" filter: ".arguments.reply_to_id // empty"