From ea8c0c09e41c38cdad83ec731530d77b6f0f3289 Mon Sep 17 00:00:00 2001 From: Will Washburn Date: Thu, 5 Mar 2026 07:38:15 -0800 Subject: [PATCH 01/49] Use cloudflare worker as router --- router/index.ts | 57 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 57 insertions(+) create mode 100644 router/index.ts diff --git a/router/index.ts b/router/index.ts new file mode 100644 index 00000000..e8b61e9e --- /dev/null +++ b/router/index.ts @@ -0,0 +1,57 @@ +interface Env { + NEXT_APP_ORIGIN: string; +} + +const proxyRoutes: Record = { + "/observer": "https://observer.relaycast.dev", +}; + +function getOrigin(pathname: string, env: Env): string { + for (const [prefix, origin] of Object.entries(proxyRoutes)) { + if (pathname.startsWith(prefix)) { + return origin; + } + } + return env.NEXT_APP_ORIGIN; +} + +export default { + async fetch(request: Request, env: Env): Promise { + const url = new URL(request.url); + + if (url.hostname === "www.agentrelay.dev") { + url.hostname = "agentrelay.dev"; + return Response.redirect(url.toString(), 301); + } + + const origin = getOrigin(url.pathname, env); + + url.hostname = new URL(origin).hostname; + url.port = ""; + url.protocol = "https:"; + + const subRequest = new Request(url.toString(), request); + + try { + const response = await fetch(subRequest); + + if (!response.body) { + return response; + } + + const { readable, writable } = new TransformStream(); + response.body.pipeTo(writable); + + return new Response(readable, { + status: response.status, + statusText: response.statusText, + headers: new Headers(response.headers), + }); + } catch (error) { + return new Response( + JSON.stringify({ error: (error as Error).message }), + { status: 500 }, + ); + } + }, +}; From dbbd8432b0d14914e76a6e0703e37a2ee8d0abfb Mon Sep 17 00:00:00 2001 From: Will Washburn Date: Thu, 5 Mar 2026 07:49:51 -0800 Subject: [PATCH 02/49] Match routes and strip proxy prefixes Rename getOrigin to matchRoute and return both origin and matched prefix. Update request handling to use the returned origin and to strip the proxy prefix from url.pathname (falling back to "/"), while preserving the default origin from env when no prefix matches. This enables forwarding to proxied origins with proper path rewriting. --- router/index.ts | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/router/index.ts b/router/index.ts index e8b61e9e..a3bd0501 100644 --- a/router/index.ts +++ b/router/index.ts @@ -6,13 +6,13 @@ const proxyRoutes: Record = { "/observer": "https://observer.relaycast.dev", }; -function getOrigin(pathname: string, env: Env): string { +function matchRoute(pathname: string, env: Env): { origin: string; prefix: string } { for (const [prefix, origin] of Object.entries(proxyRoutes)) { if (pathname.startsWith(prefix)) { - return origin; + return { origin, prefix }; } } - return env.NEXT_APP_ORIGIN; + return { origin: env.NEXT_APP_ORIGIN, prefix: "" }; } export default { @@ -24,11 +24,14 @@ export default { return Response.redirect(url.toString(), 301); } - const origin = getOrigin(url.pathname, env); + const { origin, prefix } = matchRoute(url.pathname, env); url.hostname = new URL(origin).hostname; url.port = ""; url.protocol = "https:"; + if (prefix) { + url.pathname = url.pathname.slice(prefix.length) || "/"; + } const subRequest = new Request(url.toString(), request); From 7e62f88611bb2a6323d640f69bfd6e55b4a36956 Mon Sep 17 00:00:00 2001 From: Will Washburn Date: Thu, 5 Mar 2026 12:47:29 -0800 Subject: [PATCH 03/49] fix(router): stop stripping /observer prefix The observer dashboard now uses basePath: /observer, so the worker should pass the path through as-is. Co-Authored-By: Claude Opus 4.6 --- router/index.ts | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/router/index.ts b/router/index.ts index a3bd0501..e8b61e9e 100644 --- a/router/index.ts +++ b/router/index.ts @@ -6,13 +6,13 @@ const proxyRoutes: Record = { "/observer": "https://observer.relaycast.dev", }; -function matchRoute(pathname: string, env: Env): { origin: string; prefix: string } { +function getOrigin(pathname: string, env: Env): string { for (const [prefix, origin] of Object.entries(proxyRoutes)) { if (pathname.startsWith(prefix)) { - return { origin, prefix }; + return origin; } } - return { origin: env.NEXT_APP_ORIGIN, prefix: "" }; + return env.NEXT_APP_ORIGIN; } export default { @@ -24,14 +24,11 @@ export default { return Response.redirect(url.toString(), 301); } - const { origin, prefix } = matchRoute(url.pathname, env); + const origin = getOrigin(url.pathname, env); url.hostname = new URL(origin).hostname; url.port = ""; url.protocol = "https:"; - if (prefix) { - url.pathname = url.pathname.slice(prefix.length) || "/"; - } const subRequest = new Request(url.toString(), request); From e0a07cb0fb101882e1980f855aa046398e360b0c Mon Sep 17 00:00:00 2001 From: Will Washburn Date: Fri, 6 Mar 2026 15:11:53 -0800 Subject: [PATCH 04/49] Add Footer, tweak Header, routes, favicon, types Add a new Footer component and include it on the home and examples pages. Simplify Header by removing the showGitHubStars prop, always fetching GitHub stars, and adding a Star icon alongside the GitHub link. Add a new proxy route "/openclaw" in the router. Update favicon color to #00d9ff. Remove the AgentRelayRouter entry from sst-env.d.ts typings. --- router/index.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/router/index.ts b/router/index.ts index e8b61e9e..3bd9b0cf 100644 --- a/router/index.ts +++ b/router/index.ts @@ -4,6 +4,7 @@ interface Env { const proxyRoutes: Record = { "/observer": "https://observer.relaycast.dev", + "/openclaw": "https://agentrelay.dev", }; function getOrigin(pathname: string, env: Env): string { From 51c4e4bc1509067abeb51c287eb19cfe764ba6e4 Mon Sep 17 00:00:00 2001 From: Will Washburn Date: Fri, 6 Mar 2026 15:21:32 -0800 Subject: [PATCH 05/49] Update index.ts --- router/index.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/router/index.ts b/router/index.ts index 3bd9b0cf..d5033e4e 100644 --- a/router/index.ts +++ b/router/index.ts @@ -4,7 +4,7 @@ interface Env { const proxyRoutes: Record = { "/observer": "https://observer.relaycast.dev", - "/openclaw": "https://agentrelay.dev", + "/openclaw": "https://agentrelay.net", }; function getOrigin(pathname: string, env: Env): string { From b03dc1fd1d52b7181f759a2515a2ca610db2cb5c Mon Sep 17 00:00:00 2001 From: Will Washburn Date: Fri, 6 Mar 2026 15:11:53 -0800 Subject: [PATCH 06/49] Add Footer, tweak Header, routes, favicon, types Add a new Footer component and include it on the home and examples pages. Simplify Header by removing the showGitHubStars prop, always fetching GitHub stars, and adding a Star icon alongside the GitHub link. Add a new proxy route "/openclaw" in the router. Update favicon color to #00d9ff. Remove the AgentRelayRouter entry from sst-env.d.ts typings. --- router/index.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/router/index.ts b/router/index.ts index e8b61e9e..3bd9b0cf 100644 --- a/router/index.ts +++ b/router/index.ts @@ -4,6 +4,7 @@ interface Env { const proxyRoutes: Record = { "/observer": "https://observer.relaycast.dev", + "/openclaw": "https://agentrelay.dev", }; function getOrigin(pathname: string, env: Env): string { From b50c16b1c6f734cf4ccb3c3eb9ab6e2c368e30d5 Mon Sep 17 00:00:00 2001 From: Will Washburn Date: Fri, 6 Mar 2026 15:21:32 -0800 Subject: [PATCH 07/49] Update index.ts --- router/index.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/router/index.ts b/router/index.ts index 3bd9b0cf..d5033e4e 100644 --- a/router/index.ts +++ b/router/index.ts @@ -4,7 +4,7 @@ interface Env { const proxyRoutes: Record = { "/observer": "https://observer.relaycast.dev", - "/openclaw": "https://agentrelay.dev", + "/openclaw": "https://agentrelay.net", }; function getOrigin(pathname: string, env: Env): string { From 1d5dc5c69c28f31bec0135979f14651a82a8383a Mon Sep 17 00:00:00 2001 From: Will Washburn Date: Thu, 19 Mar 2026 09:17:34 -0400 Subject: [PATCH 08/49] fix: resolve Google OAuth redirect_uri_mismatch in production The Cloudflare router rewrites the hostname to the CloudFront origin before proxying, so the Next.js app sees the CloudFront URL instead of agentrelay.dev. This causes the OAuth redirect_uri to not match what's configured in Google Cloud Console. Fix: forward X-Forwarded-Host from the router and use it in the auth routes to construct the correct public origin. Co-Authored-By: Claude Opus 4.6 (1M context) --- router/index.ts | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/router/index.ts b/router/index.ts index d5033e4e..679ae7f9 100644 --- a/router/index.ts +++ b/router/index.ts @@ -31,7 +31,16 @@ export default { url.port = ""; url.protocol = "https:"; - const subRequest = new Request(url.toString(), request); + const headers = new Headers(request.headers); + headers.set("X-Forwarded-Host", request.headers.get("Host") || ""); + headers.set("X-Forwarded-Proto", "https"); + + const subRequest = new Request(url.toString(), { + method: request.method, + headers, + body: request.body, + redirect: "manual", + }); try { const response = await fetch(subRequest); From 1b0299c9488953528e3c9acc44cf8027d0ad89cc Mon Sep 17 00:00:00 2001 From: Will Washburn Date: Thu, 19 Mar 2026 09:24:01 -0400 Subject: [PATCH 09/49] fix: rewrite Location headers in router to preserve agentrelay.dev domain Redirect responses from the CloudFront origin contain Location headers pointing to the CloudFront hostname. Without rewriting, the browser would follow redirects to the internal CloudFront URL instead of agentrelay.dev. Co-Authored-By: Claude Opus 4.6 (1M context) --- router/index.ts | 23 ++++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/router/index.ts b/router/index.ts index 679ae7f9..426822de 100644 --- a/router/index.ts +++ b/router/index.ts @@ -42,20 +42,29 @@ export default { redirect: "manual", }); + const requestHost = request.headers.get("Host") || url.hostname; + try { const response = await fetch(subRequest); + const responseHeaders = new Headers(response.headers); - if (!response.body) { - return response; + const location = responseHeaders.get("Location"); + if (location) { + try { + const loc = new URL(location); + if (loc.hostname === new URL(origin).hostname) { + loc.hostname = requestHost; + responseHeaders.set("Location", loc.toString()); + } + } catch { + // relative URL, leave as-is + } } - const { readable, writable } = new TransformStream(); - response.body.pipeTo(writable); - - return new Response(readable, { + return new Response(response.body, { status: response.status, statusText: response.statusText, - headers: new Headers(response.headers), + headers: responseHeaders, }); } catch (error) { return new Response( From 8923aa467a568cb2f2a3dea9582f5ba1efe56a55 Mon Sep 17 00:00:00 2001 From: Will Washburn Date: Thu, 19 Mar 2026 09:26:47 -0400 Subject: [PATCH 10/49] Use origin URL and forward original Host Create a URL object for the origin and use its hostname when adjusting the proxied URL. Capture the incoming request Host once (requestHost) and set X-Forwarded-Host to that value. Remove the duplicate requestHost declaration and use originUrl.hostname when comparing redirect Location hostnames, replacing them with the original request host when rewriting Location. This ensures the original Host is preserved in headers and redirect rewrites are consistent. --- router/index.ts | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/router/index.ts b/router/index.ts index 426822de..73874621 100644 --- a/router/index.ts +++ b/router/index.ts @@ -25,14 +25,15 @@ export default { return Response.redirect(url.toString(), 301); } - const origin = getOrigin(url.pathname, env); + const requestHost = request.headers.get("Host") || url.hostname; + const originUrl = new URL(getOrigin(url.pathname, env)); - url.hostname = new URL(origin).hostname; + url.hostname = originUrl.hostname; url.port = ""; url.protocol = "https:"; const headers = new Headers(request.headers); - headers.set("X-Forwarded-Host", request.headers.get("Host") || ""); + headers.set("X-Forwarded-Host", requestHost); headers.set("X-Forwarded-Proto", "https"); const subRequest = new Request(url.toString(), { @@ -42,8 +43,6 @@ export default { redirect: "manual", }); - const requestHost = request.headers.get("Host") || url.hostname; - try { const response = await fetch(subRequest); const responseHeaders = new Headers(response.headers); @@ -52,7 +51,7 @@ export default { if (location) { try { const loc = new URL(location); - if (loc.hostname === new URL(origin).hostname) { + if (loc.hostname === originUrl.hostname) { loc.hostname = requestHost; responseHeaders.set("Location", loc.toString()); } From 4665712bb203f72b56ec1d435e78b8d15b26f784 Mon Sep 17 00:00:00 2001 From: Will Washburn Date: Thu, 19 Mar 2026 10:43:46 -0400 Subject: [PATCH 11/49] Set X-Original-Host header and use for auth Add X-Original-Host to proxied requests in the router and update Google auth routes to prefer it (falling back to X-Forwarded-Host, then url.origin). This ensures redirect_uri/origin is computed from the original host reported by a proxy/load-balancer, avoiding mismatches during OAuth flows. Also removes debug logging from the auth start route. --- router/index.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/router/index.ts b/router/index.ts index 73874621..fbde968f 100644 --- a/router/index.ts +++ b/router/index.ts @@ -34,6 +34,7 @@ export default { const headers = new Headers(request.headers); headers.set("X-Forwarded-Host", requestHost); + headers.set("X-Original-Host", requestHost); headers.set("X-Forwarded-Proto", "https"); const subRequest = new Request(url.toString(), { From 01afb311e802167821f5f0f729da7cbb376d42f8 Mon Sep 17 00:00:00 2001 From: Noodle Date: Sat, 21 Mar 2026 08:05:38 -0400 Subject: [PATCH 12/49] feat: add cloud base path and route apex traffic to web --- router/index.ts | 24 +++++++++++------------- 1 file changed, 11 insertions(+), 13 deletions(-) diff --git a/router/index.ts b/router/index.ts index fbde968f..e029bdf9 100644 --- a/router/index.ts +++ b/router/index.ts @@ -2,31 +2,29 @@ interface Env { NEXT_APP_ORIGIN: string; } -const proxyRoutes: Record = { - "/observer": "https://observer.relaycast.dev", - "/openclaw": "https://agentrelay.net", -}; +const FALLBACK_PROXY_ORIGIN = "https://agentrelay.net"; +const PRIMARY_HOST = "agentrelay.dev"; +const WWW_HOST = "www.agentrelay.dev"; -function getOrigin(pathname: string, env: Env): string { - for (const [prefix, origin] of Object.entries(proxyRoutes)) { - if (pathname.startsWith(prefix)) { - return origin; - } +function getOrigin(hostname: string, env: Env): string { + if (hostname === PRIMARY_HOST) { + return env.NEXT_APP_ORIGIN; } - return env.NEXT_APP_ORIGIN; + + return FALLBACK_PROXY_ORIGIN; } export default { async fetch(request: Request, env: Env): Promise { const url = new URL(request.url); - if (url.hostname === "www.agentrelay.dev") { - url.hostname = "agentrelay.dev"; + if (url.hostname === WWW_HOST) { + url.hostname = PRIMARY_HOST; return Response.redirect(url.toString(), 301); } const requestHost = request.headers.get("Host") || url.hostname; - const originUrl = new URL(getOrigin(url.pathname, env)); + const originUrl = new URL(getOrigin(url.hostname, env)); url.hostname = originUrl.hostname; url.port = ""; From da127bd4cf3330fd37785f7e50f11e2a1eeab416 Mon Sep 17 00:00:00 2001 From: Noodle Date: Sat, 21 Mar 2026 08:52:44 -0400 Subject: [PATCH 13/49] fix: keep observer routes on relaycast --- router/index.ts | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/router/index.ts b/router/index.ts index e029bdf9..c7918a7e 100644 --- a/router/index.ts +++ b/router/index.ts @@ -3,10 +3,23 @@ interface Env { } const FALLBACK_PROXY_ORIGIN = "https://agentrelay.net"; +const OBSERVER_ORIGIN = "https://observer.relaycast.dev"; const PRIMARY_HOST = "agentrelay.dev"; const WWW_HOST = "www.agentrelay.dev"; +const OBSERVER_PATH_PREFIX = "/observer"; + +function isObserverPath(pathname: string): boolean { + return ( + pathname === OBSERVER_PATH_PREFIX || + pathname.startsWith(`${OBSERVER_PATH_PREFIX}/`) + ); +} + +function getOrigin(hostname: string, pathname: string, env: Env): string { + if (isObserverPath(pathname)) { + return OBSERVER_ORIGIN; + } -function getOrigin(hostname: string, env: Env): string { if (hostname === PRIMARY_HOST) { return env.NEXT_APP_ORIGIN; } @@ -24,7 +37,7 @@ export default { } const requestHost = request.headers.get("Host") || url.hostname; - const originUrl = new URL(getOrigin(url.hostname, env)); + const originUrl = new URL(getOrigin(url.hostname, url.pathname, env)); url.hostname = originUrl.hostname; url.port = ""; From 73e709e7e476e4bafa54e8433fa2e6229e478975 Mon Sep 17 00:00:00 2001 From: Noodle Date: Sat, 21 Mar 2026 10:15:08 -0400 Subject: [PATCH 14/49] refactor: make apex router split explicit --- router/index.ts | 37 ++++++++++++++++++++++++------------- 1 file changed, 24 insertions(+), 13 deletions(-) diff --git a/router/index.ts b/router/index.ts index c7918a7e..7a8a2037 100644 --- a/router/index.ts +++ b/router/index.ts @@ -7,21 +7,33 @@ const OBSERVER_ORIGIN = "https://observer.relaycast.dev"; const PRIMARY_HOST = "agentrelay.dev"; const WWW_HOST = "www.agentrelay.dev"; const OBSERVER_PATH_PREFIX = "/observer"; +const CLOUD_PATH_PREFIX = "/cloud"; + +function isPathWithinPrefix(pathname: string, prefix: string): boolean { + return pathname === prefix || pathname.startsWith(`${prefix}/`); +} function isObserverPath(pathname: string): boolean { - return ( - pathname === OBSERVER_PATH_PREFIX || - pathname.startsWith(`${OBSERVER_PATH_PREFIX}/`) - ); + return isPathWithinPrefix(pathname, OBSERVER_PATH_PREFIX); } -function getOrigin(hostname: string, pathname: string, env: Env): string { - if (isObserverPath(pathname)) { - return OBSERVER_ORIGIN; - } +function isCloudPath(pathname: string): boolean { + return isPathWithinPrefix(pathname, CLOUD_PATH_PREFIX); +} +export function getOrigin(hostname: string, pathname: string, env: Env): string { + // The agentrelay.dev apex is a split router: + // 1. /observer* stays on the Relaycast observer app + // 2. /cloud* goes to the Next.js cloud app + // 3. everything else falls back to the legacy proxy target if (hostname === PRIMARY_HOST) { - return env.NEXT_APP_ORIGIN; + if (isObserverPath(pathname)) { + return OBSERVER_ORIGIN; + } + + if (isCloudPath(pathname)) { + return env.NEXT_APP_ORIGIN; + } } return FALLBACK_PROXY_ORIGIN; @@ -78,10 +90,9 @@ export default { headers: responseHeaders, }); } catch (error) { - return new Response( - JSON.stringify({ error: (error as Error).message }), - { status: 500 }, - ); + return new Response(JSON.stringify({ error: (error as Error).message }), { + status: 500, + }); } }, }; From a2a21450faaed265b06426d59c1f34b511512d79 Mon Sep 17 00:00:00 2001 From: Noodle Date: Sat, 21 Mar 2026 14:06:26 -0400 Subject: [PATCH 15/49] refactor(web): use NEXT_PUBLIC_APP_URL as canonical app URL --- router/index.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/router/index.ts b/router/index.ts index 7a8a2037..43282b59 100644 --- a/router/index.ts +++ b/router/index.ts @@ -1,5 +1,5 @@ interface Env { - NEXT_APP_ORIGIN: string; + NEXT_PUBLIC_APP_URL: string; } const FALLBACK_PROXY_ORIGIN = "https://agentrelay.net"; @@ -32,7 +32,7 @@ export function getOrigin(hostname: string, pathname: string, env: Env): string } if (isCloudPath(pathname)) { - return env.NEXT_APP_ORIGIN; + return env.NEXT_PUBLIC_APP_URL; } } From 2a657e7fca3f285c1515e8b46f2cf9a629fc7a2a Mon Sep 17 00:00:00 2001 From: Noodle Date: Sat, 21 Mar 2026 15:11:01 -0400 Subject: [PATCH 16/49] refactor: simplify stage-based public app URLs --- router/index.ts | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/router/index.ts b/router/index.ts index 43282b59..af4b08ce 100644 --- a/router/index.ts +++ b/router/index.ts @@ -5,7 +5,6 @@ interface Env { const FALLBACK_PROXY_ORIGIN = "https://agentrelay.net"; const OBSERVER_ORIGIN = "https://observer.relaycast.dev"; const PRIMARY_HOST = "agentrelay.dev"; -const WWW_HOST = "www.agentrelay.dev"; const OBSERVER_PATH_PREFIX = "/observer"; const CLOUD_PATH_PREFIX = "/cloud"; @@ -22,9 +21,9 @@ function isCloudPath(pathname: string): boolean { } export function getOrigin(hostname: string, pathname: string, env: Env): string { - // The agentrelay.dev apex is a split router: + // The production agentrelay.dev apex is a split router: // 1. /observer* stays on the Relaycast observer app - // 2. /cloud* goes to the Next.js cloud app + // 2. /cloud* goes to the cloud app's public www origin // 3. everything else falls back to the legacy proxy target if (hostname === PRIMARY_HOST) { if (isObserverPath(pathname)) { @@ -42,12 +41,6 @@ export function getOrigin(hostname: string, pathname: string, env: Env): string export default { async fetch(request: Request, env: Env): Promise { const url = new URL(request.url); - - if (url.hostname === WWW_HOST) { - url.hostname = PRIMARY_HOST; - return Response.redirect(url.toString(), 301); - } - const requestHost = request.headers.get("Host") || url.hostname; const originUrl = new URL(getOrigin(url.hostname, url.pathname, env)); From 4881a4205b50e93b403188964e283ae65cace3e7 Mon Sep 17 00:00:00 2001 From: Will Washburn Date: Thu, 26 Mar 2026 02:10:13 -0400 Subject: [PATCH 17/49] Always route /cloud to cloud app Move the /cloud path check before the apex-host split so any request whose pathname matches /cloud* returns NEXT_PUBLIC_APP_URL regardless of hostname. Update tests to assert cloud routes resolve to the cloud app on any host and add coverage that non-apex hosts still use the fallback proxy for non-cloud paths. --- router/index.ts | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/router/index.ts b/router/index.ts index af4b08ce..2f9f4a36 100644 --- a/router/index.ts +++ b/router/index.ts @@ -21,18 +21,18 @@ function isCloudPath(pathname: string): boolean { } export function getOrigin(hostname: string, pathname: string, env: Env): string { + // /cloud* always goes to the Next.js cloud app regardless of host + if (isCloudPath(pathname)) { + return env.NEXT_PUBLIC_APP_URL; + } + // The production agentrelay.dev apex is a split router: // 1. /observer* stays on the Relaycast observer app - // 2. /cloud* goes to the cloud app's public www origin - // 3. everything else falls back to the legacy proxy target + // 2. everything else falls back to the legacy proxy target if (hostname === PRIMARY_HOST) { if (isObserverPath(pathname)) { return OBSERVER_ORIGIN; } - - if (isCloudPath(pathname)) { - return env.NEXT_PUBLIC_APP_URL; - } } return FALLBACK_PROXY_ORIGIN; From 6ca15d5127c99b4ee938d519a64cb453de14ce4a Mon Sep 17 00:00:00 2001 From: Will Washburn Date: Thu, 26 Mar 2026 19:51:11 -0400 Subject: [PATCH 18/49] Fix cloud domain Refactor environment/domain handling for the cloud web and router. infra/web.ts now computes routerDomain, originDomain and appUrl (with local dev fallback), configures the Next.js site to use the origin domain via Cloudflare DNS, and exposes NEXT_PUBLIC_APP_URL and S3_BUCKET to the app. The Cloudflare Worker router is only created in non-dev stages and uses routerDomain; it now receives CLOUD_APP_ORIGIN as its environment variable. packages/router/index.ts updated to read CLOUD_APP_ORIGIN instead of NEXT_PUBLIC_APP_URL. sst.config.ts returns appUrl directly (removed previous web/url and DB helper metadata). Tests were updated to reflect the new env var name and canonical domain (removed www prefix) and to assert the router routes to the cloud origin. --- router/index.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/router/index.ts b/router/index.ts index 2f9f4a36..8b60328c 100644 --- a/router/index.ts +++ b/router/index.ts @@ -1,5 +1,5 @@ interface Env { - NEXT_PUBLIC_APP_URL: string; + CLOUD_APP_ORIGIN: string; } const FALLBACK_PROXY_ORIGIN = "https://agentrelay.net"; @@ -23,7 +23,7 @@ function isCloudPath(pathname: string): boolean { export function getOrigin(hostname: string, pathname: string, env: Env): string { // /cloud* always goes to the Next.js cloud app regardless of host if (isCloudPath(pathname)) { - return env.NEXT_PUBLIC_APP_URL; + return env.CLOUD_APP_ORIGIN; } // The production agentrelay.dev apex is a split router: From 07728935670ebf51b7b5f722fe060cb422c0ab35 Mon Sep 17 00:00:00 2001 From: Will Washburn Date: Fri, 27 Mar 2026 10:29:51 -0400 Subject: [PATCH 19/49] Support asset origin and cloud base-path rewrites Add support for an asset origin and improve cloud base-path handling across the router and web app. - infra/web: expose NEXT_PUBLIC_ASSET_ORIGIN to Next.js and add assetOrigin constant. - packages/web/lib/base-path: add getConfiguredAssetOrigin() and withAssetOrigin() to build absolute asset URLs when NEXT_PUBLIC_ASSET_ORIGIN is set; keep withBasePath for routed pages. - packages/web/next.config.ts: set assetPrefix when asset origin is configured and add a rewrite to expose /cloud/:path* as /:path* for the app. - packages/router: introduce strip/add path prefix helpers, getUpstreamPath() and rewriteLocation() to properly strip /cloud for upstream requests and rewrite Location headers back to the mounted public path; include X-Forwarded-Prefix header. - UI updates: use withBasePath for internal links and withAssetOrigin for the favicon in layout. - Tests: extend base-path and router tests to cover asset origin behavior, upstream path stripping, and redirect rewriting. These changes allow proxied deployments where the app is mounted under /cloud while assets may be served from a separate origin, ensuring redirects and links are rewritten correctly. --- router/index.ts | 85 +++++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 76 insertions(+), 9 deletions(-) diff --git a/router/index.ts b/router/index.ts index 8b60328c..bc449ba5 100644 --- a/router/index.ts +++ b/router/index.ts @@ -20,6 +20,73 @@ function isCloudPath(pathname: string): boolean { return isPathWithinPrefix(pathname, CLOUD_PATH_PREFIX); } +function stripPathPrefix(pathname: string, prefix: string): string { + if (pathname === prefix) { + return "/"; + } + + if (pathname.startsWith(`${prefix}/`)) { + return pathname.slice(prefix.length); + } + + return pathname; +} + +function addPathPrefix(pathname: string, prefix: string): string { + if (!prefix) { + return pathname; + } + + if (pathname === "/") { + return prefix; + } + + if (pathname === prefix || pathname.startsWith(`${prefix}/`) || pathname.startsWith(`${prefix}?`)) { + return pathname; + } + + return `${prefix}${pathname}`; +} + +export function getUpstreamPath(pathname: string): string { + if (isCloudPath(pathname)) { + return stripPathPrefix(pathname, CLOUD_PATH_PREFIX); + } + + return pathname; +} + +export function rewriteLocation( + location: string, + originUrl: URL, + requestHost: string, + requestProtocol: string, + mountPrefix = "", +): string { + if (!location) { + return location; + } + + try { + const absolute = new URL(location); + if (absolute.hostname !== originUrl.hostname) { + return location; + } + + absolute.hostname = requestHost; + absolute.port = ""; + absolute.protocol = requestProtocol; + absolute.pathname = addPathPrefix(absolute.pathname, mountPrefix); + return absolute.toString(); + } catch { + if (location.startsWith("/")) { + return addPathPrefix(location, mountPrefix); + } + + return location; + } +} + export function getOrigin(hostname: string, pathname: string, env: Env): string { // /cloud* always goes to the Next.js cloud app regardless of host if (isCloudPath(pathname)) { @@ -43,7 +110,9 @@ export default { const url = new URL(request.url); const requestHost = request.headers.get("Host") || url.hostname; const originUrl = new URL(getOrigin(url.hostname, url.pathname, env)); + const mountPrefix = isCloudPath(url.pathname) ? CLOUD_PATH_PREFIX : ""; + url.pathname = getUpstreamPath(url.pathname); url.hostname = originUrl.hostname; url.port = ""; url.protocol = "https:"; @@ -52,6 +121,9 @@ export default { headers.set("X-Forwarded-Host", requestHost); headers.set("X-Original-Host", requestHost); headers.set("X-Forwarded-Proto", "https"); + if (mountPrefix) { + headers.set("X-Forwarded-Prefix", mountPrefix); + } const subRequest = new Request(url.toString(), { method: request.method, @@ -66,15 +138,10 @@ export default { const location = responseHeaders.get("Location"); if (location) { - try { - const loc = new URL(location); - if (loc.hostname === originUrl.hostname) { - loc.hostname = requestHost; - responseHeaders.set("Location", loc.toString()); - } - } catch { - // relative URL, leave as-is - } + responseHeaders.set( + "Location", + rewriteLocation(location, originUrl, requestHost, "https:", mountPrefix), + ); } return new Response(response.body, { From 7b472bb0e83b6471525cbab4bb70550199e1e253 Mon Sep 17 00:00:00 2001 From: Khaliq Date: Tue, 31 Mar 2026 13:08:15 +0200 Subject: [PATCH 20/49] fix: route /api/* through cloud app in edge router Previously only /cloud/* was routed to the Next.js cloud app. API routes at /api/v1/* fell through to the legacy proxy (agentrelay.net), causing 404s for workspace, CLI auth, and webhook endpoints. Co-Authored-By: Claude Opus 4.6 (1M context) --- router/index.ts | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/router/index.ts b/router/index.ts index bc449ba5..7f8137a6 100644 --- a/router/index.ts +++ b/router/index.ts @@ -87,9 +87,13 @@ export function rewriteLocation( } } +function isApiPath(pathname: string): boolean { + return isPathWithinPrefix(pathname, "/api"); +} + export function getOrigin(hostname: string, pathname: string, env: Env): string { - // /cloud* always goes to the Next.js cloud app regardless of host - if (isCloudPath(pathname)) { + // /cloud* and /api* always go to the Next.js cloud app regardless of host + if (isCloudPath(pathname) || isApiPath(pathname)) { return env.CLOUD_APP_ORIGIN; } From ac1133779a1b12716ad06cad46dd97000687efa2 Mon Sep 17 00:00:00 2001 From: Khaliq Date: Tue, 31 Mar 2026 13:35:25 +0200 Subject: [PATCH 21/49] =?UTF-8?q?fix:=20revert=20/api=20route=20=E2=80=94?= =?UTF-8?q?=20use=20/cloud/api=20prefix=20per=20convention?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Will's review: API routes should go through /cloud/api/*, not /api/*. The /cloud prefix is the established routing convention for the cloud app. Co-Authored-By: Claude Opus 4.6 (1M context) --- router/index.ts | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/router/index.ts b/router/index.ts index 7f8137a6..bc449ba5 100644 --- a/router/index.ts +++ b/router/index.ts @@ -87,13 +87,9 @@ export function rewriteLocation( } } -function isApiPath(pathname: string): boolean { - return isPathWithinPrefix(pathname, "/api"); -} - export function getOrigin(hostname: string, pathname: string, env: Env): string { - // /cloud* and /api* always go to the Next.js cloud app regardless of host - if (isCloudPath(pathname) || isApiPath(pathname)) { + // /cloud* always goes to the Next.js cloud app regardless of host + if (isCloudPath(pathname)) { return env.CLOUD_APP_ORIGIN; } From b9c5b9ecf845efea86dc725d379522bdaaf3a000 Mon Sep 17 00:00:00 2001 From: Will Washburn Date: Tue, 31 Mar 2026 07:56:31 -0400 Subject: [PATCH 22/49] Add README.md for router package documentation --- router/README.md | 47 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) create mode 100644 router/README.md diff --git a/router/README.md b/router/README.md new file mode 100644 index 00000000..e2736bc1 --- /dev/null +++ b/router/README.md @@ -0,0 +1,47 @@ + ┌────────────────────────────────────┐ + │ @agentworkforce/cloud │ + │ │ + │/packages/router (Cloudflare worker)│ + └────────────────────────────────────┘ + + ┌──────────────────────────────────────┐ + │ agentrelay.dev │ + └──────────────────────────────────────┘ + │ + │ + │ + All cloud routes │ Observer Routes + ┌───(/cloud/*, /cloud/api)───────────┼───────────(/observer*)────────────────┐ + │ │ │ + │ │ │ + │ │ │ + ▼ │ ▼ +┌────────────────────────────────────┐ │ ┌────────────────────────────────────┐ +│ https://agentrelay.cloud │ │ │ https://relaycast.dev/observer │ +└────────────────────────────────────┘ │ └────────────────────────────────────┘ + │ + │ +┌────────────────────────────────────┐ All Other Routes ┌────────────────────────────────────┐ +│ @agentworkforce/cloud │ (/, /docs, /blog, /openclaw) │ @agentworkforce/relaycast │ +│ │ │ │ │ +│ /packages/web (NextJs App) │ │ │/packages/observer-dashboard (html) │ +└────────────────────────────────────┘ │ └────────────────────────────────────┘ + │ + │ + │ + │ + │ + │ + │ + │ + ▼ + ┌────────────────────────────────────┐ + │ https://agentrelay.net │ + └────────────────────────────────────┘ + + + ┌────────────────────────────────────┐ + │ @agentworkforce/relay │ + │ │ + │ /web (NextJs App) │ + └────────────────────────────────────┘ From aa7d73a56b46ed9e0a475354ceaf54f9ba349252 Mon Sep 17 00:00:00 2001 From: Will Washburn Date: Tue, 31 Mar 2026 07:56:52 -0400 Subject: [PATCH 23/49] Add code block to README for visual structure --- router/README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/router/README.md b/router/README.md index e2736bc1..fa272ef9 100644 --- a/router/README.md +++ b/router/README.md @@ -1,3 +1,4 @@ +``` ┌────────────────────────────────────┐ │ @agentworkforce/cloud │ │ │ @@ -45,3 +46,4 @@ │ │ │ /web (NextJs App) │ └────────────────────────────────────┘ +``` From 8477e4a83defcb7bf0c111cf0efc57f17f5c5f4a Mon Sep 17 00:00:00 2001 From: Will Washburn Date: Fri, 3 Apr 2026 23:25:11 -0400 Subject: [PATCH 24/49] Switch primary domain from .dev to .com Update the primary domain from agentrelay.dev to agentrelay.com across the codebase. Changes include email sender/DMARC, router domain and routing logic, Next.js app config (app URL and PostHog host), CLI default API URL, default OpenAPI server, footer contact/status links, README diagrams, and corresponding test expectations. This aligns production config and tests with the new .com apex. --- router/README.md | 2 +- router/index.ts | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/router/README.md b/router/README.md index fa272ef9..c9c3861d 100644 --- a/router/README.md +++ b/router/README.md @@ -6,7 +6,7 @@ └────────────────────────────────────┘ ┌──────────────────────────────────────┐ - │ agentrelay.dev │ + │ agentrelay.com │ └──────────────────────────────────────┘ │ │ diff --git a/router/index.ts b/router/index.ts index bc449ba5..497d4083 100644 --- a/router/index.ts +++ b/router/index.ts @@ -4,7 +4,7 @@ interface Env { const FALLBACK_PROXY_ORIGIN = "https://agentrelay.net"; const OBSERVER_ORIGIN = "https://observer.relaycast.dev"; -const PRIMARY_HOST = "agentrelay.dev"; +const PRIMARY_HOST = "agentrelay.com"; const OBSERVER_PATH_PREFIX = "/observer"; const CLOUD_PATH_PREFIX = "/cloud"; @@ -93,7 +93,7 @@ export function getOrigin(hostname: string, pathname: string, env: Env): string return env.CLOUD_APP_ORIGIN; } - // The production agentrelay.dev apex is a split router: + // The production agentrelay.com apex is a split router: // 1. /observer* stays on the Relaycast observer app // 2. everything else falls back to the legacy proxy target if (hostname === PRIMARY_HOST) { From 26d63b9fc331fd6233553c471ce02877f55a5dca Mon Sep 17 00:00:00 2001 From: Will Washburn Date: Sun, 5 Apr 2026 11:20:07 -0400 Subject: [PATCH 25/49] Simplify preview secrets & update migration lookups MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Remove per-stage secret seeding/removal and SST Cloudflare secret declarations from preview workflows (keep Cloudflare deploy creds as environment variables). Update run-drizzle-migrations to read SST_RESOURCE_CloudVpc and SST_RESOURCE_AppDatabase (bastion/host/port) to match the AppDatabase → Aurora swap. Add multiple completed trajectory records (and remove the active trajectory) documenting these changes and related cleanups, and apply supporting updates to router, infra, sst-env types, tests, and READMEs. --- router/index.ts | 4 ---- 1 file changed, 4 deletions(-) diff --git a/router/index.ts b/router/index.ts index 497d4083..afe1a536 100644 --- a/router/index.ts +++ b/router/index.ts @@ -49,10 +49,6 @@ function addPathPrefix(pathname: string, prefix: string): string { } export function getUpstreamPath(pathname: string): string { - if (isCloudPath(pathname)) { - return stripPathPrefix(pathname, CLOUD_PATH_PREFIX); - } - return pathname; } From 250fc920f0126018199229fb7333bab25731d7af Mon Sep 17 00:00:00 2001 From: Will Washburn Date: Fri, 10 Apr 2026 17:36:02 -0400 Subject: [PATCH 26/49] Add shadcn skill, CLI docs, and UI components Introduce a new shadcn skill under .agents including SKILL.md, CLI reference, customization guide, rules (base-vs-radix, composition, forms, icons, styling), MCP server docs, evals, and agent metadata/assets. Add skills lock and skill pointers, many completed trajectory records, and tests/index updates. Add shadcn-based UI primitives and dashboard pieces to packages/web (new ui components, dashboard layouts/pages, hooks, components.json) and update web package metadata and globals.css to integrate the new components and pages. --- router/README.md | 2 +- router/index.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/router/README.md b/router/README.md index c9c3861d..ddc96839 100644 --- a/router/README.md +++ b/router/README.md @@ -37,7 +37,7 @@ │ ▼ ┌────────────────────────────────────┐ - │ https://agentrelay.net │ + │ https://origin.agentrelay.net │ └────────────────────────────────────┘ diff --git a/router/index.ts b/router/index.ts index afe1a536..5c654cd8 100644 --- a/router/index.ts +++ b/router/index.ts @@ -2,7 +2,7 @@ interface Env { CLOUD_APP_ORIGIN: string; } -const FALLBACK_PROXY_ORIGIN = "https://agentrelay.net"; +const FALLBACK_PROXY_ORIGIN = "https://orgin.agentrelay.net/"; const OBSERVER_ORIGIN = "https://observer.relaycast.dev"; const PRIMARY_HOST = "agentrelay.com"; const OBSERVER_PATH_PREFIX = "/observer"; From 7446dbbf330a3fa53e9cdc9150f923caef01689c Mon Sep 17 00:00:00 2001 From: Khaliq Date: Fri, 17 Apr 2026 15:11:52 +0200 Subject: [PATCH 27/49] Expose RelayFile observer from integrations (#201) * feat(cloud): expose relayfile observer from integrations * ci: deploy relayfile observer pages package * ci: verify published relayfile observer package * fix(router): gate file observer mount prefix by host * ci: use latest relayfile observer package --- router/README.md | 19 +++++++++++++++++-- router/index.ts | 36 ++++++++++++++++++++++++++++++++---- 2 files changed, 49 insertions(+), 6 deletions(-) diff --git a/router/README.md b/router/README.md index ddc96839..3dc7e8df 100644 --- a/router/README.md +++ b/router/README.md @@ -1,3 +1,17 @@ +Route precedence: + +- `/cloud*` routes to the Agent Relay Cloud Next.js app. +- `/observer/file*` routes to the RelayFile observer app, expected to be built with `basePath: "/observer/file"`. +- `/observer*` routes to the Relaycast observer app. +- everything else falls back to the legacy proxy target. + +RelayFile observer deployment is wired from the production job in +`.github/workflows/deploy.yml`. The job runs +`scripts/deploy-file-observer-pages.sh`, which packs +`@relayfile/file-observer@latest`, builds it with +`FILE_OBSERVER_BASE_PATH=/observer/file`, and deploys the detected static output +to the Cloudflare Pages project `relayfile-file-observer`. + ``` ┌────────────────────────────────────┐ │ @agentworkforce/cloud │ @@ -12,7 +26,7 @@ │ │ All cloud routes │ Observer Routes - ┌───(/cloud/*, /cloud/api)───────────┼───────────(/observer*)────────────────┐ + ┌───(/cloud/*, /cloud/api)───────────┼────(/observer*, /observer/file*)───────┐ │ │ │ │ │ │ │ │ │ @@ -25,7 +39,8 @@ ┌────────────────────────────────────┐ All Other Routes ┌────────────────────────────────────┐ │ @agentworkforce/cloud │ (/, /docs, /blog, /openclaw) │ @agentworkforce/relaycast │ │ │ │ │ │ -│ /packages/web (NextJs App) │ │ │/packages/observer-dashboard (html) │ +│ /packages/web (NextJs App) │ │ │ /packages/observer-dashboard │ +│ │ │ │ /@relayfile/file-observer │ └────────────────────────────────────┘ │ └────────────────────────────────────┘ │ │ diff --git a/router/index.ts b/router/index.ts index 5c654cd8..e82d61fd 100644 --- a/router/index.ts +++ b/router/index.ts @@ -1,10 +1,13 @@ interface Env { CLOUD_APP_ORIGIN: string; + FILE_OBSERVER_ORIGIN?: string; } -const FALLBACK_PROXY_ORIGIN = "https://orgin.agentrelay.net/"; +const FALLBACK_PROXY_ORIGIN = "https://orgin.agentrelay.net"; const OBSERVER_ORIGIN = "https://observer.relaycast.dev"; +const DEFAULT_FILE_OBSERVER_ORIGIN = "https://relayfile-file-observer.pages.dev"; const PRIMARY_HOST = "agentrelay.com"; +const FILE_OBSERVER_PATH_PREFIX = "/observer/file"; const OBSERVER_PATH_PREFIX = "/observer"; const CLOUD_PATH_PREFIX = "/cloud"; @@ -16,6 +19,14 @@ function isObserverPath(pathname: string): boolean { return isPathWithinPrefix(pathname, OBSERVER_PATH_PREFIX); } +function isFileObserverPath(pathname: string): boolean { + return isPathWithinPrefix(pathname, FILE_OBSERVER_PATH_PREFIX); +} + +function isPrimaryFileObserverPath(hostname: string, pathname: string): boolean { + return hostname === PRIMARY_HOST && isFileObserverPath(pathname); +} + function isCloudPath(pathname: string): boolean { return isPathWithinPrefix(pathname, CLOUD_PATH_PREFIX); } @@ -52,6 +63,18 @@ export function getUpstreamPath(pathname: string): string { return pathname; } +export function getMountPrefix(hostname: string, pathname: string): string { + if (isCloudPath(pathname)) { + return CLOUD_PATH_PREFIX; + } + + if (isPrimaryFileObserverPath(hostname, pathname)) { + return FILE_OBSERVER_PATH_PREFIX; + } + + return ""; +} + export function rewriteLocation( location: string, originUrl: URL, @@ -90,9 +113,14 @@ export function getOrigin(hostname: string, pathname: string, env: Env): string } // The production agentrelay.com apex is a split router: - // 1. /observer* stays on the Relaycast observer app - // 2. everything else falls back to the legacy proxy target + // 1. /observer/file* goes to the RelayFile file observer app + // 2. /observer* stays on the Relaycast observer app + // 3. everything else falls back to the legacy proxy target if (hostname === PRIMARY_HOST) { + if (isPrimaryFileObserverPath(hostname, pathname)) { + return env.FILE_OBSERVER_ORIGIN ?? DEFAULT_FILE_OBSERVER_ORIGIN; + } + if (isObserverPath(pathname)) { return OBSERVER_ORIGIN; } @@ -106,7 +134,7 @@ export default { const url = new URL(request.url); const requestHost = request.headers.get("Host") || url.hostname; const originUrl = new URL(getOrigin(url.hostname, url.pathname, env)); - const mountPrefix = isCloudPath(url.pathname) ? CLOUD_PATH_PREFIX : ""; + const mountPrefix = getMountPrefix(url.hostname, url.pathname); url.pathname = getUpstreamPath(url.pathname); url.hostname = originUrl.hostname; From 368801a0b0470c63c9561e5bbb69e81ff586429c Mon Sep 17 00:00:00 2001 From: Khaliq Date: Fri, 17 Apr 2026 16:00:37 +0200 Subject: [PATCH 28/49] fix(ci): create relayfile observer pages project (#205) --- router/README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/router/README.md b/router/README.md index 3dc7e8df..98defc42 100644 --- a/router/README.md +++ b/router/README.md @@ -10,7 +10,9 @@ RelayFile observer deployment is wired from the production job in `scripts/deploy-file-observer-pages.sh`, which packs `@relayfile/file-observer@latest`, builds it with `FILE_OBSERVER_BASE_PATH=/observer/file`, and deploys the detected static output -to the Cloudflare Pages project `relayfile-file-observer`. +to the Cloudflare Pages project `relayfile-file-observer`. The script creates +that Pages project with the configured production branch when it is missing, so +the first production deploy does not require manual Cloudflare setup. ``` ┌────────────────────────────────────┐ From e2321508c91f35866399f54c654bb5f71baf2e24 Mon Sep 17 00:00:00 2001 From: Khaliq Date: Fri, 17 Apr 2026 17:17:15 +0200 Subject: [PATCH 29/49] fix(router): strip relayfile observer mount prefix (#207) --- router/index.ts | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/router/index.ts b/router/index.ts index e82d61fd..41ba850a 100644 --- a/router/index.ts +++ b/router/index.ts @@ -59,7 +59,11 @@ function addPathPrefix(pathname: string, prefix: string): string { return `${prefix}${pathname}`; } -export function getUpstreamPath(pathname: string): string { +export function getUpstreamPath(hostname: string, pathname: string): string { + if (isPrimaryFileObserverPath(hostname, pathname)) { + return stripPathPrefix(pathname, FILE_OBSERVER_PATH_PREFIX); + } + return pathname; } @@ -136,7 +140,7 @@ export default { const originUrl = new URL(getOrigin(url.hostname, url.pathname, env)); const mountPrefix = getMountPrefix(url.hostname, url.pathname); - url.pathname = getUpstreamPath(url.pathname); + url.pathname = getUpstreamPath(url.hostname, url.pathname); url.hostname = originUrl.hostname; url.port = ""; url.protocol = "https:"; From b789b425bba331a3c38d7899b00a24884458dedd Mon Sep 17 00:00:00 2001 From: Khaliq Date: Fri, 17 Apr 2026 21:50:05 +0200 Subject: [PATCH 30/49] fix(router): preserve relayfile observer redirects (#210) --- router/index.ts | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/router/index.ts b/router/index.ts index 41ba850a..5af53d58 100644 --- a/router/index.ts +++ b/router/index.ts @@ -59,6 +59,18 @@ function addPathPrefix(pathname: string, prefix: string): string { return `${prefix}${pathname}`; } +function hostnameFromHost(host: string, protocol: string): string { + try { + return new URL(`${protocol}//${host}`).hostname; + } catch { + return host.split(":")[0] ?? host; + } +} + +function isPublicFileObserverLocation(hostname: string, pathname: string): boolean { + return isPrimaryFileObserverPath(hostname, pathname); +} + export function getUpstreamPath(hostname: string, pathname: string): string { if (isPrimaryFileObserverPath(hostname, pathname)) { return stripPathPrefix(pathname, FILE_OBSERVER_PATH_PREFIX); @@ -92,6 +104,10 @@ export function rewriteLocation( try { const absolute = new URL(location); + if (isPublicFileObserverLocation(absolute.hostname, absolute.pathname)) { + return location; + } + if (absolute.hostname !== originUrl.hostname) { return location; } @@ -103,6 +119,16 @@ export function rewriteLocation( return absolute.toString(); } catch { if (location.startsWith("/")) { + try { + const requestHostname = hostnameFromHost(requestHost, requestProtocol); + const locationUrl = new URL(location, `${requestProtocol}//${requestHost}`); + if (isPublicFileObserverLocation(requestHostname, locationUrl.pathname)) { + return location; + } + } catch { + // Fall through to the normal mount-prefix rewrite. + } + return addPathPrefix(location, mountPrefix); } From 36bb2e4373be97cdd6b9bcf5644c1d62d0b18373 Mon Sep 17 00:00:00 2001 From: Khaliq Date: Sun, 3 May 2026 10:41:16 +0200 Subject: [PATCH 31/49] fix(workers): audit + replace bare fetch sites (#410) Cloudflare Workers with `nodejs_compat` can hoist a bare `fetch(url, init)` identifier off `globalThis`, throwing `TypeError: Illegal invocation: function called with incorrect 'this' reference`. Convert all bare-fetch call sites in Worker-running packages to `globalThis.fetch(...)` and ensure error paths consume the response body before throwing. Sites changed: - packages/core/src/clone/github-clone-production.ts: fetchGithubTarballDirect - packages/relaycron/src/engine/executor.ts: webhook delivery - packages/router/index.ts: edge proxy subRequest - packages/specialist-worker/src/specialist/agent-card.ts: fetchSpecialistCard (also added body cancel on the !response.ok throw path) Per sage `.claude/rules/workers-fetch.md`. Co-authored-by: GitHub Clone Durable Queue Workflow Co-authored-by: Claude Opus 4.7 (1M context) --- router/index.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/router/index.ts b/router/index.ts index 5af53d58..de7d3cbe 100644 --- a/router/index.ts +++ b/router/index.ts @@ -187,7 +187,10 @@ export default { }); try { - const response = await fetch(subRequest); + // Use `globalThis.fetch` rather than a bare `fetch` identifier: Cloudflare + // Workers can hoist bare `fetch` off `globalThis` and throw + // `TypeError: Illegal invocation`. See sage `.claude/rules/workers-fetch.md`. + const response = await globalThis.fetch(subRequest); const responseHeaders = new Headers(response.headers); const location = responseHeaders.get("Location"); From 3da64fdc56e5206a415ea3c1ffae1b8838776a59 Mon Sep 17 00:00:00 2001 From: Khaliq Date: Thu, 14 May 2026 17:30:14 +0200 Subject: [PATCH 32/49] =?UTF-8?q?feat(migration):=20Phase=202=20=E2=80=94?= =?UTF-8?q?=20webhook=20Worker=20+=20CF=20Queue=20(KV-flagged,=20dormant)?= =?UTF-8?q?=20(#629)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * chore(migration): select Workers + add wrangler devDep Two things this PR does: 1. Lands docs/spikes/cf-spike-report.md with **RECOMMENDATION: WORKERS** as an operator decision. The cf-spike workflow (cloud#614, closed) surfaced one real defect (missing wrangler devDep) but its other probes cascade-failed, producing a conservative FARGATE auto-rec. Operator has accepted the Workers path; the remaining unknowns (SSE behavior, bundle size, node:* surface) become Phase 3 work items gated by the cutover spec's Layer 4. 2. Adds wrangler as devDep in packages/web/ — required peer of @opennextjs/cloudflare. Unblocks Phase 3 builds. Downstream phase workflows grep docs/spikes/cf-spike-report.md for the RECOMMENDATION: WORKERS line; this PR satisfies that gate. Co-Authored-By: Claude Opus 4.7 (1M context) * chore: add migration-phase1-hyperdrive workflow (pre-run snapshot) Self-contained version (no applyCloudRepoSetup) — single-line bash to avoid the helper's JSON.stringify multi-line quoting bug. Stacked on chore/migration-cf-decision (#615) so the spike-report gate passes without waiting for that PR to merge. * chore: document why Phase 1 setup-branch does NOT base on origin/main Per PR #616 review feedback, setup-branch in the emergency-throttle workflow now bases on origin/main. Phase 1 is stacked on chore/migration-cf-decision (#615), so applying the same change here would drop the spike report from the working tree and fail preflight. Adding a comment explaining the divergence so future readers don't 'fix' it. No functional change. * feat(migration): Phase 1 — Hyperdrive in front of RDS (additive) Stands up sst.cf.Hyperdrive pointed at the existing RDS instance and adds a Workers-compatible DB client behind a selectDbClient() factory. Lambda continues to use the existing TCP client unchanged. Changes: - infra/hyperdrive.ts (new): sst.cf.Hyperdrive bound to existing appDatabase - packages/web/lib/db/client-hyperdrive.ts (new): postgres-js + Drizzle over Hyperdrive - packages/web/lib/db/index.ts: selectDbClient(env) factory; HYPERDRIVE binding → Workers client, else TCP - packages/web/lib/boot/resource-check.ts: REQUIRED_RESOURCES gains CloudDb binding entry Stacked on chore/migration-cf-decision (#615). Spike report gate passes. Co-Authored-By: Claude Opus 4.7 (1M context) * fix(phase1): address PR #620 review — use sst.cloudflare.Hyperdrive namespace Devin review flagged that `sst.cf.Hyperdrive` is not a valid namespace — the Cloudflare provider is registered as `cloudflare` in `sst.config.ts`, so SST generates `sst.cloudflare.*` (matches every other Cloudflare resource in `infra/`: Worker, D1, Kv, Queue, Bucket, Cron). `sst.cf` would fail at deploy time when Pulumi tries to instantiate the resource. Co-Authored-By: Claude Opus 4.7 (1M context) * fix(phase1): address PR #620 review — defer CloudDb boot check until link lands Devin review flagged that adding `{ name: "CloudDb", kind: "binding" }` to REQUIRED_RESOURCES before the Hyperdrive resource is wired into `sst.config.ts` and linked into `infra/web.ts` would cause `[boot] resource binding check FAILED` on every Lambda cold start. The false alarm would pollute CloudWatch and mask real missing-resource failures. Per `.claude/rules/sst-resource-registration.md` step 7, the boot-check entry should only land in the same commit that also wires `link: [..., hyperdrive]`. Removing the entry now and leaving a NOTE pointing at the follow-up so the next phase doesn't forget to re-add it. Co-Authored-By: Claude Opus 4.7 (1M context) * fix(phase1): address PR #620 review — cache Hyperdrive client in selectDbClient CodeRabbit review flagged that `selectDbClient(env)` would call `createHyperdriveDb` on every invocation, leaking postgres connection pools when Worker handlers start using it in Phase 2+. Cache the Drizzle client at module scope keyed by the binding identity. In production `env.HYPERDRIVE` is a stable Worker binding so the identity check is a fast-path; in tests that swap envs the identity guard rebuilds correctly. Adds `resetSelectDbClientForTests` for tests. The fallback path (`getCoreDb()`) is unchanged — `@cloud/core` already caches its core client internally. Co-Authored-By: Claude Opus 4.7 (1M context) * chore: add migration-phase2-webhook-worker workflow (pre-run snapshot) Self-contained (no applyCloudRepoSetup) — single-line bash, plain string concat for bash variable interpolation, .onError('fail-fast'), uses sst.cloudflare.* (not sst.cf.*) per #620 review. Stacked on feat/migration-phase1-hyperdrive so the webhook Worker has the Hyperdrive resource to bind to. * feat(migration): Phase 2 — webhook Worker + CF Queue (KV-flagged, dormant) Moves /api/v1/webhooks/* off the OpenNext Lambda onto a dedicated Cloudflare Worker. Router rule is KV-flagged (WEBHOOK_ORIGIN). Default is lambda — ships dormant. Operator flips via: wrangler kv:key put --binding=ROUTER_CONFIG WEBHOOK_ORIGIN worker --remote Deliverables: - packages/webhook-worker/ — CF Worker entry + per-provider handlers (nango, github, composio, hookdeck) with HMAC verify ported verbatim from the Lambda routes. miniflare tests (13/13 passing). - infra/webhook-queue.ts — sst.cloudflare.Queue resource - infra/webhook-worker.ts — sst.cloudflare.Worker linked to hyperdrive + queue + provider secrets - infra/web.ts — router worker now binds ROUTER_CONFIG KV + WEBHOOK_WORKER_ORIGIN env - packages/router/index.ts — KV-flagged forward to the webhook Worker (default: lambda) Stacked on feat/migration-phase1-hyperdrive (#620). Uses sst.cloudflare.* (not sst.cf.*) per #620 review correction. Eliminates webhook contention with the user-facing tier. A Nango/GitHub webhook fan-in storm can no longer 429 dashboard requests once flipped. Co-Authored-By: Claude Opus 4.7 (1M context) * fix(phase2): address PR #629 review — exact-path match for webhook worker Devin flagged that the router's webhook prefix match would capture the Composio OAuth callback (/api/v1/webhooks/composio/connect/callback) when WEBHOOK_ORIGIN=worker, returning 404 instead of falling through to the Lambda. Replace the prefix check in shouldUseWebhookWorker() with a membership check against the exact set of paths the worker knows how to handle (composio, github, hookdeck, nango). Any other sub-path under /api/v1/webhooks/* keeps routing to the Lambda — preserving the OAuth callback and any future Next.js routes that share the prefix. Adds two router unit tests covering the regression: the four exact paths match, and known sub-paths (notably composio/connect/callback) do not. Co-Authored-By: Claude Opus 4.7 --------- Co-authored-by: Claude Opus 4.7 (1M context) --- router/index.ts | 65 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 65 insertions(+) diff --git a/router/index.ts b/router/index.ts index de7d3cbe..c685873b 100644 --- a/router/index.ts +++ b/router/index.ts @@ -1,6 +1,13 @@ interface Env { CLOUD_APP_ORIGIN: string; FILE_OBSERVER_ORIGIN?: string; + ROUTER_CONFIG?: { + get(key: string): Promise; + }; + WEBHOOK_WORKER?: { + fetch(request: Request): Promise; + }; + WEBHOOK_WORKER_ORIGIN?: string; } const FALLBACK_PROXY_ORIGIN = "https://orgin.agentrelay.net"; @@ -10,6 +17,18 @@ const PRIMARY_HOST = "agentrelay.com"; const FILE_OBSERVER_PATH_PREFIX = "/observer/file"; const OBSERVER_PATH_PREFIX = "/observer"; const CLOUD_PATH_PREFIX = "/cloud"; +const WEBHOOK_ORIGIN_FLAG_KEY = "WEBHOOK_ORIGIN"; + +// Exact paths the webhook worker handles. Other sub-paths under +// /api/v1/webhooks (notably /api/v1/webhooks/composio/connect/callback, an +// OAuth callback served by Next.js) must continue to route to the Lambda even +// when WEBHOOK_ORIGIN=worker, otherwise they 404 against the Worker. +const WEBHOOK_WORKER_PATHS = new Set([ + "/api/v1/webhooks/composio", + "/api/v1/webhooks/github", + "/api/v1/webhooks/hookdeck", + "/api/v1/webhooks/nango", +]); function isPathWithinPrefix(pathname: string, prefix: string): boolean { return pathname === prefix || pathname.startsWith(`${prefix}/`); @@ -31,6 +50,13 @@ function isCloudPath(pathname: string): boolean { return isPathWithinPrefix(pathname, CLOUD_PATH_PREFIX); } +// True only for the exact paths the webhook worker knows how to handle. Used +// to gate worker forwarding so unrelated routes under /api/v1/webhooks/* (e.g. +// the Composio OAuth callback) still reach the Lambda. +export function isWebhookWorkerPath(pathname: string): boolean { + return WEBHOOK_WORKER_PATHS.has(pathname); +} + function stripPathPrefix(pathname: string, prefix: string): string { if (pathname === prefix) { return "/"; @@ -159,9 +185,48 @@ export function getOrigin(hostname: string, pathname: string, env: Env): string return FALLBACK_PROXY_ORIGIN; } +async function shouldUseWebhookWorker(pathname: string, env: Env): Promise { + if (!isWebhookWorkerPath(pathname)) { + return false; + } + + try { + const configured = await env.ROUTER_CONFIG?.get(WEBHOOK_ORIGIN_FLAG_KEY); + return configured?.trim().toLowerCase() === "worker"; + } catch { + return false; + } +} + +function buildWorkerOriginRequest(request: Request, requestUrl: URL, workerOrigin: string): Request { + const targetUrl = new URL(requestUrl.toString()); + const originUrl = new URL(workerOrigin); + targetUrl.protocol = originUrl.protocol; + targetUrl.hostname = originUrl.hostname; + targetUrl.port = originUrl.port; + + return new Request(targetUrl.toString(), { + method: request.method, + headers: request.headers, + body: request.body, + redirect: "manual", + }); +} + export default { async fetch(request: Request, env: Env): Promise { const url = new URL(request.url); + if (await shouldUseWebhookWorker(url.pathname, env)) { + if (env.WEBHOOK_WORKER) { + return env.WEBHOOK_WORKER.fetch(request); + } + + const workerOrigin = env.WEBHOOK_WORKER_ORIGIN?.trim(); + if (workerOrigin) { + return globalThis.fetch(buildWorkerOriginRequest(request, url, workerOrigin)); + } + } + const requestHost = request.headers.get("Host") || url.hostname; const originUrl = new URL(getOrigin(url.hostname, url.pathname, env)); const mountPrefix = getMountPrefix(url.hostname, url.pathname); From 8300d1cfc4e8f63541b8b1785a8a151ed23255f2 Mon Sep 17 00:00:00 2001 From: Khaliq Date: Thu, 14 May 2026 17:50:59 +0200 Subject: [PATCH 33/49] =?UTF-8?q?feat(migration):=20Phase=200=20=E2=80=94?= =?UTF-8?q?=20production=20traffic=20recorder=20+=20R2=20(#619)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * chore: add migration-phase0-recorder workflow (pre-run snapshot) Self-contained version — inlines setup-branch + install-deps to avoid applyCloudRepoSetup's multi-line bash quoting bug. Committing pre-run so ricky auto-fix mutations are recoverable via git checkout. * feat(migration): Phase 0 — production traffic recorder + R2 Adds NDJSON request/response capture to the existing CF router worker. Redaction, deny-list for secret-bearing routes, 14-day R2 lifecycle. KV-tunable sample rate (default 100%). Additive — no routing behavior change. Feeds Phase 3 Layer 3 replay corpus. Co-Authored-By: Claude Opus 4.7 (1M context) --------- Co-authored-by: Claude Opus 4.7 (1M context) --- router/index.ts | 30 ++++-- router/src/recorder.ts | 140 +++++++++++++++++++++++++ router/src/redact.ts | 132 ++++++++++++++++++++++++ router/test/recorder.test.ts | 192 +++++++++++++++++++++++++++++++++++ router/test/redact.test.ts | 78 ++++++++++++++ router/tsconfig.json | 17 ++++ 6 files changed, 580 insertions(+), 9 deletions(-) create mode 100644 router/src/recorder.ts create mode 100644 router/src/redact.ts create mode 100644 router/test/recorder.test.ts create mode 100644 router/test/redact.test.ts create mode 100644 router/tsconfig.json diff --git a/router/index.ts b/router/index.ts index c685873b..46b01ddf 100644 --- a/router/index.ts +++ b/router/index.ts @@ -1,15 +1,20 @@ +import { maybeRecord, type RecorderEnv } from "./src/recorder.js"; + interface Env { CLOUD_APP_ORIGIN: string; FILE_OBSERVER_ORIGIN?: string; - ROUTER_CONFIG?: { - get(key: string): Promise; - }; + TRAFFIC_RECORDER?: RecorderEnv["TRAFFIC_RECORDER"]; + ROUTER_CONFIG?: RecorderEnv["ROUTER_CONFIG"]; WEBHOOK_WORKER?: { fetch(request: Request): Promise; }; WEBHOOK_WORKER_ORIGIN?: string; } +function hasRecorderEnv(env: Env): env is Env & RecorderEnv { + return Boolean(env.TRAFFIC_RECORDER && env.ROUTER_CONFIG); +} + const FALLBACK_PROXY_ORIGIN = "https://orgin.agentrelay.net"; const OBSERVER_ORIGIN = "https://observer.relaycast.dev"; const DEFAULT_FILE_OBSERVER_ORIGIN = "https://relayfile-file-observer.pages.dev"; @@ -214,7 +219,7 @@ function buildWorkerOriginRequest(request: Request, requestUrl: URL, workerOrigi } export default { - async fetch(request: Request, env: Env): Promise { + async fetch(request: Request, env: Env, ctx: ExecutionContext): Promise { const url = new URL(request.url); if (await shouldUseWebhookWorker(url.pathname, env)) { if (env.WEBHOOK_WORKER) { @@ -244,6 +249,7 @@ export default { headers.set("X-Forwarded-Prefix", mountPrefix); } + const recordingRequest = hasRecorderEnv(env) ? (request.clone() as unknown as Request) : null; const subRequest = new Request(url.toString(), { method: request.method, headers, @@ -255,8 +261,8 @@ export default { // Use `globalThis.fetch` rather than a bare `fetch` identifier: Cloudflare // Workers can hoist bare `fetch` off `globalThis` and throw // `TypeError: Illegal invocation`. See sage `.claude/rules/workers-fetch.md`. - const response = await globalThis.fetch(subRequest); - const responseHeaders = new Headers(response.headers); + const upstreamResponse = await globalThis.fetch(subRequest); + const responseHeaders = new Headers(upstreamResponse.headers); const location = responseHeaders.get("Location"); if (location) { @@ -266,11 +272,17 @@ export default { ); } - return new Response(response.body, { - status: response.status, - statusText: response.statusText, + const response = new Response(upstreamResponse.body, { + status: upstreamResponse.status, + statusText: upstreamResponse.statusText, headers: responseHeaders, }); + + if (recordingRequest && hasRecorderEnv(env)) { + ctx.waitUntil(maybeRecord(recordingRequest, response.clone(), env, ctx)); + } + + return response; } catch (error) { return new Response(JSON.stringify({ error: (error as Error).message }), { status: 500, diff --git a/router/src/recorder.ts b/router/src/recorder.ts new file mode 100644 index 00000000..be92f1e7 --- /dev/null +++ b/router/src/recorder.ts @@ -0,0 +1,140 @@ +import { redactBody, redactHeaders, shouldRecord } from "./redact.js"; + +const DEFAULT_SAMPLE_RATE = 100; +const MAX_BODY_BYTES = 256 * 1024; +const OVERSIZE_PREFIX = "[OVERSIZE:"; +const textEncoder = new TextEncoder(); + +export interface RecorderEnv { + TRAFFIC_RECORDER: R2Bucket; + ROUTER_CONFIG: KVNamespace; +} + +type RecorderContext = Pick; +type BodyReader = { + body: ReadableStream | null; + bodyUsed: boolean; + headers: Headers; + text(): Promise; +}; + +type TrafficRecord = { + ts: string; + method: string; + path: string; + query: string; + request_headers: Record; + request_body: string | null; + response_status: number; + response_headers: Record; + response_body: string | null; + request_id: string; +}; + +function normalizeSampleRate(value: string | null): number { + const parsed = Number(value); + if (!Number.isFinite(parsed)) { + return DEFAULT_SAMPLE_RATE; + } + + return Math.min(100, Math.max(0, parsed)); +} + +function makeOversizePlaceholder(size: number): string { + return `${OVERSIZE_PREFIX}${size}]`; +} + +async function readBody( + payload: BodyReader | null, + path: string, + contentType: string | null, +): Promise { + if (!payload || payload.body == null || payload.bodyUsed) { + return null; + } + + const body = await payload.text(); + const redactedBody = redactBody(path, body, contentType); + if (redactedBody == null) { + return null; + } + + const bodySize = textEncoder.encode(redactedBody).length; + if (bodySize > MAX_BODY_BYTES) { + return makeOversizePlaceholder(bodySize); + } + + return redactedBody; +} + +function padDatePart(value: number): string { + return String(value).padStart(2, "0"); +} + +function buildRecordKey(timestamp: Date, requestId: string): string { + const year = timestamp.getUTCFullYear(); + const month = padDatePart(timestamp.getUTCMonth() + 1); + const day = padDatePart(timestamp.getUTCDate()); + const hour = padDatePart(timestamp.getUTCHours()); + return `corpus/${year}/${month}/${day}/${hour}/${requestId}.ndjson`; +} + +function getRequestId(request: Request): string { + return ( + request.headers.get("x-request-id") ?? + request.headers.get("cf-ray") ?? + request.headers.get("traceparent") ?? + crypto.randomUUID() + ); +} + +export async function maybeRecord( + request: Request, + response: Response, + env: RecorderEnv, + ctx: RecorderContext, +): Promise { + try { + const url = new URL(request.url); + if (!shouldRecord(url.pathname)) { + return; + } + + const sampleRate = normalizeSampleRate(await env.ROUTER_CONFIG.get("RECORDER_SAMPLE_RATE")); + if (Math.random() * 100 >= sampleRate) { + return; + } + + const requestClone = request.bodyUsed ? null : request.clone(); + const responseClone = response.bodyUsed ? null : response.clone(); + const timestamp = new Date(); + const requestId = getRequestId(request); + + const record: TrafficRecord = { + ts: timestamp.toISOString(), + method: request.method, + path: url.pathname, + query: url.search.startsWith("?") ? url.search.slice(1) : url.search, + request_headers: redactHeaders(request.headers), + request_body: await readBody( + requestClone, + url.pathname, + requestClone?.headers.get("content-type") ?? null, + ), + response_status: response.status, + response_headers: redactHeaders(response.headers), + response_body: await readBody( + responseClone, + url.pathname, + responseClone?.headers.get("content-type") ?? null, + ), + request_id: requestId, + }; + + const line = `${JSON.stringify(record)}\n`; + const key = buildRecordKey(timestamp, requestId); + ctx.waitUntil(env.TRAFFIC_RECORDER.put(key, line)); + } catch (error) { + console.error("traffic recorder failed", error); + } +} diff --git a/router/src/redact.ts b/router/src/redact.ts new file mode 100644 index 00000000..b633f12f --- /dev/null +++ b/router/src/redact.ts @@ -0,0 +1,132 @@ +const HEADER_DENY_LIST = new Set([ + "authorization", + "cookie", + "set-cookie", + "x-relayauth-token", + "x-api-key", +]); + +const SENSITIVE_HEADER_NAME = /(token|secret|password|signature)/i; + +const FULL_BODY_REDACTION_PATHS = [ + /^\/api\/v1\/workspaces\/[a-zA-Z0-9_-]+\/secrets(?:\/.*)?$/, + /^\/api\/auth(?:\/.*)?$/, + /^\/api\/v1\/auth(?:\/.*)?$/, + /^\/api\/v1\/cli\/auth(?:\/.*)?$/, + /^\/api\/v1\/cli\/login(?:\/.*)?$/, + /^\/api\/v1\/integrations\/nango\/connect-link(?:\/.*)?$/, + /^\/api\/v1\/workspaces\/[a-zA-Z0-9_-]+\/integrations\/connect-session(?:\/.*)?$/, +]; + +const JSON_SECRET_KEYS = new Set([ + "token", + "accesstoken", + "refreshtoken", + "secret", + "apikey", + "password", + "clientsecret", + "privatekey", +]); + +type HeaderInput = Headers | Iterable<[string, string]> | Record; + +const textEncoder = new TextEncoder(); + +function utf8ByteLength(value: string): number { + return textEncoder.encode(value).length; +} + +function redactValue(value: string): string { + return `[REDACTED:${utf8ByteLength(value)}]`; +} + +function shouldRedactHeader(name: string): boolean { + const normalized = name.toLowerCase(); + return HEADER_DENY_LIST.has(normalized) || SENSITIVE_HEADER_NAME.test(normalized); +} + +function headerEntries(headers: HeaderInput): Iterable<[string, string]> { + if (headers instanceof Headers) { + const entries: Array<[string, string]> = []; + headers.forEach((value, name) => { + entries.push([name, value]); + }); + return entries; + } + + if (Symbol.iterator in Object(headers)) { + return headers as Iterable<[string, string]>; + } + + return Object.entries(headers); +} + +function shouldFullyRedactBody(path: string): boolean { + return FULL_BODY_REDACTION_PATHS.some((pattern) => pattern.test(path)); +} + +function isJsonContentType(contentType: string | null | undefined): boolean { + return typeof contentType === "string" && contentType.toLowerCase().includes("application/json"); +} + +function redactJsonValue(value: unknown): unknown { + if (Array.isArray(value)) { + return value.map((item) => redactJsonValue(item)); + } + + if (!value || typeof value !== "object") { + return value; + } + + const output: Record = {}; + for (const [key, nested] of Object.entries(value)) { + output[key] = JSON_SECRET_KEYS.has(key.toLowerCase()) + ? redactValue(typeof nested === "string" ? nested : JSON.stringify(nested) ?? "") + : redactJsonValue(nested); + } + + return output; +} + +export function redactHeaders(headers: HeaderInput): Record { + const output: Record = {}; + + for (const [name, value] of headerEntries(headers)) { + output[name] = shouldRedactHeader(name) ? redactValue(value) : value; + } + + return output; +} + +export function redactBody( + path: string, + body: string | null | undefined, + contentType?: string | null, +): string | null | undefined { + if (body == null) { + return body; + } + + if (shouldFullyRedactBody(path)) { + return redactValue(body); + } + + if (!isJsonContentType(contentType)) { + return body; + } + + try { + return JSON.stringify(redactJsonValue(JSON.parse(body))); + } catch { + return body; + } +} + +export function shouldRecord(path: string): boolean { + if (path === "/favicon.ico" || path === "/api/health") { + return false; + } + + return path !== "/observer" && !path.startsWith("/observer/"); +} diff --git a/router/test/recorder.test.ts b/router/test/recorder.test.ts new file mode 100644 index 00000000..bd59dc9f --- /dev/null +++ b/router/test/recorder.test.ts @@ -0,0 +1,192 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +import { maybeRecord, type RecorderEnv } from "../src/recorder.js"; + +const FIXED_TIME = new Date("2026-05-14T09:08:07.000Z"); +const REQUEST_ID = "123e4567-e89b-12d3-a456-426614174000"; +const KEY_PATTERN = /^corpus\/\d{4}\/\d{2}\/\d{2}\/\d{2}\/[a-f0-9-]+\.ndjson$/; + +function createEnv(sampleRate: string): { + env: RecorderEnv; + put: ReturnType; + get: ReturnType; +} { + const put = vi.fn(async () => undefined); + const get = vi.fn(async () => sampleRate); + + return { + env: { + TRAFFIC_RECORDER: { + put, + } as unknown as R2Bucket, + ROUTER_CONFIG: { + get, + } as unknown as KVNamespace, + }, + put, + get, + }; +} + +function createCtx(): { + ctx: Pick; + waitUntil: ReturnType; + waits: Promise[]; +} { + const waits: Promise[] = []; + const waitUntil = vi.fn((promise: Promise) => { + waits.push(promise); + }); + + return { + ctx: { waitUntil }, + waitUntil, + waits, + }; +} + +describe("maybeRecord", () => { + beforeEach(() => { + vi.useFakeTimers(); + vi.setSystemTime(FIXED_TIME); + vi.spyOn(Math, "random").mockReturnValue(0); + }); + + afterEach(() => { + vi.restoreAllMocks(); + vi.useRealTimers(); + }); + + it("writes a record when sampling is 100 percent", async () => { + const { env, put, get } = createEnv("100"); + const { ctx, waitUntil, waits } = createCtx(); + const request = new Request("https://router.test/api/v1/messages?foo=bar", { + method: "POST", + headers: { + "content-type": "application/json", + authorization: "Bearer secret-token", + "x-request-id": REQUEST_ID, + }, + body: JSON.stringify({ + prompt: "hello", + token: "secret-token", + }), + }); + const response = new Response( + JSON.stringify({ + ok: true, + refreshToken: "refresh-me", + }), + { + status: 201, + headers: { + "content-type": "application/json", + "set-cookie": "sid=123", + }, + }, + ); + + await maybeRecord(request, response, env, ctx); + await Promise.all(waits); + + expect(get).toHaveBeenCalledWith("RECORDER_SAMPLE_RATE"); + expect(waitUntil).toHaveBeenCalledOnce(); + expect(put).toHaveBeenCalledOnce(); + + const [key, line] = put.mock.calls[0] as [string, string]; + expect(key).toBe("corpus/2026/05/14/09/123e4567-e89b-12d3-a456-426614174000.ndjson"); + expect(key).toMatch(KEY_PATTERN); + + const record = JSON.parse(line) as Record; + expect(record).toEqual({ + ts: "2026-05-14T09:08:07.000Z", + method: "POST", + path: "/api/v1/messages", + query: "foo=bar", + request_headers: { + authorization: "[REDACTED:19]", + "content-type": "application/json", + "x-request-id": REQUEST_ID, + }, + request_body: JSON.stringify({ + prompt: "hello", + token: "[REDACTED:12]", + }), + response_status: 201, + response_headers: { + "content-type": "application/json", + "set-cookie": "[REDACTED:7]", + }, + response_body: JSON.stringify({ + ok: true, + refreshToken: "[REDACTED:10]", + }), + request_id: REQUEST_ID, + }); + }); + + it("does not write when sampling is 0 percent", async () => { + const { env, put } = createEnv("0"); + const { ctx, waitUntil } = createCtx(); + + await maybeRecord( + new Request("https://router.test/api/v1/messages", { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ ok: true }), + }), + new Response("ok"), + env, + ctx, + ); + + expect(waitUntil).not.toHaveBeenCalled(); + expect(put).not.toHaveBeenCalled(); + }); + + it("skips deny-listed paths without touching storage", async () => { + const { env, put, get } = createEnv("100"); + const { ctx, waitUntil } = createCtx(); + + await maybeRecord( + new Request("https://router.test/observer/session/123"), + new Response("ok"), + env, + ctx, + ); + + expect(get).not.toHaveBeenCalled(); + expect(waitUntil).not.toHaveBeenCalled(); + expect(put).not.toHaveBeenCalled(); + }); + + it("replaces oversize bodies with a placeholder before writing", async () => { + const { env, put } = createEnv("100"); + const { ctx, waits } = createCtx(); + const oversized = "x".repeat(256 * 1024 + 1); + + await maybeRecord( + new Request("https://router.test/api/v1/messages", { + method: "POST", + headers: { + "content-type": "text/plain", + "x-request-id": REQUEST_ID, + }, + body: oversized, + }), + new Response("ok", { + headers: { "content-type": "text/plain" }, + }), + env, + ctx, + ); + await Promise.all(waits); + + expect(put).toHaveBeenCalledOnce(); + const [, line] = put.mock.calls[0] as [string, string]; + const record = JSON.parse(line) as { request_body: string; response_body: string }; + + expect(record.request_body).toBe("[OVERSIZE:262145]"); + expect(record.response_body).toBe("ok"); + }); +}); diff --git a/router/test/redact.test.ts b/router/test/redact.test.ts new file mode 100644 index 00000000..f5302c2d --- /dev/null +++ b/router/test/redact.test.ts @@ -0,0 +1,78 @@ +import { describe, expect, it } from "vitest"; + +import { redactBody, redactHeaders, shouldRecord } from "../src/redact.js"; + +describe("router redaction", () => { + it("redacts deny-listed and sensitive headers", () => { + const headers = redactHeaders({ + authorization: "Bearer secret-token", + "x-api-key": "topsecret", + "x-signature": "sig-123", + "content-type": "application/json", + }); + + expect(headers).toEqual({ + authorization: "[REDACTED:19]", + "x-api-key": "[REDACTED:9]", + "x-signature": "[REDACTED:7]", + "content-type": "application/json", + }); + }); + + it("fully redacts bodies for deny-listed paths", () => { + expect( + redactBody( + "/api/v1/workspaces/ws_123/secrets", + JSON.stringify({ apiKey: "secret-value" }), + "application/json", + ), + ).toBe("[REDACTED:25]"); + }); + + it("scrubs sensitive keys from JSON bodies", () => { + expect( + redactBody( + "/api/v1/messages", + JSON.stringify({ + ok: true, + nested: { + token: "secret-token", + profile: { + password: "swordfish", + }, + }, + items: [ + { + refreshToken: "refresh-me", + }, + ], + }), + "application/json; charset=utf-8", + ), + ).toBe( + JSON.stringify({ + ok: true, + nested: { + token: "[REDACTED:12]", + profile: { + password: "[REDACTED:9]", + }, + }, + items: [ + { + refreshToken: "[REDACTED:10]", + }, + ], + }), + ); + }); + + it("applies shouldRecord route rules", () => { + expect(shouldRecord("/favicon.ico")).toBe(false); + expect(shouldRecord("/api/health")).toBe(false); + expect(shouldRecord("/observer")).toBe(false); + expect(shouldRecord("/observer/session/123")).toBe(false); + expect(shouldRecord("/observer-files")).toBe(true); + expect(shouldRecord("/cloud/dashboard")).toBe(true); + }); +}); diff --git a/router/tsconfig.json b/router/tsconfig.json new file mode 100644 index 00000000..00c428ac --- /dev/null +++ b/router/tsconfig.json @@ -0,0 +1,17 @@ +{ + "compilerOptions": { + "target": "ES2022", + "module": "ES2022", + "moduleResolution": "bundler", + "lib": ["ES2022", "WebWorker"], + "strict": true, + "noEmit": true, + "skipLibCheck": true, + "esModuleInterop": true, + "resolveJsonModule": true, + "isolatedModules": true, + "types": ["@cloudflare/workers-types", "vitest/globals"] + }, + "include": ["index.ts", "src/**/*.ts", "test/**/*.ts"], + "exclude": ["node_modules", "dist"] +} From b0a5a70773a38735fe4649901ed0f7c15dd85470 Mon Sep 17 00:00:00 2001 From: Khaliq Date: Thu, 14 May 2026 21:22:45 +0200 Subject: [PATCH 34/49] =?UTF-8?q?feat(migration):=20Phase=203=20bundle=20?= =?UTF-8?q?=E2=80=94=20Hyperdrive=20Option=20B=20+=20Worker=20dormant=20+?= =?UTF-8?q?=20foundation=20+=20rate=20limit=20+=20ACU=20bump=20+=20writeba?= =?UTF-8?q?ck=20backfill=20+=20docs=20(#647)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat(migration): Phase 0 foundation — acceptance suite + replay harness + handler tests (salvage) Salvages the Phase 0 foundation work that the `fargate-migration-phase0-foundation` workflow generated but failed to commit when its verify gates died on a bash quoting bug (unrelated to the agent output). The deliverables themselves are sound and gate Phase 3 cutover, so they're landing here in a clean PR from origin/main. Brings in: - `packages/acceptance/` — Layer 1 contract suite scaffold + 42 acceptance `.test.ts` files across auth / workspaces / integrations / runtime / api, plus helpers (auth, fixtures, env, runtime, server) and fixture seeds. Intended to run against deployed environments via `ACCEPTANCE_BASE_URL` (not exercised in this PR — see test plan). - `packages/replay-harness/` — Layer 3 replay oracle: CLI, corpus reader, S3-backed replay engine, equivalence comparator, report writer, plus 2 unit tests covering equivalence + replay-smoke. Used by Phase 3 to prove Worker responses match Lambda for a captured corpus. - `packages/web/test/handlers/` — Layer 2 integration tests: 45 handler-coverage `.test.ts` files (many are thin re-exports of existing top-level tests with `// @handler` route headers so coverage tooling can enforce one integration test per route). - `packages/web/test/helpers/{pglite-db,localstack,boot-app}.ts` — shared helpers for in-memory Postgres + LocalStack-backed tests. - `scripts/check-route-coverage.mjs` + `scripts/check-handler-coverage.ts` + `scripts/route-coverage-allowlist.json` — coverage gates that enforce every `/api/v1/...` route has a corresponding handler test (or an allowlist entry). Supporting changes: - Add `undici` to root devDependencies (used by acceptance helpers). - Add `allowImportingTsExtensions: true` to `packages/web/tsconfig.json` to match root tsconfig — the handler-coverage re-exports drag in top-level `tests/*.ts` files that use explicit `.ts` import extensions. - Update `tests/github-webhook-route.test.ts` `loadRoute` signature to `NextRequest`/`NextResponse` so the route module typechecks cleanly when pulled in via the new handler-coverage re-export. Validation in this PR (run in the new worktree): - `packages/replay-harness`: `tsc --noEmit` clean; vitest 2 files / 5 tests PASS. - `packages/acceptance`: `tsc --noEmit` clean. Acceptance suite NOT run — requires `ACCEPTANCE_BASE_URL` + a test workspace; landing as code only. - `packages/web/test/handlers`: `tsc --noEmit` clean (under web tsconfig); vitest 279/280 pass. One flaky test (`workflow-run-response-pushed-to.test.ts > merges disjoint pushedTo keys when two updates fire concurrently`) times out under parallel load but passes in isolation — to investigate as a follow-up. - Root `tsc --noEmit`: 0 new errors introduced by salvaged code (13 remaining errors pre-exist on origin/main and are unrelated). What's NOT in this PR / known follow-ups: - Acceptance suite has not been executed against a real environment; only typechecked. Wiring `ACCEPTANCE_BASE_URL` into CI is a follow-up. - The flaky concurrent-update handler test needs a higher timeout or a serial fixture. - Pre-existing root typecheck errors (`packages/web/lib/integrations/ nango-sync-queue.ts`, `relayauth/packages/server/src/__tests__/ test-helpers.ts`) are not addressed here — they exist on main. Spec: `docs/fargate-migration-spec.md`. Co-Authored-By: Claude Opus 4.7 (1M context) * docs(migration): Hyperdrive network-path spec (Phase 3 prereq) Phase 3 of the cloud-web migration moves web onto Cloudflare Workers, which can only reach Postgres via Hyperdrive. Hyperdrive runs at CF's edge and connects out to a Postgres endpoint over the public internet; our Aurora cluster lives on private VPC subnets and is not reachable today. This spec lays out three options for bridging that gap (public Aurora endpoint with allowlist; PrivateLink + Cloudflare Tunnel; public read replica with private writer), a decision matrix, recommended starting point (Option A, subject to security signoff), risks, and rollout plan. Status: Proposed. Decision belongs to a human reviewer with security sign-off. Co-Authored-By: Claude Opus 4.7 * chore: add migration-hyperdrive-option-b workflow (pre-run snapshot) Self-contained workflow for Hyperdrive Option B infrastructure: CF Tunnel + Access app + service token + cloudflared Fargate daemon + real Hyperdrive resource replacing the null placeholder. All declarative via @pulumi/cloudflare. Patterns: inlined setup-branch + install-deps, single-line bash, plain string concat (no template-literal traps), .onError('fail-fast'). * chore: add migration-phase3-web-worker-build workflow (pre-run snapshot) Self-contained workflow for the Phase 3 cloud-web Worker build: @opennextjs/cloudflare build of packages/web, wrangler.toml bindings, sst.cloudflare.Worker resource, KV-flagged router rule. Ships dormant (KV default: lambda). * fix(workflow): relax preflight; install @opennextjs/cloudflare in-workflow Original preflight required @opennextjs/cloudflare to be preinstalled as a precondition, but that dep should be installed BY the workflow, not before it. Removed the preinstall check + the strict ALLOWED_DIRTY gate; added an explicit install-opennext-cf step. The standalone-output check moves into the fix-build agent loop where the agent can patch next.config.ts if needed. * fix(workflow): remove false-positive verify rules; rely on claude reviewer The 'no process.env' shell regex was a false positive — process.env.CLOUDFLARE_DEFAULT_ACCOUNT_ID is the canonical pattern in this repo (infra/sage.ts, infra/traffic-recorder.ts, infra/cloudflare-worker-migrations.ts all use it). The 'publicly_accessible' regex also fired spuriously due to set -e + || subshell interaction. The semantic checks (no secrets via process.env, Aurora stays private) are already enumerated in the final-review step's task prompt for the claude reviewer agent — keep them there where judgement can apply, drop the brittle regex. * fix(acceptance): retry 503 + correct stale auth-error contract Resolves five categories of failures from running the suite against current prod: - helpers/server.ts now retries 503 as well as 429 with exponential backoff. Prod is in throttle recovery (Lambda URL OAC was just hardened in #631) and the suite shouldn't fail PRs while the edge works through the backlog. - integrations/{github,linear,notion}-query.test.ts and integrations/proxy-slack.test.ts now assert 401 (the unified auth envelope) instead of the historical 403. The bearer-auth helper was unified months ago; the tests were encoding pre-unification behavior. - workspaces/workspaces-join.test.ts now correctly asserts that anonymous workspaces ignore a forged bearer (the existing route contract — anon join is open) instead of asserting an impossible 401. - runtime/{agents,deploy}.test.ts no longer assert exact substring "Internal Server Error" in non-JSON error bodies — CloudFront / Lambda edge pages vary; we now just assert a non-empty body. Co-Authored-By: Claude Opus 4.7 (1M context) * feat(coverage): real handler-coverage gate + issue-tracked allowlist The Phase 0 coverage gates were paper: check-handler-coverage.ts only grep'd for `// @handler` comment markers, and check-route-coverage.mjs already had a `reason` field but no `issue` requirement so deferrals weren't actionable. Three 2-line handler-test files were also re-exporting node:test suites that the vitest runner can't see. This commit: - Rewrites scripts/check-handler-coverage.ts to validate every claimed `@handler` route by walking the file's import graph (static, dynamic, and the bare-string-literal-to-route.ts pattern node:test handler shims use) and confirming (a) at least one reachable file imports vitest, and (b) at least one reachable file lives under the route's directory in packages/web/app/api/. Files that satisfy only the `@handler` marker but never load any of the route's source fail. - Adds destructured-export detection (`export const { POST } = ...`) to the stateful-route walker so factory-pattern routes like /api/v1/agents/deploy are no longer silently dropped. - Adds `issue` as a required allowlist field in both gates. Deferred coverage now requires a tracking reference, not just a freeform reason. (route gate previously only required `reason`.) - Deletes 3 handler test files that re-exported node:test suites: github-clone-route, proactive-deploy-routes, workspace-secrets-routes. Their underlying tests still run via npm run github:validate / orchestrator:test; the deletion just removes the false 0-test-claim. - Adds allowlist entries for the 4 corresponding routes referencing tracking issue #638 (vitest-port follow-up). - Adds allowlist entries for 10 routes added post-Phase-0 that have no acceptance coverage yet (digest-functions, gitlab, relayfile batch / audit) referencing tracking issue #639. - Makes check-route-coverage.mjs fall back to walking packages/web/app/api/ when the generated .next/types/routes.d.ts is not present (local dev / pre-build CI invocation). - Bumps the timeout on tests/workflow-run-response-pushed-to.test.ts's concurrent-merge test from the default 5s to 30s — the PR author had already flagged this as flaky under parallel load; the fix is trivial. After this commit: $ node scripts/check-route-coverage.mjs Route coverage OK (181 route-method pairs checked). $ npx tsx scripts/check-handler-coverage.ts Handler coverage OK (41 stateful routes, 47 real-covered, 11 allowlisted). $ vitest run --dir packages/web/test/handlers 280 passed (280), 0 failed. Co-Authored-By: Claude Opus 4.7 (1M context) * ci+docs: wire phase0-tests job and document the suite CI: - Adds a `phase0-tests` job to .github/workflows/ci.yml that runs on every PR: 1. node scripts/check-route-coverage.mjs — acceptance gate 2. npx tsx scripts/check-handler-coverage.ts — handler gate 3. vitest run --dir packages/web/test/handlers — Layer 2 (offline) 4. vitest run (replay-harness) — Layer 3 (offline) 5. vitest run (acceptance) against prod — Layer 1 Authenticated acceptance tests are it.skip'd when no credentials are provided, so the suite is safe to run on any PR without secrets. Docs: - Rewrites packages/acceptance/README.md from a layout stub into a living playbook: how-to-run, how-to-add-a-test-for-a-new-route, what-gets-enforced, when-tests-fail-in-CI, what-is-NOT-tested, and the two follow-up issues (#638, #639). - Replaces packages/acceptance/AGENTS.md, which was full of stale workflow guidance, with a real agent guide: contract conventions, the // @route marker contract, allowlist rules, and a do-not list. - Adds packages/web/test/handlers/README.md (didn't exist) with the two test-authoring patterns (in-file vitest vs. re-export), the coverage-gate contract, common failure modes, and open follow-ups. Co-Authored-By: Claude Opus 4.7 (1M context) * feat(migration): Hyperdrive Option B — Tunnel + Access + private Aurora Phase 3 prerequisite. Wires up the full Hyperdrive-via-Cloudflare-Tunnel infrastructure so Workers can reach Aurora WITHOUT exposing Aurora to the public internet. All declarative — no operator dashboard clicks (CF API token already has Tunnel/Access/Hyperdrive scopes per operator setup). Deliverables: - infra/cloudflared-tunnel.ts Tunnel + Access app + service token - infra/aurora-tunnel-daemon.ts cloudflared Fargate task in VPC - infra/hyperdrive.ts real HyperdriveConfig (replaces null) - infra/webhook-worker.ts restore hyperdrive link Spec: docs/migration/hyperdrive-network-path.md (PR #634) Co-Authored-By: Claude Opus 4.7 (1M context) * docs(architecture): defense-in-depth doc for preventing single-client outages Direct response to 2026-05-14 incident (issue #640): a single internal client's fsnotify-driven retry loop saturated production for ~6.5h because five independent rate-limit / backoff defense layers were simultaneously missing. This doc maps the seven defense layers, what each does, what's in place, what's missing, and where each gap is tracked. Living document — updates as defense layers land. Co-Authored-By: Claude Opus 4.7 (1M context) * feat(edge): per-key rate limit at Cloudflare Worker gateways Closes the server-side rate-limit gap exposed by the 2026-05-14 relayfile-mount runaway loop. A single workspace hitting /v1/workspaces//sync/status at ~30 req/sec exhausted the upstream Lambda's 10-slot concurrency cap and 429'd unrelated users; neither the cloud router nor the relayfile-api worker had any per-key throttle in place. Adds a shared KV-backed rate limiter to both edge Workers: 60 req/min per workspace (or user, or IP), 1000 req/min global, tunable at runtime through the existing ROUTER_CONFIG KV namespace. A RATE_LIMIT_DISABLED kill switch is available for emergency rollback without a redeploy. The limiter runs before any other routing in both Workers, so a runaway key gets a 429 at the edge and never reaches CloudFront or Lambda. Counters are bucketed by minute in a dedicated RATE_LIMIT_COUNTERS KV namespace bound to both gateways, so a client that hits both endpoints does not get to consume two separate budgets. Known limitation: KV is eventually consistent, so the effective ceiling at high traffic is ~2-3x configured (180 req/min worst- case for the 60 default). That's accepted for now — still ~100x lower than the incident-day pattern. Upgrade path to Durable Object atomicity is documented in docs/security/rate-limiting.md. Co-Authored-By: Claude Opus 4.7 (1M context) * fix(acceptance): validate redactions per secret match * fix(acceptance): address review nits for fixtures and helpers * feat(migration): Phase 3 — cloud-web Worker build (KV-flagged, dormant) OpenNext-CF build of packages/web, wrangler.toml bindings, SST Worker resource, KV-flagged router rule. Ships dormant — router defaults to lambda; cutover is Phase 4. Deliverables: - packages/web/.open-next/worker.js (build artifact) - packages/web/wrangler.toml (staging deploy + wrangler dev config) - infra/web-worker.ts (sst.cloudflare.Worker for staging) - packages/router/index.ts (cloudOrigin KV flag, defaults to lambda) Spec: docs/fargate-migration-spec.md (Phase 3) Co-Authored-By: Claude Opus 4.7 (1M context) * fix(phase3): address PR #641 review — wire daemon, harden secret bindings, scope egress Devin findings (both 🔴): - BUG-0001: aurora-tunnel-daemon.ts imported `{ cluster, vpc }` but there was no ECS cluster declared anywhere. Added `sst.aws.Cluster("CloudCluster")` export to infra/vpc.ts so the tunnel daemon Fargate service has a cluster to land in. - BUG-0002: aurora-tunnel-daemon.ts was never imported from sst.config.ts, so the daemon, SG ingress rule, and CW alarm would silently not deploy and the Hyperdrive → Aurora path would have no tunnel connector. sst.config.ts now imports both cloudflared-tunnel and aurora-tunnel-daemon explicitly. CodeRabbit findings: - Aurora SG resolution: removed the fragile `vpc.securityGroups[0]` fallback that would have attached the 5432 ingress rule to whatever SG happened to be first (NOT Aurora's). Now throws explicitly if appDatabase.nodes.cluster.vpcSecurityGroupIds is empty. - Scoped cloudflared daemon SG egress from "all ports, all destinations" to 443/tcp only — cloudflared only needs outbound HTTPS to CF's edge. Other self-review fixes: - infra/hyperdrive.ts: account ID now reads CLOUDFLARE_DEFAULT_ACCOUNT_ID directly (canonical pattern used by sage.ts, traffic-recorder.ts, cloudflare-worker-migrations.ts) instead of the over-clever `pulumi.all([tunnel.accountId, serviceToken.accountId])` chain that bought nothing but inconsistency. - infra/webhook-worker.ts: GITHUB_WEBHOOK_SECRET and COMPOSIO_WEBHOOK_SECRET bindings are now conditional via spread — omit the binding entirely when the env var is unset, rather than binding empty strings. An HMAC compared against "" would pass any signature; the prior behavior was a security hole the moment the KV flag flipped. Cutover preflight must verify both are populated. - infra/cloudflared-tunnel.ts: added comment block before accessApplication documenting the `.cfargotunnel.com` domain assumption and the CNAME-from-owned-zone fallback if CF rejects it at deploy time. Workflow side-effects also committed: - infra/relayfile.ts, infra/relaycron.ts, infra/sage.ts, infra/cataloging-agent-{github,linear}.ts, infra/credential-proxy.ts, infra/relayauth.ts: hoisted scriptName constants to module top instead of reading `.nodes.worker.scriptName` after construction. The latter pattern can trip a Pulumi cycle for queue/cron consumers that reference the producing worker's scriptName as part of their own binding. - infra/sst-globals.d.ts + tsconfig change: typecheck shim so `infra/` builds before `sst install` materializes .sst/platform/config.d.ts. SST's own deploy regenerates config.d.ts at run time so this only affects local typecheck. Co-Authored-By: Claude Opus 4.7 * feat(edge): make rate-limit bypass list runtime-tunable via KV Adds RATE_LIMIT_BYPASS_PREFIXES_EXTRA to ROUTER_CONFIG so operators can whitelist additional paths from rate limiting without a deploy. Static BYPASS_PREFIXES stays as the fail-safe (KV unreachable still exempts health/observer/static), and the dynamic check fails open on KV errors so a transient KV blip cannot start rate-limiting healthcheck traffic. Per-isolate cache with 60s TTL keeps the hot path cheap; propagation across edges is <60s plus KV's consistency window. Now ALL rate-limit config (per-key limit, global limit, kill switch, bypass list) is runtime-tunable via the same KV namespace. Tests: 13 new — parseBypassExtras defensive behavior (null/empty/ whitespace/non-path/dedup), isBypassPathDynamic static+KV path, cache TTL boundary, KV error fail-open, end-to-end via maybeRateLimit showing a KV-whitelisted path bypassing a tight per-key limit. Co-Authored-By: Claude Opus 4.7 (1M context) * fix(phase3): nodejs_compat on deployed Worker; consolidate wrangler config Critical: - infra/web-worker.ts: add `transform: { worker(args) { args.compatibilityFlags = ["nodejs_compat"] } }`. OpenNext's Next.js → Worker output requires the flag at runtime (it imports node:buffer / node:process / etc). Without it, the deployed Worker boots fine but crashes on the first request touching a Node API. Setting it via SST's transform is the canonical pattern in this repo (see infra/webhook-worker.ts:63). The `compatibility_flags` line in packages/web/wrangler.toml only applies to local `wrangler dev`, NOT to the SST-deployed Worker — that file is consulted by Wrangler, not by SST. Cleanup: - Deleted packages/web/wrangler.jsonc. The PR shipped both a `.jsonc` and a `.toml` with conflicting `name` ("web" vs "cloud-web-staging") and conflicting `compatibility_date` (2026-04-15 vs 2026-05-14). Keep `wrangler.toml` as the single source of truth — it has the full set of bindings (Hyperdrive, KV, R2, Queues, vars) that the .jsonc was missing. - packages/web/wrangler.toml: expanded the header comment to (a) call out that the `` placeholders are documentation, not valid IDs, and `wrangler dev` will refuse to start until they're replaced, and (b) make explicit that this file does NOT govern the SST-deployed Worker — the `transform` block in `infra/web-worker.ts` does. Hyperdrive ternary in `link:` is intentionally left as-is — `hyperdrive` is still the `null` placeholder from #632 on this branch until #641 merges and this branch rebases. Removing the ternary now would break standalone builds of this branch. Co-Authored-By: Claude Opus 4.7 * chore(infra): bump prod Aurora max ACU 4→8 ahead of Workers Phase 3 Production cluster (cloud-production-appdatabasecluster) is currently configured to scale 1-4 ACU and peaked at the ceiling (4 ACU) multiple times in the 24h before this change. The Cloudflare Workers Phase 3 cutover will land more concurrent isolates against the same cluster than the current Lambda path does, so the compute/IO ceiling needs to move before that traffic shape arrives. Doubling (4→8) is the minimum viable hedge — it doesn't change the min, doesn't change preview/dev (which stay at 0-4 with auto-pause), and Aurora Serverless v2 pricing is on-demand so the new ceiling is free until it's actually reached. Connection count is not the bottleneck (peak observed: 18 connections; 8 ACU = ~450 max_connections), and RDS Proxy continues to front the cluster in prod with its existing pool config. Followups (tracked separately, not in this PR): - Hyperdrive resource + binding lands with the Phase 3 web Worker PR - CloudWatch alarms on DatabaseConnections (>150) and ACU (>=7) Co-Authored-By: Claude Opus 4.7 (1M context) * fix(acceptance): address remaining helper review defects * chore(bundle): cherry-pick #623 — RelayFile writeback CF backfill Brings in the 4 net-new files + the push.ts signature change without the stale-base PR's 15k lines of accidental main reverts. See #623 for the original review history. Co-Authored-By: Claude Opus 4.7 * fix(phase3): address Devin findings on #647 — daemon egress to Aurora + acceptance retry body cancel Devin BUG-0001 🔴 — tunnel daemon SG blocks egress to Aurora:5432. The previous CodeRabbit-driven scoping of the daemon SG to 443/tcp egress only covered the Cloudflare control-plane tunnel; it never permitted the *data-plane* hop from cloudflared to Aurora. AWS SGs are stateful on the inbound side, but the initial outbound TCP SYN still needs an explicit egress rule. Tunnel would have established to CF cleanly and every Hyperdrive query would have timed out at the daemon-→-Aurora hop. Added `auroraTunnelDaemonEgressToAppDatabaseRule` — 5432/tcp egress scoped to Aurora's SG specifically (NOT 0.0.0.0/0). Updated the SG def comment to reference both hops. Devin BUG-0002 🟡 — acceptance retry helper canceled the response body before returning on the last attempt, so callers got a response with a disturbed stream and `.json()` / `.text()` threw "stream already canceled". Swapped the order: check `attempt === MAX_ATTEMPTS - 1` *first* and return the response untouched, only cancel the body when we're going to retry. Co-Authored-By: Claude Opus 4.7 * fix(phase3): address all 5 Codex findings on bundle PR #647 P1.1 🔴 — `infra/web-worker.ts` was declared but never imported in `sst.config.ts`, so SST never materialized the CloudWebWorker resource. Same root cause as Devin BUG-0002 on #641's aurora-tunnel-daemon. Added `await import("./infra/web-worker")` to the import chain. P1.1b 🔴 — Router's `env.CLOUD_WEB_WORKER` service binding was never declared on the AgentRelayRouter in `infra/web.ts`. `cloudOrigin=workers` would have been a silent no-op falling back to Lambda. Added the service binding via `appendWorkerBindings`. Also hoisted `cloudWebWorkerScriptName` const in web-worker.ts (matches the pattern sage / relayauth / relayfile-api use) so the binding can reference a known name at construction time instead of reading it post-creation. P1.2 🔴 — Hyperdrive private-DB flow now matches CF docs: - Real public hostname: `aurora-db..agentrelay.cloud` (internal zone, matches the catalog-* pattern already in use). - `cloudflare.DnsRecord` CNAMEs it to `.cfargotunnel.com` with `proxied: true`. - Tunnel ingress now declares the hostname explicitly (with a `http_status:404` catch-all per CF requirement). - `ZeroTrustAccessApplication.domain` uses the real hostname, `dependsOn: [auroraDbDnsRecord]` so DNS is in place first. - `HyperdriveConfig.origin.host` uses the same hostname, `dependsOn: [accessApplication]` so the Access app exists when Hyperdrive validates the service-token credentials at create time. The earlier `.cfargotunnel.com` pattern is for IP-routed private networks, not the hostname-routed Access flow Hyperdrive needs. P1.3 🔴 — Added `npx @opennextjs/cloudflare@1.19.10 build` step to `_deploy-cloud-stage.yml` so `packages/web/.open-next/worker.js` actually exists at `sst deploy` time. The existing `@opennextjs/aws` build produces a different `.open-next/` layout that SST's CloudWebWorker handler path cannot find. P1.4 🟠 — Added `/api/v1/webhooks` to `BYPASS_PREFIXES` in `packages/router/src/rate-limit.ts`. Webhook ingress has no auth token and no workspace path, so it would bucket by IP and trip the 60/min cap with even modest provider burst traffic. Webhook worker already has queue-backed backpressure. P1.5 🟠 — Gated the POST `/api/waitlist` happy-path test in `packages/acceptance/src/api/public-routes.test.ts` behind `ACCEPTANCE_WRITES_OK=1`. Default OFF on PR CI so the test stops writing real rows to the prod waitlist on every PR run. The 400-case and GET tests still run unconditionally. All findings credited to Codex review on PR #647. Local gates pass: infra typecheck clean, 43 router rate-limit tests, 276 handler tests. Co-Authored-By: Claude Opus 4.7 * fix(rate-limit): key cookie-auth /cloud* traffic by session cookie, not IP Cookie-authenticated requests to /cloud* had no workspace ID and no Authorization header, so extractLimitKey() was falling through to the IP bucket — collapsing all users behind a shared NAT into one rate limit bucket. Add session-cookie parsing (cookie name: agent_relay_session) as precedence level 3, between auth-header and IP. The cookie value is hashed with FNV-1a 32 and keyed as `session:` so the raw token never appears in the counter store. Precedence order is now: 1. ws: — workspace path segment 2. user: — Authorization header 3. session: — agent_relay_session cookie (new) 4. ip: — cf-connecting-ip / x-forwarded-for 5. anon — no signal Co-Authored-By: Claude Sonnet 4.6 * fix(phase3): Track D — make replay/backfill tooling safe-by-default - backfill script now defaults to dry-run; requires explicit --apply to mutate (old --dry-run flag kept as an alias so existing invocations don't break); prints clear warnings in dry-run mode - runbook fixes the non-existent npm script reference to the correct `npx tsx scripts/...` invocation via sst shell, and documents the --apply requirement with a quick-reference options table - replay harness strips the /cloud mount prefix by default before building target URLs (configurable via stripMountPrefix option) - replay harness skips POST/PUT/PATCH/DELETE unless --allow-mutations is passed; MutationBlockedError exported for programmatic use - replay harness adds per-request timeout (default 30 s) and overall replay timeout (default 5 min) with AbortController + deadline guard - 13/13 replay-harness tests pass (5 new tests cover the new behaviours) Co-Authored-By: Claude Sonnet 4.6 * feat(phase3): Track B — DB client factory, CF env propagation, S3 storage on Workers B1: Add `packages/core/src/db/factory.ts` with `selectDbClient(env?)` that routes to postgres-js/Hyperdrive when `env.HYPERDRIVE.connectionString` is set, else falls back to the existing Lambda pg-pool path. Includes connection caching and `setDbForTesting` override support. Add `factory.test.ts` (3 vitest cases). Add `postgres` to `@cloud/core` deps. B2: Update `packages/web/lib/db/index.ts` to make `getDb()` context-aware: reads the Cloudflare context via `Symbol.for("__cloudflare-context__")`, calls `selectDbClient({ HYPERDRIVE })` on Workers, and falls through to Lambda client otherwise. No call-site changes (51 handlers unmodified). B3: Add `packages/web/lib/storage.ts` with `createWorkflowStorageS3Client()` and `readWorkflowStorageBucket()` that read credentials from Worker env bindings first, then SST Resource, then process.env. Wire all S3 usage sites (artifact-store, evidence-builder, bundle-store, digest-functions/store, github-push-back, 3 route handlers) to use the centralized storage helpers. Delete `client-hyperdrive.ts` (now redundant). Add `lib/cloudflare-context.ts` internal helper and `scripts/patch-opennext-native-shims.mjs` for Worker build compat. Verification: factory.test.ts 3/3 pass; handler tests 276/276 pass; infra tsc clean. Co-Authored-By: Claude Sonnet 4.6 * fix(phase3): Track A — alias ssh2/cpu-features to empty shim in CF Worker bundle The Worker never invokes SSH code paths, but @agent-relay/sdk → @agent-relay/cloud → ssh2 → cpu-features pulls in a native .node binary that esbuild cannot bundle for a Cloudflare Worker. Three-layer fix: 1. next.config.ts outputFileTracingExcludes: prevents @vercel/nft from copying ssh2 and cpu-features into the standalone output. 2. next.config.ts turbopack.resolveAlias + webpack: aliases ssh2 and cpu-features to OpenNext's empty shim during next build so the file tracer never sees the packages. 3. package.json postinstall runs scripts/patch-opennext-native-shims.mjs: patches bundle-server.js to add the same aliases in the OpenNext esbuild step (which does not read next.config.ts aliases). Verified: npx @opennextjs/cloudflare@1.19.10 build exits 0 and produces worker.js. Co-Authored-By: Claude Sonnet 4.6 * fix(phase3): hard-error when --apply and --dry-run are passed together Silently last-flag-wins was confusing and potentially dangerous for prod ops. Co-Authored-By: Claude Sonnet 4.6 * fix(phase3): address Codex round-3 findings on bundle PR #647 P1.1 🔴 + P1.2 🔴 — CloudWebWorker missing bindings + environment. The Worker resource only set scriptName + nodejs_compat. `link:` materializes SST resources but does NOT generate the env.* bindings on the CF Worker. So `env.HYPERDRIVE` was undefined, `getDb()` fell through to the Lambda pg+Resource.AppDatabase path (which the Worker can't run), and process.env reads (URLs, cookie domain, S3 bucket, webhook config) all returned blank. Added explicit `appendWorkerBindings` for HYPERDRIVE, ROUTER_CONFIG, RATE_LIMIT_COUNTERS. Mirrored infra/web.ts:148 environment block so the Worker sees the same NEXT_PUBLIC_APP_URL / RELAYAUTH_URL / RELAYFILE_URL / RELAY_LLM_PROXY / RELAYCRON_URL / CREDENTIAL_PROXY_URL / cookie domain / SAGE_WEBHOOK_URL / S3_BUCKET / WEBHOOK_CONSUMERS_JSON / WEBHOOK_MSD_BACKEND_* that Lambda gets. OpenNext-CF's init.js copies these into process.env on first request, so the existing handler code reads them unchanged. P1.4 🔴 — Acceptance CI still mutated prod beyond the waitlist gate. `workspaces-crud.test.ts:82` POSTed `/api/v1/workspaces/create` and `workspaces-join.test.ts:31` did the same in beforeAll, both unauthenticated. The destroyWorkspace helper at `_helpers.ts:78` explicitly skips unauth workspaces, so every PR CI run was leaking a workspaces row in prod. Both now gated on `ACCEPTANCE_WRITES_OK=1` — same env var pattern as the waitlist gate. crud uses `it.skipIf(...)`, join uses `describe.skipIf(...)` since the whole suite hangs off the beforeAll write. P1.5 🔴 — `npx tsx scripts/backfill-relayfile-writeback-cf.ts --help` was broken at the entrypoint. The static named-import form failed at Node's ESM parse time because packages/web has no `"type": "module"`, so tsx loaded the .ts via CJS shape and exposed exports under `default`. Node ESM rejected the named binding before main() ever ran. Switched to dynamic `await import(...)` inside main() with a guard that accepts both module shapes. Runbook command from `docs/operations/relayfile-writeback-cf-backfill.md` now works end-to-end. P2.6 🟠 — Recorder bypassed the cloud-web Worker path. `packages/router/index.ts:256` returned `env.CLOUD_WEB_WORKER.fetch()` early, BEFORE the `recordingRequest = request.clone()` at line 288. So once cloudOrigin flipped to "workers", the replay harness would have no /cloud Worker corpus — exactly the wrong moment to lose recording, since that's when we need replay evidence to prove cutover equivalence. Moved the recorder clone to the top of fetch(), kept the existing Lambda path's `recordingRequest` symbol pointing at the same clone so the Lambda-origin code path is unchanged. Each early-return branch now ctx.waitUntil(maybeRecord(...)) the cloned request + response. Both the cloud-web Worker AND webhook Worker paths now capture. P1.3 (STS-on-Worker) acknowledged out of scope — 6 routes still call Resource.WorkflowStorage.stsRoleArn and can't trivially run on a Worker without a separate credential broker. Documented as known gap in the previous bundle review comment. Co-Authored-By: Claude Opus 4.7 * fix(phase3): partial Codex round-4 — WORKER_SELF_REFERENCE + acceptance auth-session gate P1.1 partial 🟠 — Added a `service` binding for WORKER_SELF_REFERENCE pointing at the Worker's own script name. OpenNext-CF emits revalidation / on-demand-ISR paths that use this binding to call back into the Worker without going through the CF edge. Matches the binding documented in packages/web/wrangler.toml. The sibling ASSETS binding is intentionally NOT added in this PR — it requires the `cloudflare.WorkersAssetsUpload` resource flow (hash each file in .open-next/assets/, upload via CF API, obtain completion JWT, pass to WorkersScript.assets). SST doesn't expose this directly; it's a substantial workstream tracked as a follow-up to scope the cutover. Without ASSETS, the Worker still boots and serves regular requests — just can't serve `/cloud/_next/static/*` images through the Worker path. The bundle ships dormant so this doesn't affect current traffic. P2 🟢 — Auth-session test now requires explicit ACCEPTANCE_ALT_WORKSPACE_ID when running session-authed. Previously fell back to provisionTestWorkspace() which silently leaked a workspaces row because teardownTestWorkspace() is a no-op (no delete endpoint for acceptance-created workspaces yet). PR CI never hit this today (no session auth wired), but it was a latent leak for any future authenticated run. This PR is now considered cutover-DORMANT-ready, not cutover-LIVE-ready. Remaining cutover gaps (P1.2 S3 creds, P1.3 STS, P1.4 /api/v1/deploy, plus ASSETS) require structural decisions and ship as follow-up PRs. Co-Authored-By: Claude Opus 4.7 --------- Co-authored-by: Claude Opus 4.7 (1M context) Co-authored-by: Phase 3 Web Worker Build Bot --- router/index.ts | 78 ++++- router/src/rate-limit.ts | 550 ++++++++++++++++++++++++++++++ router/test/rate-limit.test.ts | 588 +++++++++++++++++++++++++++++++++ 3 files changed, 1212 insertions(+), 4 deletions(-) create mode 100644 router/src/rate-limit.ts create mode 100644 router/test/rate-limit.test.ts diff --git a/router/index.ts b/router/index.ts index 46b01ddf..4c3bd718 100644 --- a/router/index.ts +++ b/router/index.ts @@ -1,10 +1,15 @@ import { maybeRecord, type RecorderEnv } from "./src/recorder.js"; +import { maybeRateLimit, type RateLimitEnv } from "./src/rate-limit.js"; interface Env { CLOUD_APP_ORIGIN: string; + CLOUD_WEB_WORKER?: { + fetch(request: Request): Promise; + }; FILE_OBSERVER_ORIGIN?: string; TRAFFIC_RECORDER?: RecorderEnv["TRAFFIC_RECORDER"]; ROUTER_CONFIG?: RecorderEnv["ROUTER_CONFIG"]; + RATE_LIMIT_COUNTERS?: RateLimitEnv["RATE_LIMIT_COUNTERS"]; WEBHOOK_WORKER?: { fetch(request: Request): Promise; }; @@ -22,6 +27,7 @@ const PRIMARY_HOST = "agentrelay.com"; const FILE_OBSERVER_PATH_PREFIX = "/observer/file"; const OBSERVER_PATH_PREFIX = "/observer"; const CLOUD_PATH_PREFIX = "/cloud"; +const CLOUD_ORIGIN_FLAG_KEY = "cloudOrigin"; const WEBHOOK_ORIGIN_FLAG_KEY = "WEBHOOK_ORIGIN"; // Exact paths the webhook worker handles. Other sub-paths under @@ -168,7 +174,9 @@ export function rewriteLocation( } export function getOrigin(hostname: string, pathname: string, env: Env): string { - // /cloud* always goes to the Next.js cloud app regardless of host + // /cloud* defaults to the Next.js cloud app regardless of host. Requests only + // reach this branch when the KV flag has not already forwarded them to the + // cloud-web Worker. if (isCloudPath(pathname)) { return env.CLOUD_APP_ORIGIN; } @@ -190,6 +198,19 @@ export function getOrigin(hostname: string, pathname: string, env: Env): string return FALLBACK_PROXY_ORIGIN; } +async function shouldUseCloudWebWorker(pathname: string, env: Env): Promise { + if (!isCloudPath(pathname)) { + return false; + } + + try { + const configured = await env.ROUTER_CONFIG?.get(CLOUD_ORIGIN_FLAG_KEY); + return configured?.trim().toLowerCase() === "workers"; + } catch { + return false; + } +} + async function shouldUseWebhookWorker(pathname: string, env: Env): Promise { if (!isWebhookWorkerPath(pathname)) { return false; @@ -221,14 +242,59 @@ function buildWorkerOriginRequest(request: Request, requestUrl: URL, workerOrigi export default { async fetch(request: Request, env: Env, ctx: ExecutionContext): Promise { const url = new URL(request.url); + + // Per-key rate limiting runs BEFORE any worker routing so a runaway + // workspace gets bounded everywhere — including webhook ingress and + // /cloud* traffic. The bypass list inside maybeRateLimit exempts + // health and observer paths. See packages/router/src/rate-limit.ts + // and docs/security/rate-limiting.md. + const rateLimited = await maybeRateLimit(request, env); + if (rateLimited) { + return rateLimited; + } + + // Clone the request up front so any branch that returns early (cloud-web + // Worker service binding, webhook Worker service binding, etc.) can + // still feed the recorder the original payload. Without this clone, the + // /cloud Worker path bypasses the recorder entirely and the replay + // harness has no corpus to prove equivalence during Phase 4 cutover. + // See Codex P2.6 on bundle PR #647. + const recorderRequestClone = hasRecorderEnv(env) + ? (request.clone() as unknown as Request) + : null; + + if ((await shouldUseCloudWebWorker(url.pathname, env)) && env.CLOUD_WEB_WORKER) { + const workerResponse = await env.CLOUD_WEB_WORKER.fetch(request); + if (recorderRequestClone) { + ctx.waitUntil( + maybeRecord(recorderRequestClone, workerResponse.clone(), env, ctx), + ); + } + return workerResponse; + } + if (await shouldUseWebhookWorker(url.pathname, env)) { if (env.WEBHOOK_WORKER) { - return env.WEBHOOK_WORKER.fetch(request); + const workerResponse = await env.WEBHOOK_WORKER.fetch(request); + if (recorderRequestClone) { + ctx.waitUntil( + maybeRecord(recorderRequestClone, workerResponse.clone(), env, ctx), + ); + } + return workerResponse; } const workerOrigin = env.WEBHOOK_WORKER_ORIGIN?.trim(); if (workerOrigin) { - return globalThis.fetch(buildWorkerOriginRequest(request, url, workerOrigin)); + const originResponse = await globalThis.fetch( + buildWorkerOriginRequest(request, url, workerOrigin), + ); + if (recorderRequestClone) { + ctx.waitUntil( + maybeRecord(recorderRequestClone, originResponse.clone(), env, ctx), + ); + } + return originResponse; } } @@ -249,7 +315,11 @@ export default { headers.set("X-Forwarded-Prefix", mountPrefix); } - const recordingRequest = hasRecorderEnv(env) ? (request.clone() as unknown as Request) : null; + // Reuse the clone made at the top of fetch() for the recorder. The + // earlier clone covers all branches; making a second one here would + // be wasted bytes on every request and would double-record on the + // Lambda origin path. + const recordingRequest = recorderRequestClone; const subRequest = new Request(url.toString(), { method: request.method, headers, diff --git a/router/src/rate-limit.ts b/router/src/rate-limit.ts new file mode 100644 index 00000000..d7c6c734 --- /dev/null +++ b/router/src/rate-limit.ts @@ -0,0 +1,550 @@ +// Per-key request rate limiting for the Cloudflare edge gateways. +// +// Two Workers front the platform: the `AgentRelayRouter` worker +// (agentrelay.com / cloud router) and the `RelayfileApi` worker +// (api.relayfile.dev). A misbehaving client today (e.g. a relayfile +// mount stuck in a polling loop) can hammer either gateway with +// dozens of requests per second; the upstream Lambda has a 10-slot +// concurrency cap, so once a single key saturates the gateway every +// other user starts seeing 429s. The rate limiter in this module +// runs at the edge BEFORE the request reaches the Lambda, so a +// runaway key is contained to its own per-workspace bucket and +// doesn't deplete shared capacity. +// +// Design choices (intentional trade-offs): +// +// * Counter storage is Cloudflare KV (a `RATE_LIMIT_COUNTERS` +// namespace bound to both Workers). KV is eventually consistent +// across edges, so the effective ceiling at high traffic can be +// ~2-3x the configured per-minute limit. For the 60 req/min +// target that means ~180 req/min worst case — still 100x lower +// than the runaway today and good enough for the runaway-client +// class of bug. Sub-second precision and global atomicity would +// require a Durable Object, which we may add later if KV drift +// bites — but the simpler path ships first. +// +// * Buckets are minute-aligned (epoch-minute integer). Each +// `(limitKey, bucket)` pair is one KV key with a 120-second TTL +// so the previous and current minute coexist briefly during the +// turn. Reads are racy w.r.t. writes; that's accepted (see above). +// +// * Limits are read from `ROUTER_CONFIG` KV at request time, so +// operators can dial them up/down or kill-switch the limiter +// entirely via `wrangler kv:key put` without redeploying. +// +// * Defaults: 60 req/min per workspace-or-user-or-ip key, 1000 +// req/min global ceiling (a separate safety-net counter shared +// across all keys). Both defaults are intentionally conservative +// given the actual runaway pattern observed (30+ req/sec from a +// single key). +// +// * Bypass list mirrors the recorder's deny list plus a few +// observability paths — these must never be rate limited because +// they're either health probes, browser asset requests, or +// observer/webhook ingress that already has its own backpressure. + +const ONE_MINUTE_SECONDS = 60; +// KV TTL on counter entries. Must be >= 60 to cover a full bucket +// window even if the request arrives at the very start of the +// minute. We pick 120s so the previous bucket is still readable +// briefly during the turnover, which means a borderline client that +// straddles the boundary still sees its prior count and can't double +// up by ~timing requests across the second. +const COUNTER_TTL_SECONDS = 120; + +// Default config — operators tune via `wrangler kv:key put` on the +// `ROUTER_CONFIG` namespace; we re-read every request. +const DEFAULT_PER_KEY_PER_MIN = 60; +const DEFAULT_GLOBAL_PER_MIN = 1000; + +// KV config keys we read from `ROUTER_CONFIG`. Centralized here so +// the docs and the infra seed step refer to the same names. +export const RATE_LIMIT_CONFIG_KEYS = { + PER_KEY_PER_MIN: "RATE_LIMIT_PER_WORKSPACE_PER_MIN", + GLOBAL_PER_MIN: "RATE_LIMIT_GLOBAL_PER_MIN", + DISABLED: "RATE_LIMIT_DISABLED", + // Comma-separated path prefixes added to the bypass list at + // runtime. Static `BYPASS_PREFIXES` below is the safe core; this + // key lets an operator whitelist a new path (e.g. a freshly-added + // health probe) without redeploying. Values are matched the same + // way as static prefixes — exact match or `${prefix}/...` startswith. + // Example KV value: `/api/k8s-readiness,/internal/uptime` + BYPASS_PREFIXES_EXTRA: "RATE_LIMIT_BYPASS_PREFIXES_EXTRA", +} as const; + +// Paths that must NEVER be rate limited. Health probes, static +// assets, and observability endpoints. The router/relayfile-api +// share this list — both Workers see these paths even though only +// one of them actually serves them, and accidentally rate-limiting +// a health probe would self-DoS the upstream monitoring. +const BYPASS_PREFIXES: readonly string[] = [ + "/api/health", + "/health", + "/observer", + "/_next/static", + "/favicon.ico", + // Webhook ingress has no auth token and no workspace path, so it + // would bucket by IP and trip the 60/min cap with even modest + // burst traffic from a single provider (GitHub, Composio, Nango, + // Hookdeck). The webhook worker already has queue-backed + // backpressure, so rate-limiting at the edge would only cause us + // to drop legitimate events. See Codex P1.4 on bundle PR #647. + "/api/v1/webhooks", +]; + +export interface RateLimitEnv { + // Counter storage — shared across both gateway Workers so a + // workspace's limit isn't accidentally doubled by hitting both. + RATE_LIMIT_COUNTERS?: KVNamespace; + // Configuration knobs — same KV namespace the router already uses + // for other flags (e.g. WEBHOOK_ORIGIN). Optional so unit tests + // can run without it. + ROUTER_CONFIG?: KVNamespace; +} + +export interface RateLimitDecision { + allowed: boolean; + retryAfterSec?: number; + counter: number; + limit: number; + // The key the decision applies to — exposed so callers can put it + // in the 429 body (helps operators correlate gateway logs with the + // offending workspace/user/IP). + key: string; +} + +// Pure path helper so we can unit test it without a real Request. +// Checks ONLY the static hardcoded prefix list — does not consult KV. +// Use `isBypassPathDynamic` when env is available and you want the +// runtime-extensible bypass list to apply. +export function isBypassPath(pathname: string): boolean { + return matchesAnyPrefix(pathname, BYPASS_PREFIXES); +} + +function matchesAnyPrefix( + pathname: string, + prefixes: readonly string[], +): boolean { + for (const prefix of prefixes) { + if (pathname === prefix || pathname.startsWith(`${prefix}/`)) { + return true; + } + } + return false; +} + +// Module-scoped cache for the KV-resolved extra bypass prefixes. +// Refreshed at most once per `BYPASS_EXTRAS_TTL_MS` per Worker +// isolate. Operators tuning the KV value see propagation within +// roughly `BYPASS_EXTRAS_TTL_MS` plus the time CF's eventually- +// consistent KV takes to fan out to edges (typically <60s). This +// is the same staleness window that applies to the existing numeric +// config keys (`RATE_LIMIT_PER_WORKSPACE_PER_MIN`, etc.) — keeping +// the model consistent across all runtime knobs. +const BYPASS_EXTRAS_TTL_MS = 60_000; +let cachedBypassExtras: readonly string[] = []; +let cachedBypassExtrasFetchedAt = 0; + +/** + * Test-only: reset the bypass-extras cache between test cases. + * Production code never calls this. + */ +export function __resetBypassExtrasCacheForTests(): void { + cachedBypassExtras = []; + cachedBypassExtrasFetchedAt = 0; +} + +async function getBypassExtras( + env: RateLimitEnv, + now: number, +): Promise { + if (!env.ROUTER_CONFIG) { + return []; + } + if (now - cachedBypassExtrasFetchedAt < BYPASS_EXTRAS_TTL_MS) { + return cachedBypassExtras; + } + try { + const raw = await env.ROUTER_CONFIG.get( + RATE_LIMIT_CONFIG_KEYS.BYPASS_PREFIXES_EXTRA, + ); + cachedBypassExtras = parseBypassExtras(raw); + cachedBypassExtrasFetchedAt = now; + } catch { + // ROUTER_CONFIG read failures should never break the data plane. + // Keep the last known list (or the empty initial) and try again + // after the TTL. + cachedBypassExtrasFetchedAt = now; + } + return cachedBypassExtras; +} + +/** + * Exported for unit tests. Parses the comma-separated KV value into a + * deduped, trimmed list of prefixes, filtering out anything that + * doesn't look like an absolute path (must start with `/`). Defensive + * parsing — bad operator input shouldn't be able to break the bypass + * check. + */ +export function parseBypassExtras(raw: string | null): readonly string[] { + if (!raw) { + return []; + } + const out: string[] = []; + const seen = new Set(); + for (const entry of raw.split(",")) { + const trimmed = entry.trim(); + if (trimmed.length === 0 || !trimmed.startsWith("/")) { + continue; + } + if (seen.has(trimmed)) { + continue; + } + seen.add(trimmed); + out.push(trimmed); + } + return out; +} + +/** + * Returns true if the path should bypass rate limiting — checks both + * the static `BYPASS_PREFIXES` list and the KV-runtime-tunable + * `RATE_LIMIT_BYPASS_PREFIXES_EXTRA` list. The static check is fast + * (no KV read); the KV list is cached per-isolate for + * `BYPASS_EXTRAS_TTL_MS`. Used by `maybeRateLimit` internally. + * + * @param pathname - the URL pathname to check + * @param env - rate limit env containing the KV bindings + * @param now - wall-clock ms; injectable for tests + */ +export async function isBypassPathDynamic( + pathname: string, + env: RateLimitEnv, + now: number = Date.now(), +): Promise { + if (isBypassPath(pathname)) { + return true; + } + const extras = await getBypassExtras(env, now); + return matchesAnyPrefix(pathname, extras); +} + +// Extract a workspace ID from any of the path shapes both gateways +// see. Matches the literal "workspaces/" segment (Hono routes both +// `/v1/workspaces/:id/...` on the relayfile worker and +// `/cloud/api/v1/workspaces/:id/...` on the cloud router). +function extractWorkspaceId(pathname: string): string | null { + const match = pathname.match(/\/workspaces\/([a-zA-Z0-9_-]+)/); + return match?.[1] ?? null; +} + +// Cheap, non-cryptographic hash for opaque bearer tokens so we can +// shard rate counters by token without ever storing the token. We +// don't need collision resistance against an adversary — we just +// want different tokens to map to different buckets. FNV-1a 32-bit +// fits in a few lines and is fine for this. +function fnv1a32Hex(input: string): string { + let hash = 0x811c9dc5; + for (let i = 0; i < input.length; i++) { + hash ^= input.charCodeAt(i); + // 32-bit multiply, kept unsigned via `>>> 0`. + hash = Math.imul(hash, 0x01000193) >>> 0; + } + return hash.toString(16).padStart(8, "0"); +} + +// The session cookie name used by the web app. Must match +// `SESSION_COOKIE_NAME` in `packages/core/src/session/jwt.ts`. +// Defined inline here to keep the rate-limit module self-contained — +// the router package has no dependency on the web package. +const SESSION_COOKIE_NAME = "agent_relay_session"; + +// Parse a single named cookie out of a Cookie header value. Returns +// null when the cookie is absent or has an empty value. Handles both +// quoted and unquoted values; uses the first match when the name +// appears more than once (which is invalid per RFC 6265 but occurs +// in the wild). +function parseCookieValue(cookieHeader: string, name: string): string | null { + // Regex: look for `; name=` or start-of-string `name=`, then capture + // the value up to the next `;` or end-of-string. + const pattern = new RegExp( + `(?:^|;)\\s*${name}\\s*=\\s*"?([^";]*)"?(?:;|$)`, + ); + const match = cookieHeader.match(pattern); + const value = match?.[1]?.trim() ?? null; + return value && value.length > 0 ? value : null; +} + +// Build a stable limit key from the incoming request. Pure function — +// no network calls, no JWKS verify (too expensive for the hot path). +// Order of precedence reflects which signal is most specific: +// +// 1. workspace ID from the path (covers the actual runaway shape) +// 2. authenticated user via bearer token hash (covers non-workspace +// authenticated traffic — e.g. /api/v1/me) +// 3. session cookie for cookie-authenticated /cloud* traffic. Without +// this, cookie-auth requests have neither a workspace ID nor an +// auth header and collapse to IP buckets — conflating all users +// that share a NAT IP into one rate limit bucket. We hash the +// cookie value with FNV-1a 32 so the rate counter key never +// contains the raw session token. +// 4. CF-connecting-IP (anonymous fallback; X-Forwarded-For untrusted +// because anyone can set it, but `cf-connecting-ip` is set by +// Cloudflare edge and can't be spoofed by the client) +// 5. "anon" — last-resort bucket so a misbehaving anon flood is +// still bounded, just bundled together (acceptable: this is the +// least-trusted traffic anyway) +export function extractLimitKey(request: Request): string { + const url = new URL(request.url); + const workspaceId = extractWorkspaceId(url.pathname); + if (workspaceId) { + return `ws:${workspaceId}`; + } + + const auth = request.headers.get("authorization"); + if (auth) { + // Strip the scheme prefix ("Bearer ", "Basic ", etc.) before + // hashing so the same token doesn't bucket differently when a + // client toggles capitalization or omits the scheme. + const token = auth.replace(/^[A-Za-z]+\s+/, "").trim(); + if (token.length > 0) { + return `user:${fnv1a32Hex(token)}`; + } + } + + const cookieHeader = request.headers.get("cookie"); + if (cookieHeader) { + const sessionToken = parseCookieValue(cookieHeader, SESSION_COOKIE_NAME); + if (sessionToken) { + return `session:${fnv1a32Hex(sessionToken)}`; + } + } + + const ip = + request.headers.get("cf-connecting-ip") ?? + request.headers.get("x-forwarded-for")?.split(",")[0]?.trim() ?? + null; + if (ip) { + return `ip:${ip}`; + } + + return "anon"; +} + +async function readNumberConfig( + kv: KVNamespace | undefined, + configKey: string, + fallback: number, +): Promise { + if (!kv) { + return fallback; + } + try { + const raw = await kv.get(configKey); + if (raw == null) { + return fallback; + } + const parsed = Number(raw); + if (!Number.isFinite(parsed) || parsed <= 0) { + return fallback; + } + return parsed; + } catch { + // ROUTER_CONFIG read failures should never break the data plane — + // we'd rather rate-limit at the default than 500 every request. + return fallback; + } +} + +async function readBooleanConfig( + kv: KVNamespace | undefined, + configKey: string, +): Promise { + if (!kv) { + return false; + } + try { + const raw = await kv.get(configKey); + return raw?.trim().toLowerCase() === "true"; + } catch { + return false; + } +} + +// Increment-and-read with KV. Not atomic — two concurrent requests at +// the same edge can both read N, both write N+1, and we lose a count. +// Accepted per the module-level note. The `now` parameter is injected +// for testability. +async function bumpCounter( + kv: KVNamespace, + bucketKey: string, + now: number, +): Promise { + let prior = 0; + try { + const raw = await kv.get(bucketKey); + if (raw != null) { + const parsed = Number.parseInt(raw, 10); + if (Number.isFinite(parsed) && parsed >= 0) { + prior = parsed; + } + } + } catch { + // Counter-store read failure — treat as zero and continue. + // Failing closed (blocking the request) on a transient KV error + // would amplify a single CF KV hiccup into a customer outage. + prior = 0; + } + + const next = prior + 1; + try { + await kv.put(bucketKey, String(next), { + expirationTtl: COUNTER_TTL_SECONDS, + }); + } catch { + // Same reasoning — log via the surrounding fetch handler if + // observability is needed, but never break the request. + } + void now; // reserved for future jittered TTLs; keeps the signature stable + return next; +} + +function bucketForNow(now: number): number { + return Math.floor(now / 1000 / ONE_MINUTE_SECONDS); +} + +function secondsUntilNextBucket(now: number): number { + const elapsedInMinute = Math.floor(now / 1000) % ONE_MINUTE_SECONDS; + return Math.max(1, ONE_MINUTE_SECONDS - elapsedInMinute); +} + +export interface CheckRateLimitOptions { + // Injected for deterministic tests; defaults to Date.now() in prod. + now?: () => number; +} + +export async function checkRateLimit( + key: string, + env: RateLimitEnv, + options: CheckRateLimitOptions = {}, +): Promise { + const nowFn = options.now ?? (() => Date.now()); + const now = nowFn(); + + const [disabled, perKeyLimit, globalLimit] = await Promise.all([ + readBooleanConfig(env.ROUTER_CONFIG, RATE_LIMIT_CONFIG_KEYS.DISABLED), + readNumberConfig( + env.ROUTER_CONFIG, + RATE_LIMIT_CONFIG_KEYS.PER_KEY_PER_MIN, + DEFAULT_PER_KEY_PER_MIN, + ), + readNumberConfig( + env.ROUTER_CONFIG, + RATE_LIMIT_CONFIG_KEYS.GLOBAL_PER_MIN, + DEFAULT_GLOBAL_PER_MIN, + ), + ]); + + if (disabled || !env.RATE_LIMIT_COUNTERS) { + return { allowed: true, counter: 0, limit: perKeyLimit, key }; + } + + const bucket = bucketForNow(now); + const perKeyBucket = `rl:${key}:${bucket}`; + const globalBucket = `rl:__global__:${bucket}`; + + // Sequential reads/writes — KV doesn't support transactions and + // we want the global counter to include this request even if the + // per-key one rejects. Order is intentional: per-key first so a + // single runaway key doesn't get to skew the global ceiling by + // also incrementing the global bucket *before* its per-key check + // would have rejected it. Net effect: a blocked request still + // contributes 1 to per-key but NOT to global. + const perKeyCount = await bumpCounter( + env.RATE_LIMIT_COUNTERS, + perKeyBucket, + now, + ); + + if (perKeyCount > perKeyLimit) { + return { + allowed: false, + retryAfterSec: secondsUntilNextBucket(now), + counter: perKeyCount, + limit: perKeyLimit, + key, + }; + } + + const globalCount = await bumpCounter( + env.RATE_LIMIT_COUNTERS, + globalBucket, + now, + ); + + if (globalCount > globalLimit) { + return { + allowed: false, + retryAfterSec: secondsUntilNextBucket(now), + counter: globalCount, + limit: globalLimit, + key: "__global__", + }; + } + + return { + allowed: true, + counter: perKeyCount, + limit: perKeyLimit, + key, + }; +} + +// Builds the 429 response. Centralized so both Workers return the +// same shape; clients (relayfile mount, CLI) can rely on it. +export function rateLimitResponse( + retryAfterSec: number, + key: string, +): Response { + const body = JSON.stringify({ + error: "rate_limited", + retryAfter: retryAfterSec, + limitKey: key, + }); + return new Response(body, { + status: 429, + headers: { + "content-type": "application/json", + "retry-after": String(retryAfterSec), + "x-ratelimit-key": key, + }, + }); +} + +// Convenience wrapper that combines bypass check, key extraction, +// the actual limit check, and the 429 response. Both Workers call +// this at the very top of their fetch handler. +// +// Returns: +// * `null` — request is allowed; continue to normal handling +// * `Response` — return this 429 immediately +export async function maybeRateLimit( + request: Request, + env: RateLimitEnv, + options: CheckRateLimitOptions = {}, +): Promise { + const url = new URL(request.url); + // Check both the static `BYPASS_PREFIXES` and the KV-runtime-tunable + // extras (cached 60s per isolate). Static check is fast; KV cache + // means at most one KV read per minute per isolate per Worker. + if (await isBypassPathDynamic(url.pathname, env, options.now?.() ?? Date.now())) { + return null; + } + + const key = extractLimitKey(request); + const decision = await checkRateLimit(key, env, options); + if (decision.allowed) { + return null; + } + return rateLimitResponse(decision.retryAfterSec ?? 60, decision.key); +} diff --git a/router/test/rate-limit.test.ts b/router/test/rate-limit.test.ts new file mode 100644 index 00000000..369f55b4 --- /dev/null +++ b/router/test/rate-limit.test.ts @@ -0,0 +1,588 @@ +import { beforeEach, describe, expect, it } from "vitest"; + +import { + __resetBypassExtrasCacheForTests, + checkRateLimit, + extractLimitKey, + isBypassPath, + isBypassPathDynamic, + maybeRateLimit, + parseBypassExtras, + rateLimitResponse, + RATE_LIMIT_CONFIG_KEYS, + type RateLimitEnv, +} from "../src/rate-limit.js"; + +// Tiny in-memory KV shim. Counter increments only need get/put; we +// also store an optional `expirationTtl` so a future test could +// assert TTLs, though the current cases don't depend on it. +type StoredEntry = { value: string; expiresAt: number | null }; + +class MemoryKV { + readonly store = new Map(); + private nowFn: () => number; + + constructor(nowFn: () => number = () => Date.now()) { + this.nowFn = nowFn; + } + + async get(key: string): Promise { + const entry = this.store.get(key); + if (!entry) return null; + if (entry.expiresAt !== null && entry.expiresAt <= this.nowFn()) { + this.store.delete(key); + return null; + } + return entry.value; + } + + async put( + key: string, + value: string, + options?: { expirationTtl?: number }, + ): Promise { + const ttl = options?.expirationTtl; + const expiresAt = + typeof ttl === "number" && ttl > 0 ? this.nowFn() + ttl * 1000 : null; + this.store.set(key, { value, expiresAt }); + } + + // Helpers — not part of the KVNamespace surface. + size(): number { + return this.store.size; + } +} + +function asKV(kv: MemoryKV): KVNamespace { + return kv as unknown as KVNamespace; +} + +function buildEnv(opts: { + counters?: MemoryKV; + config?: MemoryKV; +}): RateLimitEnv { + return { + RATE_LIMIT_COUNTERS: opts.counters ? asKV(opts.counters) : undefined, + ROUTER_CONFIG: opts.config ? asKV(opts.config) : undefined, + }; +} + +const FIXED_NOW = 1715683680000; // 2024-05-14T11:28:00Z — middle of an even minute +const fixedNow = () => FIXED_NOW; + +describe("isBypassPath", () => { + it.each([ + ["/api/health", true], + ["/api/health/db", true], + ["/health", true], + ["/observer", true], + ["/observer/session/abc", true], + ["/_next/static/chunks/123.js", true], + ["/favicon.ico", true], + ["/v1/workspaces/ws_123/sync/status", false], + ["/cloud/api/v1/workspaces/ws_123/sandboxes", false], + ["/", false], + ])("path %s bypass=%s", (path, expected) => { + expect(isBypassPath(path)).toBe(expected); + }); +}); + +describe("extractLimitKey", () => { + it("picks workspace id from /v1/workspaces/:id/...", () => { + const req = new Request( + "https://api.relayfile.dev/v1/workspaces/ws_abc123/sync/status", + ); + expect(extractLimitKey(req)).toBe("ws:ws_abc123"); + }); + + it("picks workspace id from /cloud/api/v1/workspaces/:id/...", () => { + const req = new Request( + "https://agentrelay.com/cloud/api/v1/workspaces/ws_xyz/sandboxes", + ); + expect(extractLimitKey(req)).toBe("ws:ws_xyz"); + }); + + it("hashes bearer token when no workspace path is present", () => { + const req = new Request("https://agentrelay.com/cloud/api/v1/me", { + headers: { authorization: "Bearer abcdef0123456789" }, + }); + const key = extractLimitKey(req); + expect(key).toMatch(/^user:[0-9a-f]{8}$/); + }); + + it("buckets different tokens to different user keys", () => { + const a = extractLimitKey( + new Request("https://agentrelay.com/cloud/api/v1/me", { + headers: { authorization: "Bearer aaaaaaaa" }, + }), + ); + const b = extractLimitKey( + new Request("https://agentrelay.com/cloud/api/v1/me", { + headers: { authorization: "Bearer bbbbbbbb" }, + }), + ); + expect(a).not.toBe(b); + expect(a).toMatch(/^user:/); + expect(b).toMatch(/^user:/); + }); + + it("falls back to cf-connecting-ip for anonymous traffic", () => { + const req = new Request("https://agentrelay.com/", { + headers: { "cf-connecting-ip": "203.0.113.5" }, + }); + expect(extractLimitKey(req)).toBe("ip:203.0.113.5"); + }); + + it("prefers cf-connecting-ip over x-forwarded-for", () => { + const req = new Request("https://agentrelay.com/", { + headers: { + "cf-connecting-ip": "203.0.113.5", + "x-forwarded-for": "10.0.0.1, 10.0.0.2", + }, + }); + expect(extractLimitKey(req)).toBe("ip:203.0.113.5"); + }); + + it("uses x-forwarded-for first hop when cf-connecting-ip is missing", () => { + const req = new Request("https://agentrelay.com/", { + headers: { "x-forwarded-for": "10.0.0.1, 10.0.0.2" }, + }); + expect(extractLimitKey(req)).toBe("ip:10.0.0.1"); + }); + + it("keys off session cookie for /cloud* traffic with no auth header", () => { + const req = new Request("https://agentrelay.com/cloud/api/v1/me", { + headers: { cookie: "agent_relay_session=mysecrettoken123" }, + }); + const key = extractLimitKey(req); + expect(key).toMatch(/^session:[0-9a-f]{8}$/); + }); + + it("auth header takes priority over session cookie", () => { + const req = new Request("https://agentrelay.com/cloud/api/v1/me", { + headers: { + authorization: "Bearer bearertoken999", + cookie: "agent_relay_session=mysecrettoken123", + }, + }); + const key = extractLimitKey(req); + // auth header wins — produces a user: key, not session: + expect(key).toMatch(/^user:[0-9a-f]{8}$/); + }); + + it("buckets different session cookie values to different session keys", () => { + const a = extractLimitKey( + new Request("https://agentrelay.com/cloud/api/v1/me", { + headers: { cookie: "agent_relay_session=tokenA" }, + }), + ); + const b = extractLimitKey( + new Request("https://agentrelay.com/cloud/api/v1/me", { + headers: { cookie: "agent_relay_session=tokenB" }, + }), + ); + expect(a).not.toBe(b); + expect(a).toMatch(/^session:/); + expect(b).toMatch(/^session:/); + }); + + it("/cloud* with no cookie and no auth falls back to IP", () => { + const req = new Request("https://agentrelay.com/cloud/api/v1/me", { + headers: { "cf-connecting-ip": "198.51.100.1" }, + }); + expect(extractLimitKey(req)).toBe("ip:198.51.100.1"); + }); + + it("/cloud* with no cookie, no auth, no IP falls back to anon", () => { + const req = new Request("https://agentrelay.com/cloud/api/v1/me"); + expect(extractLimitKey(req)).toBe("anon"); + }); + + it("falls back to anon when no signal is present", () => { + const req = new Request("https://agentrelay.com/"); + expect(extractLimitKey(req)).toBe("anon"); + }); +}); + +describe("checkRateLimit", () => { + let counters: MemoryKV; + let config: MemoryKV; + let env: RateLimitEnv; + + beforeEach(() => { + counters = new MemoryKV(fixedNow); + config = new MemoryKV(fixedNow); + env = buildEnv({ counters, config }); + }); + + it("allows requests under the per-key limit", async () => { + for (let i = 1; i <= 5; i++) { + const result = await checkRateLimit("ws:demo", env, { now: fixedNow }); + expect(result.allowed).toBe(true); + expect(result.counter).toBe(i); + } + }); + + it("returns 429 once the per-key limit is exceeded", async () => { + // Pin the per-workspace limit low for deterministic assertions. + await config.put(RATE_LIMIT_CONFIG_KEYS.PER_KEY_PER_MIN, "3"); + + // Sequential calls — the in-memory KV shim mirrors a single edge + // processing requests in order. KV's eventual consistency across + // edges is acknowledged in the module-level docstring but is not + // what this test asserts. + const results = [ + await checkRateLimit("ws:demo", env, { now: fixedNow }), + await checkRateLimit("ws:demo", env, { now: fixedNow }), + await checkRateLimit("ws:demo", env, { now: fixedNow }), + ]; + expect(results.map((r) => r.allowed)).toEqual([true, true, true]); + + const blocked = await checkRateLimit("ws:demo", env, { now: fixedNow }); + expect(blocked.allowed).toBe(false); + expect(blocked.counter).toBe(4); + expect(blocked.limit).toBe(3); + expect(blocked.retryAfterSec).toBeGreaterThan(0); + expect(blocked.retryAfterSec).toBeLessThanOrEqual(60); + expect(blocked.key).toBe("ws:demo"); + }); + + it("does not pollute the global counter on a per-key reject", async () => { + await config.put(RATE_LIMIT_CONFIG_KEYS.PER_KEY_PER_MIN, "1"); + await checkRateLimit("ws:a", env, { now: fixedNow }); + const blocked = await checkRateLimit("ws:a", env, { now: fixedNow }); + expect(blocked.allowed).toBe(false); + + // Verify the global counter only ever incremented once (for the + // single allowed request) — the blocked one should NOT have + // incremented it. This is intentional so a single misbehaving + // key can't push the global ceiling around. + const bucket = Math.floor(FIXED_NOW / 1000 / 60); + const stored = await counters.get(`rl:__global__:${bucket}`); + expect(stored).toBe("1"); + }); + + it("returns 429 once the global limit is exceeded", async () => { + await config.put(RATE_LIMIT_CONFIG_KEYS.GLOBAL_PER_MIN, "2"); + await config.put(RATE_LIMIT_CONFIG_KEYS.PER_KEY_PER_MIN, "10"); + + expect( + (await checkRateLimit("ws:a", env, { now: fixedNow })).allowed, + ).toBe(true); + expect( + (await checkRateLimit("ws:b", env, { now: fixedNow })).allowed, + ).toBe(true); + + const blocked = await checkRateLimit("ws:c", env, { now: fixedNow }); + expect(blocked.allowed).toBe(false); + // Global rejects surface the synthetic `__global__` key so the + // 429 body tells operators it's the global ceiling, not the + // per-key one. + expect(blocked.key).toBe("__global__"); + }); + + it("respects RATE_LIMIT_DISABLED kill switch", async () => { + await config.put(RATE_LIMIT_CONFIG_KEYS.PER_KEY_PER_MIN, "1"); + await config.put(RATE_LIMIT_CONFIG_KEYS.DISABLED, "true"); + + // Even though the limit is 1, the kill switch should let + // unlimited requests through. + for (let i = 0; i < 5; i++) { + const result = await checkRateLimit("ws:demo", env, { now: fixedNow }); + expect(result.allowed).toBe(true); + } + // No counter writes when disabled — saves KV operations. + expect(counters.size()).toBe(0); + }); + + it("resets per minute (different bucket)", async () => { + await config.put(RATE_LIMIT_CONFIG_KEYS.PER_KEY_PER_MIN, "2"); + + const firstMinute = fixedNow; + const secondMinute = () => FIXED_NOW + 60_000; + + expect( + (await checkRateLimit("ws:demo", env, { now: firstMinute })).allowed, + ).toBe(true); + expect( + (await checkRateLimit("ws:demo", env, { now: firstMinute })).allowed, + ).toBe(true); + expect( + (await checkRateLimit("ws:demo", env, { now: firstMinute })).allowed, + ).toBe(false); + + // Roll forward one full minute — bucket key changes, counter + // starts over. + const reset = await checkRateLimit("ws:demo", env, { now: secondMinute }); + expect(reset.allowed).toBe(true); + expect(reset.counter).toBe(1); + }); + + it("falls back to defaults when ROUTER_CONFIG is unbound", async () => { + const noConfigEnv = buildEnv({ counters }); + const result = await checkRateLimit("ws:demo", noConfigEnv, { + now: fixedNow, + }); + expect(result.allowed).toBe(true); + expect(result.limit).toBe(60); // DEFAULT_PER_KEY_PER_MIN + }); + + it("treats missing RATE_LIMIT_COUNTERS as allow-all (fail open)", async () => { + const noCounterEnv = buildEnv({ config }); + await config.put(RATE_LIMIT_CONFIG_KEYS.PER_KEY_PER_MIN, "1"); + // Without a counter store we cannot enforce, so behavior is + // intentionally permissive (the limit is documented as + // best-effort). This guards against a missing binding silently + // breaking the data plane. + for (let i = 0; i < 10; i++) { + expect( + (await checkRateLimit("ws:demo", noCounterEnv, { now: fixedNow })) + .allowed, + ).toBe(true); + } + }); +}); + +describe("rateLimitResponse", () => { + it("emits a 429 with the documented shape", async () => { + const res = rateLimitResponse(42, "ws:demo"); + expect(res.status).toBe(429); + expect(res.headers.get("retry-after")).toBe("42"); + expect(res.headers.get("x-ratelimit-key")).toBe("ws:demo"); + expect(res.headers.get("content-type")).toBe("application/json"); + const body = (await res.json()) as Record; + expect(body).toEqual({ + error: "rate_limited", + retryAfter: 42, + limitKey: "ws:demo", + }); + }); +}); + +describe("maybeRateLimit", () => { + it("returns null for bypass paths without touching KV", async () => { + const counters = new MemoryKV(fixedNow); + const config = new MemoryKV(fixedNow); + const env = buildEnv({ counters, config }); + + const result = await maybeRateLimit( + new Request("https://api.relayfile.dev/health"), + env, + { now: fixedNow }, + ); + expect(result).toBeNull(); + expect(counters.size()).toBe(0); + }); + + it("returns null when under the limit", async () => { + const counters = new MemoryKV(fixedNow); + const config = new MemoryKV(fixedNow); + const env = buildEnv({ counters, config }); + + const result = await maybeRateLimit( + new Request("https://api.relayfile.dev/v1/workspaces/ws_a/sync/status"), + env, + { now: fixedNow }, + ); + expect(result).toBeNull(); + }); + + it("returns a 429 once limit is exceeded", async () => { + const counters = new MemoryKV(fixedNow); + const config = new MemoryKV(fixedNow); + const env = buildEnv({ counters, config }); + await config.put(RATE_LIMIT_CONFIG_KEYS.PER_KEY_PER_MIN, "1"); + + const url = + "https://api.relayfile.dev/v1/workspaces/ws_a/sync/status"; + const first = await maybeRateLimit(new Request(url), env, { + now: fixedNow, + }); + expect(first).toBeNull(); + + const blocked = await maybeRateLimit(new Request(url), env, { + now: fixedNow, + }); + expect(blocked).not.toBeNull(); + expect(blocked!.status).toBe(429); + expect(blocked!.headers.get("x-ratelimit-key")).toBe("ws:ws_a"); + }); +}); + +describe("parseBypassExtras", () => { + it("returns empty array for null / empty input", () => { + expect(parseBypassExtras(null)).toEqual([]); + expect(parseBypassExtras("")).toEqual([]); + expect(parseBypassExtras(" ")).toEqual([]); + }); + + it("parses comma-separated absolute paths", () => { + expect(parseBypassExtras("/api/k8s-readiness,/internal/uptime")).toEqual([ + "/api/k8s-readiness", + "/internal/uptime", + ]); + }); + + it("trims whitespace around entries", () => { + expect(parseBypassExtras(" /a , /b , /c ")).toEqual(["/a", "/b", "/c"]); + }); + + it("ignores entries that don't start with /", () => { + // Defensive: bad operator input shouldn't poison the list. + expect(parseBypassExtras("/ok,not-a-path,/also-ok")).toEqual([ + "/ok", + "/also-ok", + ]); + }); + + it("dedupes repeated entries", () => { + expect(parseBypassExtras("/x,/x,/y,/x")).toEqual(["/x", "/y"]); + }); +}); + +describe("isBypassPathDynamic", () => { + beforeEach(() => { + __resetBypassExtrasCacheForTests(); + }); + + it("returns true for static bypass paths without touching KV", async () => { + const env = buildEnv({}); // no ROUTER_CONFIG bound + expect(await isBypassPathDynamic("/api/health", env, FIXED_NOW)).toBe(true); + expect(await isBypassPathDynamic("/observer/foo", env, FIXED_NOW)).toBe( + true, + ); + }); + + it("returns false for non-bypass paths when no KV extras set", async () => { + const config = new MemoryKV(fixedNow); + const env = buildEnv({ config }); + expect( + await isBypassPathDynamic("/v1/workspaces/ws_a/sync/status", env, FIXED_NOW), + ).toBe(false); + }); + + it("respects KV-configured extras (exact match)", async () => { + const config = new MemoryKV(fixedNow); + const env = buildEnv({ config }); + await config.put( + RATE_LIMIT_CONFIG_KEYS.BYPASS_PREFIXES_EXTRA, + "/api/k8s-readiness", + ); + expect( + await isBypassPathDynamic("/api/k8s-readiness", env, FIXED_NOW), + ).toBe(true); + }); + + it("respects KV-configured extras (prefix match)", async () => { + const config = new MemoryKV(fixedNow); + const env = buildEnv({ config }); + await config.put( + RATE_LIMIT_CONFIG_KEYS.BYPASS_PREFIXES_EXTRA, + "/internal/uptime", + ); + expect( + await isBypassPathDynamic("/internal/uptime/check", env, FIXED_NOW), + ).toBe(true); + }); + + it("caches KV reads for 60s (does not re-read on every call)", async () => { + const config = new MemoryKV(fixedNow); + const env = buildEnv({ config }); + await config.put( + RATE_LIMIT_CONFIG_KEYS.BYPASS_PREFIXES_EXTRA, + "/api/k8s-readiness", + ); + + // Count KV reads by wrapping the real one. + let reads = 0; + const originalGet = env.ROUTER_CONFIG!.get.bind(env.ROUTER_CONFIG!); + env.ROUTER_CONFIG!.get = ((k: string) => { + reads++; + return originalGet(k); + }) as NonNullable["get"]; + + // First call hits KV. + await isBypassPathDynamic("/v1/x", env, FIXED_NOW); + expect(reads).toBe(1); + // Second call within 60s reuses the cache. + await isBypassPathDynamic("/v1/y", env, FIXED_NOW + 30_000); + expect(reads).toBe(1); + // Past the TTL boundary, fetches again. + await isBypassPathDynamic("/v1/z", env, FIXED_NOW + 61_000); + expect(reads).toBe(2); + }); + + it("falls back to last-known list on KV transient errors", async () => { + const config = new MemoryKV(fixedNow); + const env = buildEnv({ config }); + await config.put( + RATE_LIMIT_CONFIG_KEYS.BYPASS_PREFIXES_EXTRA, + "/cached-ok", + ); + + // Warm the cache once. + await isBypassPathDynamic("/v1/x", env, FIXED_NOW); + expect(await isBypassPathDynamic("/cached-ok", env, FIXED_NOW)).toBe(true); + + // Now break the KV. After TTL expiry the read will throw — we + // should fall back to the cached value rather than crashing. + env.ROUTER_CONFIG!.get = (() => { + throw new Error("KV unavailable"); + }) as NonNullable["get"]; + + expect( + await isBypassPathDynamic("/cached-ok", env, FIXED_NOW + 120_000), + ).toBe(true); + }); + + it("static and dynamic bypass coexist (static still works with KV extras present)", async () => { + const config = new MemoryKV(fixedNow); + const env = buildEnv({ config }); + await config.put( + RATE_LIMIT_CONFIG_KEYS.BYPASS_PREFIXES_EXTRA, + "/internal/uptime", + ); + // Static + dynamic both bypass. + expect(await isBypassPathDynamic("/api/health", env, FIXED_NOW)).toBe(true); + expect( + await isBypassPathDynamic("/internal/uptime", env, FIXED_NOW), + ).toBe(true); + }); +}); + +describe("maybeRateLimit honors KV bypass extras", () => { + beforeEach(() => { + __resetBypassExtrasCacheForTests(); + }); + + it("bypasses a runtime-whitelisted path", async () => { + const counters = new MemoryKV(fixedNow); + const config = new MemoryKV(fixedNow); + const env = buildEnv({ counters, config }); + // Tight limit so anything not bypassed would 429 on call #2. + await config.put(RATE_LIMIT_CONFIG_KEYS.PER_KEY_PER_MIN, "1"); + await config.put( + RATE_LIMIT_CONFIG_KEYS.BYPASS_PREFIXES_EXTRA, + "/api/k8s-readiness", + ); + + // Hit a non-bypassed path twice → second call 429s. + const url = "https://api.relayfile.dev/v1/workspaces/ws_a/sync/status"; + await maybeRateLimit(new Request(url), env, { now: fixedNow }); + const blocked = await maybeRateLimit(new Request(url), env, { + now: fixedNow, + }); + expect(blocked!.status).toBe(429); + + // Hit the runtime-whitelisted path 5 times → never 429s, regardless + // of the very tight per-key limit. + for (let i = 0; i < 5; i++) { + const r = await maybeRateLimit( + new Request("https://api.relayfile.dev/api/k8s-readiness"), + env, + { now: fixedNow }, + ); + expect(r).toBeNull(); + } + }); +}); From 7057512abf0346ff31e41791362fa77609cdb570 Mon Sep 17 00:00:00 2001 From: Khaliq Date: Tue, 19 May 2026 15:07:53 +0200 Subject: [PATCH 35/49] feat(webhook-worker): add dormant queue foundation (#744) Co-authored-by: Claude Opus 4.7 (1M context) --- router/index.ts | 25 +++++++++++++++++++------ 1 file changed, 19 insertions(+), 6 deletions(-) diff --git a/router/index.ts b/router/index.ts index 4c3bd718..76cc6447 100644 --- a/router/index.ts +++ b/router/index.ts @@ -211,17 +211,30 @@ async function shouldUseCloudWebWorker(pathname: string, env: Env): Promise { + try { + return (await env.ROUTER_CONFIG?.get(WEBHOOK_ORIGIN_FLAG_KEY)) ?? null; + } catch { + return null; + } +} + +export async function shouldUseNangoWebhookWorkerRoute( + pathname: string, + env: Env, +): Promise { + void pathname; + void (await readWebhookOriginFlag(env)); + return false; +} + async function shouldUseWebhookWorker(pathname: string, env: Env): Promise { if (!isWebhookWorkerPath(pathname)) { return false; } - try { - const configured = await env.ROUTER_CONFIG?.get(WEBHOOK_ORIGIN_FLAG_KEY); - return configured?.trim().toLowerCase() === "worker"; - } catch { - return false; - } + const configured = await readWebhookOriginFlag(env); + return configured?.trim().toLowerCase() === "worker"; } function buildWorkerOriginRequest(request: Request, requestUrl: URL, workerOrigin: string): Request { From 6392e9bcfa0a5b6150922b766c6ab95d7493aefe Mon Sep 17 00:00:00 2001 From: Khaliq Date: Tue, 19 May 2026 21:58:06 +0200 Subject: [PATCH 36/49] feat(router): gate nango webhook worker cutover (#753) --- router/index.ts | 68 ++++++++++++++++++++++++++++++++----------------- 1 file changed, 44 insertions(+), 24 deletions(-) diff --git a/router/index.ts b/router/index.ts index 76cc6447..7ff89954 100644 --- a/router/index.ts +++ b/router/index.ts @@ -40,6 +40,7 @@ const WEBHOOK_WORKER_PATHS = new Set([ "/api/v1/webhooks/hookdeck", "/api/v1/webhooks/nango", ]); +const NANGO_WEBHOOK_WORKER_PATH = "/api/v1/webhooks/nango"; function isPathWithinPrefix(pathname: string, prefix: string): boolean { return pathname === prefix || pathname.startsWith(`${prefix}/`); @@ -65,7 +66,11 @@ function isCloudPath(pathname: string): boolean { // to gate worker forwarding so unrelated routes under /api/v1/webhooks/* (e.g. // the Composio OAuth callback) still reach the Lambda. export function isWebhookWorkerPath(pathname: string): boolean { - return WEBHOOK_WORKER_PATHS.has(pathname); + return WEBHOOK_WORKER_PATHS.has(stripPathPrefix(pathname, CLOUD_PATH_PREFIX)); +} + +function isNangoWebhookWorkerPath(pathname: string): boolean { + return stripPathPrefix(pathname, CLOUD_PATH_PREFIX) === NANGO_WEBHOOK_WORKER_PATH; } function stripPathPrefix(pathname: string, prefix: string): string { @@ -203,6 +208,10 @@ async function shouldUseCloudWebWorker(pathname: string, env: Env): Promise { - void pathname; - void (await readWebhookOriginFlag(env)); - return false; -} - -async function shouldUseWebhookWorker(pathname: string, env: Env): Promise { - if (!isWebhookWorkerPath(pathname)) { + if (!isNangoWebhookWorkerPath(pathname)) { return false; } @@ -237,19 +240,34 @@ async function shouldUseWebhookWorker(pathname: string, env: Env): Promise { + return shouldUseNangoWebhookWorkerRoute(pathname, env); +} + +function buildWebhookWorkerRequest( + request: Request, + requestUrl: URL, + workerOrigin?: string, +): Request { const targetUrl = new URL(requestUrl.toString()); - const originUrl = new URL(workerOrigin); - targetUrl.protocol = originUrl.protocol; - targetUrl.hostname = originUrl.hostname; - targetUrl.port = originUrl.port; + targetUrl.pathname = stripPathPrefix(targetUrl.pathname, CLOUD_PATH_PREFIX); - return new Request(targetUrl.toString(), { + if (workerOrigin) { + const originUrl = new URL(workerOrigin); + targetUrl.protocol = originUrl.protocol; + targetUrl.hostname = originUrl.hostname; + targetUrl.port = originUrl.port; + } + + const init: RequestInit & { duplex?: "half" } = { method: request.method, headers: request.headers, body: request.body, redirect: "manual", - }); + duplex: "half", + }; + + return new Request(targetUrl.toString(), init); } export default { @@ -272,26 +290,28 @@ export default { // /cloud Worker path bypasses the recorder entirely and the replay // harness has no corpus to prove equivalence during Phase 4 cutover. // See Codex P2.6 on bundle PR #647. - const recorderRequestClone = hasRecorderEnv(env) + const recorderEnv = hasRecorderEnv(env) ? env : null; + const recorderRequestClone = recorderEnv ? (request.clone() as unknown as Request) : null; if ((await shouldUseCloudWebWorker(url.pathname, env)) && env.CLOUD_WEB_WORKER) { const workerResponse = await env.CLOUD_WEB_WORKER.fetch(request); - if (recorderRequestClone) { + if (recorderRequestClone && recorderEnv) { ctx.waitUntil( - maybeRecord(recorderRequestClone, workerResponse.clone(), env, ctx), + maybeRecord(recorderRequestClone, workerResponse.clone(), recorderEnv, ctx), ); } return workerResponse; } if (await shouldUseWebhookWorker(url.pathname, env)) { + const webhookWorkerRequest = buildWebhookWorkerRequest(request, url); if (env.WEBHOOK_WORKER) { - const workerResponse = await env.WEBHOOK_WORKER.fetch(request); - if (recorderRequestClone) { + const workerResponse = await env.WEBHOOK_WORKER.fetch(webhookWorkerRequest); + if (recorderRequestClone && recorderEnv) { ctx.waitUntil( - maybeRecord(recorderRequestClone, workerResponse.clone(), env, ctx), + maybeRecord(recorderRequestClone, workerResponse.clone(), recorderEnv, ctx), ); } return workerResponse; @@ -300,11 +320,11 @@ export default { const workerOrigin = env.WEBHOOK_WORKER_ORIGIN?.trim(); if (workerOrigin) { const originResponse = await globalThis.fetch( - buildWorkerOriginRequest(request, url, workerOrigin), + buildWebhookWorkerRequest(request, url, workerOrigin), ); - if (recorderRequestClone) { + if (recorderRequestClone && recorderEnv) { ctx.waitUntil( - maybeRecord(recorderRequestClone, originResponse.clone(), env, ctx), + maybeRecord(recorderRequestClone, originResponse.clone(), recorderEnv, ctx), ); } return originResponse; From e55df921d50482a8a6d7b2b69b451ca46fd2bfe0 Mon Sep 17 00:00:00 2001 From: Khaliq Date: Wed, 20 May 2026 22:16:19 +0200 Subject: [PATCH 37/49] Drop cloudOrigin router gate (#854) --- router/index.ts | 32 +++++++++++++++++++++++--------- 1 file changed, 23 insertions(+), 9 deletions(-) diff --git a/router/index.ts b/router/index.ts index 7ff89954..71665086 100644 --- a/router/index.ts +++ b/router/index.ts @@ -27,8 +27,8 @@ const PRIMARY_HOST = "agentrelay.com"; const FILE_OBSERVER_PATH_PREFIX = "/observer/file"; const OBSERVER_PATH_PREFIX = "/observer"; const CLOUD_PATH_PREFIX = "/cloud"; -const CLOUD_ORIGIN_FLAG_KEY = "cloudOrigin"; const WEBHOOK_ORIGIN_FLAG_KEY = "WEBHOOK_ORIGIN"; +let loggedPhase5aLambdaEliminated = false; // Exact paths the webhook worker handles. Other sub-paths under // /api/v1/webhooks (notably /api/v1/webhooks/composio/connect/callback, an @@ -180,8 +180,7 @@ export function rewriteLocation( export function getOrigin(hostname: string, pathname: string, env: Env): string { // /cloud* defaults to the Next.js cloud app regardless of host. Requests only - // reach this branch when the KV flag has not already forwarded them to the - // cloud-web Worker. + // reach this fallback when the cloud-web Worker service binding is absent. if (isCloudPath(pathname)) { return env.CLOUD_APP_ORIGIN; } @@ -212,12 +211,16 @@ async function shouldUseCloudWebWorker(pathname: string, env: Env): Promise { @@ -295,7 +298,18 @@ export default { ? (request.clone() as unknown as Request) : null; - if ((await shouldUseCloudWebWorker(url.pathname, env)) && env.CLOUD_WEB_WORKER) { + if (await shouldUseCloudWebWorker(url.pathname, env)) { + logPhase5aLambdaEliminatedOnce(); + if (!env.CLOUD_WEB_WORKER) { + return new Response( + JSON.stringify({ error: "cloud web worker binding unavailable" }), + { + status: 503, + headers: { "content-type": "application/json" }, + }, + ); + } + const workerResponse = await env.CLOUD_WEB_WORKER.fetch(request); if (recorderRequestClone && recorderEnv) { ctx.waitUntil( From 0721d9a97c3709992694b05fd6b741137b8392f2 Mon Sep 17 00:00:00 2001 From: Will Washburn Date: Thu, 21 May 2026 22:14:15 -0400 Subject: [PATCH 38/49] fix(router): prefer corrected relay web origin --- router/index.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/router/index.ts b/router/index.ts index 71665086..daaa3b1d 100644 --- a/router/index.ts +++ b/router/index.ts @@ -20,7 +20,7 @@ function hasRecorderEnv(env: Env): env is Env & RecorderEnv { return Boolean(env.TRAFFIC_RECORDER && env.ROUTER_CONFIG); } -const FALLBACK_PROXY_ORIGIN = "https://orgin.agentrelay.net"; +const FALLBACK_PROXY_ORIGIN = "https://origin.agentrelay.net"; const OBSERVER_ORIGIN = "https://observer.relaycast.dev"; const DEFAULT_FILE_OBSERVER_ORIGIN = "https://relayfile-file-observer.pages.dev"; const PRIMARY_HOST = "agentrelay.com"; @@ -188,7 +188,7 @@ export function getOrigin(hostname: string, pathname: string, env: Env): string // The production agentrelay.com apex is a split router: // 1. /observer/file* goes to the RelayFile file observer app // 2. /observer* stays on the Relaycast observer app - // 3. everything else falls back to the legacy proxy target + // 3. everything else falls back to the relay web origin if (hostname === PRIMARY_HOST) { if (isPrimaryFileObserverPath(hostname, pathname)) { return env.FILE_OBSERVER_ORIGIN ?? DEFAULT_FILE_OBSERVER_ORIGIN; From 2b8c06932db47f1e00b20c6c4ae497620c261e90 Mon Sep 17 00:00:00 2001 From: Khaliq Date: Fri, 22 May 2026 14:04:14 +0200 Subject: [PATCH 39/49] fix(router): break /webhooks/nango loop on webhook-worker forward header (#946) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit PR #944 fixed webhook-worker's classification of Nango forward envelopes (`ingress: "nango-forward"` instead of `nango-sync`), and the queue consumer now correctly POSTs the raw envelope to `origin.agentrelay.cloud/cloud/api/v1/webhooks/nango` so cloud-web's `routeForwardEvent` → `handleGitHubForward` can run. Problem: the router worker catches every `/api/v1/webhooks/nango` request when `WEBHOOK_ORIGIN=worker` (per ws-option-b flip, 2026-05-20) and forwards it to webhook-worker — including webhook-worker's own outbound forwards. The queue consumer re-enqueues the bounced request, queue redelivers, ad infinitum until `maxRetries: 5` exhausts. Cloud-web's `handleGitHubForward` is never invoked, `dispatchIntegrationWatchEvent` never fires, and proactive agents triggered on github events stay inert. Evidence post-#944 deploy: - 0 "Nango webhook received" logs on cloud-web-worker - 0 `integration-watch-dispatch` lines (PR #939 diag) - All 196+ POSTs to /api/v1/webhooks/nango landed on webhook-worker - inbound count to webhook-worker rose ~2x post-deploy (loop traffic) Fix: honour the `x-cloud-webhook-worker-forwarded: webhook-worker` header that `buildForwardHeaders` already sets in webhook-worker's queue consumer. When the router sees that header, it skips the webhook-worker redirect and lets the request fall through to cloud-web — the actual destination. Tests: - New `packages/router/test/webhook-routing.test.ts` (4 cases): - routes to webhook-worker when WEBHOOK_ORIGIN=worker (existing) - bypasses webhook-worker when loop-break header present (new) - ignores unrelated header values (don't widen bypass) - regression: flag unset still routes to cloud-web - `npm run typecheck`: clean - Existing router tests (rate-limit, recorder, redact): 60 tests pass Co-authored-by: Hubspot Phase B Bot Co-authored-by: Claude Opus 4.7 (1M context) --- router/index.ts | 39 ++++++- router/test/webhook-routing.test.ts | 157 ++++++++++++++++++++++++++++ 2 files changed, 191 insertions(+), 5 deletions(-) create mode 100644 router/test/webhook-routing.test.ts diff --git a/router/index.ts b/router/index.ts index daaa3b1d..df9cfde6 100644 --- a/router/index.ts +++ b/router/index.ts @@ -28,6 +28,18 @@ const FILE_OBSERVER_PATH_PREFIX = "/observer/file"; const OBSERVER_PATH_PREFIX = "/observer"; const CLOUD_PATH_PREFIX = "/cloud"; const WEBHOOK_ORIGIN_FLAG_KEY = "WEBHOOK_ORIGIN"; + +// Header set by webhook-worker's queue consumer +// (`packages/webhook-worker/src/queue-consumer.ts`'s `buildForwardHeaders`) on +// outbound forwards to cloud-web's `/api/v1/webhooks/nango`. When the +// `WEBHOOK_ORIGIN` flag is `"worker"`, the router would otherwise redirect +// every `/api/v1/webhooks/nango` POST back to webhook-worker — including +// webhook-worker's own outbound forwards, producing an infinite redelivery +// loop bounded only by `maxRetries`. Honouring this header lets webhook-worker +// reach cloud-web's `routeForwardEvent` → `handleGitHubForward` path which is +// the actual destination of the forward. +const WEBHOOK_WORKER_FORWARDED_HEADER = "x-cloud-webhook-worker-forwarded"; +const WEBHOOK_WORKER_FORWARDED_VALUE = "webhook-worker"; let loggedPhase5aLambdaEliminated = false; // Exact paths the webhook worker handles. Other sub-paths under @@ -202,12 +214,16 @@ export function getOrigin(hostname: string, pathname: string, env: Env): string return FALLBACK_PROXY_ORIGIN; } -async function shouldUseCloudWebWorker(pathname: string, env: Env): Promise { +async function shouldUseCloudWebWorker( + pathname: string, + request: Request, + env: Env, +): Promise { if (!isCloudPath(pathname)) { return false; } - if (await shouldUseWebhookWorker(pathname, env)) { + if (await shouldUseWebhookWorker(pathname, request, env)) { return false; } @@ -243,7 +259,20 @@ export async function shouldUseNangoWebhookWorkerRoute( return configured?.trim().toLowerCase() === "worker"; } -async function shouldUseWebhookWorker(pathname: string, env: Env): Promise { +async function shouldUseWebhookWorker( + pathname: string, + request: Request, + env: Env, +): Promise { + // Break the redelivery loop: webhook-worker's queue consumer forwards the + // raw envelope back to the same `/api/v1/webhooks/nango` route on + // `origin.agentrelay.cloud` so cloud-web's `handleGitHubForward` can run. + // Without this header check, the router catches that forward and redirects + // it back to webhook-worker, which re-enqueues, ad infinitum (or until + // `maxRetries`). + if (request.headers.get(WEBHOOK_WORKER_FORWARDED_HEADER) === WEBHOOK_WORKER_FORWARDED_VALUE) { + return false; + } return shouldUseNangoWebhookWorkerRoute(pathname, env); } @@ -298,7 +327,7 @@ export default { ? (request.clone() as unknown as Request) : null; - if (await shouldUseCloudWebWorker(url.pathname, env)) { + if (await shouldUseCloudWebWorker(url.pathname, request, env)) { logPhase5aLambdaEliminatedOnce(); if (!env.CLOUD_WEB_WORKER) { return new Response( @@ -319,7 +348,7 @@ export default { return workerResponse; } - if (await shouldUseWebhookWorker(url.pathname, env)) { + if (await shouldUseWebhookWorker(url.pathname, request, env)) { const webhookWorkerRequest = buildWebhookWorkerRequest(request, url); if (env.WEBHOOK_WORKER) { const workerResponse = await env.WEBHOOK_WORKER.fetch(webhookWorkerRequest); diff --git a/router/test/webhook-routing.test.ts b/router/test/webhook-routing.test.ts new file mode 100644 index 00000000..8daa494d --- /dev/null +++ b/router/test/webhook-routing.test.ts @@ -0,0 +1,157 @@ +import { describe, expect, it, vi } from "vitest"; + +import worker from "../index.js"; + +type StoredEntry = { value: string }; + +class MemoryKV { + readonly store = new Map(); + + async get(key: string): Promise { + return this.store.get(key)?.value ?? null; + } + + async put(key: string, value: string): Promise { + this.store.set(key, { value }); + } +} + +function asKV(kv: MemoryKV): KVNamespace { + return kv as unknown as KVNamespace; +} + +type WorkerBinding = { fetch: ReturnType }; + +function makeBinding(): WorkerBinding { + return { + fetch: vi.fn(async () => new Response("ok", { status: 200 })), + }; +} + +function buildEnv(opts: { + webhookOriginFlag?: string; + cloudWebWorker?: WorkerBinding; + webhookWorker?: WorkerBinding; +}) { + const routerConfig = new MemoryKV(); + if (opts.webhookOriginFlag !== undefined) { + routerConfig.store.set("WEBHOOK_ORIGIN", { value: opts.webhookOriginFlag }); + } + return { + CLOUD_APP_ORIGIN: "https://origin.test.invalid", + ROUTER_CONFIG: asKV(routerConfig), + CLOUD_WEB_WORKER: opts.cloudWebWorker, + WEBHOOK_WORKER: opts.webhookWorker, + } as unknown as Parameters[1]; +} + +function buildCtx(): ExecutionContext { + return { + waitUntil: () => undefined, + passThroughOnException: () => undefined, + } as unknown as ExecutionContext; +} + +describe("router webhook routing", () => { + it("routes /api/v1/webhooks/nango to the webhook-worker binding when WEBHOOK_ORIGIN=worker", async () => { + const cloudWebWorker = makeBinding(); + const webhookWorker = makeBinding(); + const env = buildEnv({ + webhookOriginFlag: "worker", + cloudWebWorker, + webhookWorker, + }); + + const request = new Request( + "https://origin.agentrelay.cloud/cloud/api/v1/webhooks/nango", + { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ type: "forward", from: "github-app-oauth" }), + }, + ); + + await worker.fetch(request, env, buildCtx()); + + expect(webhookWorker.fetch).toHaveBeenCalledOnce(); + expect(cloudWebWorker.fetch).not.toHaveBeenCalled(); + }); + + it("bypasses webhook-worker when request carries x-cloud-webhook-worker-forwarded header (loop break)", async () => { + const cloudWebWorker = makeBinding(); + const webhookWorker = makeBinding(); + const env = buildEnv({ + webhookOriginFlag: "worker", + cloudWebWorker, + webhookWorker, + }); + + const request = new Request( + "https://origin.agentrelay.cloud/cloud/api/v1/webhooks/nango", + { + method: "POST", + headers: { + "content-type": "application/json", + "x-cloud-webhook-worker-forwarded": "webhook-worker", + "x-cloud-webhook-worker-request-id": "req-loop-1", + }, + body: JSON.stringify({ type: "forward", from: "github-app-oauth" }), + }, + ); + + await worker.fetch(request, env, buildCtx()); + + expect(cloudWebWorker.fetch).toHaveBeenCalledOnce(); + expect(webhookWorker.fetch).not.toHaveBeenCalled(); + }); + + it("ignores the loop-break header for an unrelated value (don't accidentally widen bypass)", async () => { + const cloudWebWorker = makeBinding(); + const webhookWorker = makeBinding(); + const env = buildEnv({ + webhookOriginFlag: "worker", + cloudWebWorker, + webhookWorker, + }); + + const request = new Request( + "https://origin.agentrelay.cloud/cloud/api/v1/webhooks/nango", + { + method: "POST", + headers: { + "content-type": "application/json", + "x-cloud-webhook-worker-forwarded": "something-else", + }, + body: JSON.stringify({ type: "forward", from: "github-app-oauth" }), + }, + ); + + await worker.fetch(request, env, buildCtx()); + + expect(webhookWorker.fetch).toHaveBeenCalledOnce(); + expect(cloudWebWorker.fetch).not.toHaveBeenCalled(); + }); + + it("routes to cloud-web when WEBHOOK_ORIGIN flag is unset (regression: still works without flag)", async () => { + const cloudWebWorker = makeBinding(); + const webhookWorker = makeBinding(); + const env = buildEnv({ + cloudWebWorker, + webhookWorker, + }); + + const request = new Request( + "https://origin.agentrelay.cloud/cloud/api/v1/webhooks/nango", + { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ type: "forward", from: "github-app-oauth" }), + }, + ); + + await worker.fetch(request, env, buildCtx()); + + expect(cloudWebWorker.fetch).toHaveBeenCalledOnce(); + expect(webhookWorker.fetch).not.toHaveBeenCalled(); + }); +}); From 895fb683a55d1996e94ede2f478d73d4b733f2ad Mon Sep 17 00:00:00 2001 From: Khaliq Date: Wed, 27 May 2026 03:06:17 +0200 Subject: [PATCH 40/49] Repair preflight clone materialization --- router/README.md | 66 ---- router/index.ts | 438 --------------------- router/src/rate-limit.ts | 550 -------------------------- router/src/recorder.ts | 140 ------- router/src/redact.ts | 132 ------- router/test/rate-limit.test.ts | 588 ---------------------------- router/test/recorder.test.ts | 192 --------- router/test/redact.test.ts | 78 ---- router/test/webhook-routing.test.ts | 157 -------- router/tsconfig.json | 17 - 10 files changed, 2358 deletions(-) delete mode 100644 router/README.md delete mode 100644 router/index.ts delete mode 100644 router/src/rate-limit.ts delete mode 100644 router/src/recorder.ts delete mode 100644 router/src/redact.ts delete mode 100644 router/test/rate-limit.test.ts delete mode 100644 router/test/recorder.test.ts delete mode 100644 router/test/redact.test.ts delete mode 100644 router/test/webhook-routing.test.ts delete mode 100644 router/tsconfig.json diff --git a/router/README.md b/router/README.md deleted file mode 100644 index 98defc42..00000000 --- a/router/README.md +++ /dev/null @@ -1,66 +0,0 @@ -Route precedence: - -- `/cloud*` routes to the Agent Relay Cloud Next.js app. -- `/observer/file*` routes to the RelayFile observer app, expected to be built with `basePath: "/observer/file"`. -- `/observer*` routes to the Relaycast observer app. -- everything else falls back to the legacy proxy target. - -RelayFile observer deployment is wired from the production job in -`.github/workflows/deploy.yml`. The job runs -`scripts/deploy-file-observer-pages.sh`, which packs -`@relayfile/file-observer@latest`, builds it with -`FILE_OBSERVER_BASE_PATH=/observer/file`, and deploys the detected static output -to the Cloudflare Pages project `relayfile-file-observer`. The script creates -that Pages project with the configured production branch when it is missing, so -the first production deploy does not require manual Cloudflare setup. - -``` - ┌────────────────────────────────────┐ - │ @agentworkforce/cloud │ - │ │ - │/packages/router (Cloudflare worker)│ - └────────────────────────────────────┘ - - ┌──────────────────────────────────────┐ - │ agentrelay.com │ - └──────────────────────────────────────┘ - │ - │ - │ - All cloud routes │ Observer Routes - ┌───(/cloud/*, /cloud/api)───────────┼────(/observer*, /observer/file*)───────┐ - │ │ │ - │ │ │ - │ │ │ - ▼ │ ▼ -┌────────────────────────────────────┐ │ ┌────────────────────────────────────┐ -│ https://agentrelay.cloud │ │ │ https://relaycast.dev/observer │ -└────────────────────────────────────┘ │ └────────────────────────────────────┘ - │ - │ -┌────────────────────────────────────┐ All Other Routes ┌────────────────────────────────────┐ -│ @agentworkforce/cloud │ (/, /docs, /blog, /openclaw) │ @agentworkforce/relaycast │ -│ │ │ │ │ -│ /packages/web (NextJs App) │ │ │ /packages/observer-dashboard │ -│ │ │ │ /@relayfile/file-observer │ -└────────────────────────────────────┘ │ └────────────────────────────────────┘ - │ - │ - │ - │ - │ - │ - │ - │ - ▼ - ┌────────────────────────────────────┐ - │ https://origin.agentrelay.net │ - └────────────────────────────────────┘ - - - ┌────────────────────────────────────┐ - │ @agentworkforce/relay │ - │ │ - │ /web (NextJs App) │ - └────────────────────────────────────┘ -``` diff --git a/router/index.ts b/router/index.ts deleted file mode 100644 index df9cfde6..00000000 --- a/router/index.ts +++ /dev/null @@ -1,438 +0,0 @@ -import { maybeRecord, type RecorderEnv } from "./src/recorder.js"; -import { maybeRateLimit, type RateLimitEnv } from "./src/rate-limit.js"; - -interface Env { - CLOUD_APP_ORIGIN: string; - CLOUD_WEB_WORKER?: { - fetch(request: Request): Promise; - }; - FILE_OBSERVER_ORIGIN?: string; - TRAFFIC_RECORDER?: RecorderEnv["TRAFFIC_RECORDER"]; - ROUTER_CONFIG?: RecorderEnv["ROUTER_CONFIG"]; - RATE_LIMIT_COUNTERS?: RateLimitEnv["RATE_LIMIT_COUNTERS"]; - WEBHOOK_WORKER?: { - fetch(request: Request): Promise; - }; - WEBHOOK_WORKER_ORIGIN?: string; -} - -function hasRecorderEnv(env: Env): env is Env & RecorderEnv { - return Boolean(env.TRAFFIC_RECORDER && env.ROUTER_CONFIG); -} - -const FALLBACK_PROXY_ORIGIN = "https://origin.agentrelay.net"; -const OBSERVER_ORIGIN = "https://observer.relaycast.dev"; -const DEFAULT_FILE_OBSERVER_ORIGIN = "https://relayfile-file-observer.pages.dev"; -const PRIMARY_HOST = "agentrelay.com"; -const FILE_OBSERVER_PATH_PREFIX = "/observer/file"; -const OBSERVER_PATH_PREFIX = "/observer"; -const CLOUD_PATH_PREFIX = "/cloud"; -const WEBHOOK_ORIGIN_FLAG_KEY = "WEBHOOK_ORIGIN"; - -// Header set by webhook-worker's queue consumer -// (`packages/webhook-worker/src/queue-consumer.ts`'s `buildForwardHeaders`) on -// outbound forwards to cloud-web's `/api/v1/webhooks/nango`. When the -// `WEBHOOK_ORIGIN` flag is `"worker"`, the router would otherwise redirect -// every `/api/v1/webhooks/nango` POST back to webhook-worker — including -// webhook-worker's own outbound forwards, producing an infinite redelivery -// loop bounded only by `maxRetries`. Honouring this header lets webhook-worker -// reach cloud-web's `routeForwardEvent` → `handleGitHubForward` path which is -// the actual destination of the forward. -const WEBHOOK_WORKER_FORWARDED_HEADER = "x-cloud-webhook-worker-forwarded"; -const WEBHOOK_WORKER_FORWARDED_VALUE = "webhook-worker"; -let loggedPhase5aLambdaEliminated = false; - -// Exact paths the webhook worker handles. Other sub-paths under -// /api/v1/webhooks (notably /api/v1/webhooks/composio/connect/callback, an -// OAuth callback served by Next.js) must continue to route to the Lambda even -// when WEBHOOK_ORIGIN=worker, otherwise they 404 against the Worker. -const WEBHOOK_WORKER_PATHS = new Set([ - "/api/v1/webhooks/composio", - "/api/v1/webhooks/github", - "/api/v1/webhooks/hookdeck", - "/api/v1/webhooks/nango", -]); -const NANGO_WEBHOOK_WORKER_PATH = "/api/v1/webhooks/nango"; - -function isPathWithinPrefix(pathname: string, prefix: string): boolean { - return pathname === prefix || pathname.startsWith(`${prefix}/`); -} - -function isObserverPath(pathname: string): boolean { - return isPathWithinPrefix(pathname, OBSERVER_PATH_PREFIX); -} - -function isFileObserverPath(pathname: string): boolean { - return isPathWithinPrefix(pathname, FILE_OBSERVER_PATH_PREFIX); -} - -function isPrimaryFileObserverPath(hostname: string, pathname: string): boolean { - return hostname === PRIMARY_HOST && isFileObserverPath(pathname); -} - -function isCloudPath(pathname: string): boolean { - return isPathWithinPrefix(pathname, CLOUD_PATH_PREFIX); -} - -// True only for the exact paths the webhook worker knows how to handle. Used -// to gate worker forwarding so unrelated routes under /api/v1/webhooks/* (e.g. -// the Composio OAuth callback) still reach the Lambda. -export function isWebhookWorkerPath(pathname: string): boolean { - return WEBHOOK_WORKER_PATHS.has(stripPathPrefix(pathname, CLOUD_PATH_PREFIX)); -} - -function isNangoWebhookWorkerPath(pathname: string): boolean { - return stripPathPrefix(pathname, CLOUD_PATH_PREFIX) === NANGO_WEBHOOK_WORKER_PATH; -} - -function stripPathPrefix(pathname: string, prefix: string): string { - if (pathname === prefix) { - return "/"; - } - - if (pathname.startsWith(`${prefix}/`)) { - return pathname.slice(prefix.length); - } - - return pathname; -} - -function addPathPrefix(pathname: string, prefix: string): string { - if (!prefix) { - return pathname; - } - - if (pathname === "/") { - return prefix; - } - - if (pathname === prefix || pathname.startsWith(`${prefix}/`) || pathname.startsWith(`${prefix}?`)) { - return pathname; - } - - return `${prefix}${pathname}`; -} - -function hostnameFromHost(host: string, protocol: string): string { - try { - return new URL(`${protocol}//${host}`).hostname; - } catch { - return host.split(":")[0] ?? host; - } -} - -function isPublicFileObserverLocation(hostname: string, pathname: string): boolean { - return isPrimaryFileObserverPath(hostname, pathname); -} - -export function getUpstreamPath(hostname: string, pathname: string): string { - if (isPrimaryFileObserverPath(hostname, pathname)) { - return stripPathPrefix(pathname, FILE_OBSERVER_PATH_PREFIX); - } - - return pathname; -} - -export function getMountPrefix(hostname: string, pathname: string): string { - if (isCloudPath(pathname)) { - return CLOUD_PATH_PREFIX; - } - - if (isPrimaryFileObserverPath(hostname, pathname)) { - return FILE_OBSERVER_PATH_PREFIX; - } - - return ""; -} - -export function rewriteLocation( - location: string, - originUrl: URL, - requestHost: string, - requestProtocol: string, - mountPrefix = "", -): string { - if (!location) { - return location; - } - - try { - const absolute = new URL(location); - if (isPublicFileObserverLocation(absolute.hostname, absolute.pathname)) { - return location; - } - - if (absolute.hostname !== originUrl.hostname) { - return location; - } - - absolute.hostname = requestHost; - absolute.port = ""; - absolute.protocol = requestProtocol; - absolute.pathname = addPathPrefix(absolute.pathname, mountPrefix); - return absolute.toString(); - } catch { - if (location.startsWith("/")) { - try { - const requestHostname = hostnameFromHost(requestHost, requestProtocol); - const locationUrl = new URL(location, `${requestProtocol}//${requestHost}`); - if (isPublicFileObserverLocation(requestHostname, locationUrl.pathname)) { - return location; - } - } catch { - // Fall through to the normal mount-prefix rewrite. - } - - return addPathPrefix(location, mountPrefix); - } - - return location; - } -} - -export function getOrigin(hostname: string, pathname: string, env: Env): string { - // /cloud* defaults to the Next.js cloud app regardless of host. Requests only - // reach this fallback when the cloud-web Worker service binding is absent. - if (isCloudPath(pathname)) { - return env.CLOUD_APP_ORIGIN; - } - - // The production agentrelay.com apex is a split router: - // 1. /observer/file* goes to the RelayFile file observer app - // 2. /observer* stays on the Relaycast observer app - // 3. everything else falls back to the relay web origin - if (hostname === PRIMARY_HOST) { - if (isPrimaryFileObserverPath(hostname, pathname)) { - return env.FILE_OBSERVER_ORIGIN ?? DEFAULT_FILE_OBSERVER_ORIGIN; - } - - if (isObserverPath(pathname)) { - return OBSERVER_ORIGIN; - } - } - - return FALLBACK_PROXY_ORIGIN; -} - -async function shouldUseCloudWebWorker( - pathname: string, - request: Request, - env: Env, -): Promise { - if (!isCloudPath(pathname)) { - return false; - } - - if (await shouldUseWebhookWorker(pathname, request, env)) { - return false; - } - - return true; -} - -function logPhase5aLambdaEliminatedOnce(): void { - if (loggedPhase5aLambdaEliminated) { - return; - } - - loggedPhase5aLambdaEliminated = true; - console.log(JSON.stringify({ router_phase: "5a_lambda_eliminated" })); -} - -export async function readWebhookOriginFlag(env: Env): Promise { - try { - return (await env.ROUTER_CONFIG?.get(WEBHOOK_ORIGIN_FLAG_KEY)) ?? null; - } catch { - return null; - } -} - -export async function shouldUseNangoWebhookWorkerRoute( - pathname: string, - env: Env, -): Promise { - if (!isNangoWebhookWorkerPath(pathname)) { - return false; - } - - const configured = await readWebhookOriginFlag(env); - return configured?.trim().toLowerCase() === "worker"; -} - -async function shouldUseWebhookWorker( - pathname: string, - request: Request, - env: Env, -): Promise { - // Break the redelivery loop: webhook-worker's queue consumer forwards the - // raw envelope back to the same `/api/v1/webhooks/nango` route on - // `origin.agentrelay.cloud` so cloud-web's `handleGitHubForward` can run. - // Without this header check, the router catches that forward and redirects - // it back to webhook-worker, which re-enqueues, ad infinitum (or until - // `maxRetries`). - if (request.headers.get(WEBHOOK_WORKER_FORWARDED_HEADER) === WEBHOOK_WORKER_FORWARDED_VALUE) { - return false; - } - return shouldUseNangoWebhookWorkerRoute(pathname, env); -} - -function buildWebhookWorkerRequest( - request: Request, - requestUrl: URL, - workerOrigin?: string, -): Request { - const targetUrl = new URL(requestUrl.toString()); - targetUrl.pathname = stripPathPrefix(targetUrl.pathname, CLOUD_PATH_PREFIX); - - if (workerOrigin) { - const originUrl = new URL(workerOrigin); - targetUrl.protocol = originUrl.protocol; - targetUrl.hostname = originUrl.hostname; - targetUrl.port = originUrl.port; - } - - const init: RequestInit & { duplex?: "half" } = { - method: request.method, - headers: request.headers, - body: request.body, - redirect: "manual", - duplex: "half", - }; - - return new Request(targetUrl.toString(), init); -} - -export default { - async fetch(request: Request, env: Env, ctx: ExecutionContext): Promise { - const url = new URL(request.url); - - // Per-key rate limiting runs BEFORE any worker routing so a runaway - // workspace gets bounded everywhere — including webhook ingress and - // /cloud* traffic. The bypass list inside maybeRateLimit exempts - // health and observer paths. See packages/router/src/rate-limit.ts - // and docs/security/rate-limiting.md. - const rateLimited = await maybeRateLimit(request, env); - if (rateLimited) { - return rateLimited; - } - - // Clone the request up front so any branch that returns early (cloud-web - // Worker service binding, webhook Worker service binding, etc.) can - // still feed the recorder the original payload. Without this clone, the - // /cloud Worker path bypasses the recorder entirely and the replay - // harness has no corpus to prove equivalence during Phase 4 cutover. - // See Codex P2.6 on bundle PR #647. - const recorderEnv = hasRecorderEnv(env) ? env : null; - const recorderRequestClone = recorderEnv - ? (request.clone() as unknown as Request) - : null; - - if (await shouldUseCloudWebWorker(url.pathname, request, env)) { - logPhase5aLambdaEliminatedOnce(); - if (!env.CLOUD_WEB_WORKER) { - return new Response( - JSON.stringify({ error: "cloud web worker binding unavailable" }), - { - status: 503, - headers: { "content-type": "application/json" }, - }, - ); - } - - const workerResponse = await env.CLOUD_WEB_WORKER.fetch(request); - if (recorderRequestClone && recorderEnv) { - ctx.waitUntil( - maybeRecord(recorderRequestClone, workerResponse.clone(), recorderEnv, ctx), - ); - } - return workerResponse; - } - - if (await shouldUseWebhookWorker(url.pathname, request, env)) { - const webhookWorkerRequest = buildWebhookWorkerRequest(request, url); - if (env.WEBHOOK_WORKER) { - const workerResponse = await env.WEBHOOK_WORKER.fetch(webhookWorkerRequest); - if (recorderRequestClone && recorderEnv) { - ctx.waitUntil( - maybeRecord(recorderRequestClone, workerResponse.clone(), recorderEnv, ctx), - ); - } - return workerResponse; - } - - const workerOrigin = env.WEBHOOK_WORKER_ORIGIN?.trim(); - if (workerOrigin) { - const originResponse = await globalThis.fetch( - buildWebhookWorkerRequest(request, url, workerOrigin), - ); - if (recorderRequestClone && recorderEnv) { - ctx.waitUntil( - maybeRecord(recorderRequestClone, originResponse.clone(), recorderEnv, ctx), - ); - } - return originResponse; - } - } - - const requestHost = request.headers.get("Host") || url.hostname; - const originUrl = new URL(getOrigin(url.hostname, url.pathname, env)); - const mountPrefix = getMountPrefix(url.hostname, url.pathname); - - url.pathname = getUpstreamPath(url.hostname, url.pathname); - url.hostname = originUrl.hostname; - url.port = ""; - url.protocol = "https:"; - - const headers = new Headers(request.headers); - headers.set("X-Forwarded-Host", requestHost); - headers.set("X-Original-Host", requestHost); - headers.set("X-Forwarded-Proto", "https"); - if (mountPrefix) { - headers.set("X-Forwarded-Prefix", mountPrefix); - } - - // Reuse the clone made at the top of fetch() for the recorder. The - // earlier clone covers all branches; making a second one here would - // be wasted bytes on every request and would double-record on the - // Lambda origin path. - const recordingRequest = recorderRequestClone; - const subRequest = new Request(url.toString(), { - method: request.method, - headers, - body: request.body, - redirect: "manual", - }); - - try { - // Use `globalThis.fetch` rather than a bare `fetch` identifier: Cloudflare - // Workers can hoist bare `fetch` off `globalThis` and throw - // `TypeError: Illegal invocation`. See sage `.claude/rules/workers-fetch.md`. - const upstreamResponse = await globalThis.fetch(subRequest); - const responseHeaders = new Headers(upstreamResponse.headers); - - const location = responseHeaders.get("Location"); - if (location) { - responseHeaders.set( - "Location", - rewriteLocation(location, originUrl, requestHost, "https:", mountPrefix), - ); - } - - const response = new Response(upstreamResponse.body, { - status: upstreamResponse.status, - statusText: upstreamResponse.statusText, - headers: responseHeaders, - }); - - if (recordingRequest && hasRecorderEnv(env)) { - ctx.waitUntil(maybeRecord(recordingRequest, response.clone(), env, ctx)); - } - - return response; - } catch (error) { - return new Response(JSON.stringify({ error: (error as Error).message }), { - status: 500, - }); - } - }, -}; diff --git a/router/src/rate-limit.ts b/router/src/rate-limit.ts deleted file mode 100644 index d7c6c734..00000000 --- a/router/src/rate-limit.ts +++ /dev/null @@ -1,550 +0,0 @@ -// Per-key request rate limiting for the Cloudflare edge gateways. -// -// Two Workers front the platform: the `AgentRelayRouter` worker -// (agentrelay.com / cloud router) and the `RelayfileApi` worker -// (api.relayfile.dev). A misbehaving client today (e.g. a relayfile -// mount stuck in a polling loop) can hammer either gateway with -// dozens of requests per second; the upstream Lambda has a 10-slot -// concurrency cap, so once a single key saturates the gateway every -// other user starts seeing 429s. The rate limiter in this module -// runs at the edge BEFORE the request reaches the Lambda, so a -// runaway key is contained to its own per-workspace bucket and -// doesn't deplete shared capacity. -// -// Design choices (intentional trade-offs): -// -// * Counter storage is Cloudflare KV (a `RATE_LIMIT_COUNTERS` -// namespace bound to both Workers). KV is eventually consistent -// across edges, so the effective ceiling at high traffic can be -// ~2-3x the configured per-minute limit. For the 60 req/min -// target that means ~180 req/min worst case — still 100x lower -// than the runaway today and good enough for the runaway-client -// class of bug. Sub-second precision and global atomicity would -// require a Durable Object, which we may add later if KV drift -// bites — but the simpler path ships first. -// -// * Buckets are minute-aligned (epoch-minute integer). Each -// `(limitKey, bucket)` pair is one KV key with a 120-second TTL -// so the previous and current minute coexist briefly during the -// turn. Reads are racy w.r.t. writes; that's accepted (see above). -// -// * Limits are read from `ROUTER_CONFIG` KV at request time, so -// operators can dial them up/down or kill-switch the limiter -// entirely via `wrangler kv:key put` without redeploying. -// -// * Defaults: 60 req/min per workspace-or-user-or-ip key, 1000 -// req/min global ceiling (a separate safety-net counter shared -// across all keys). Both defaults are intentionally conservative -// given the actual runaway pattern observed (30+ req/sec from a -// single key). -// -// * Bypass list mirrors the recorder's deny list plus a few -// observability paths — these must never be rate limited because -// they're either health probes, browser asset requests, or -// observer/webhook ingress that already has its own backpressure. - -const ONE_MINUTE_SECONDS = 60; -// KV TTL on counter entries. Must be >= 60 to cover a full bucket -// window even if the request arrives at the very start of the -// minute. We pick 120s so the previous bucket is still readable -// briefly during the turnover, which means a borderline client that -// straddles the boundary still sees its prior count and can't double -// up by ~timing requests across the second. -const COUNTER_TTL_SECONDS = 120; - -// Default config — operators tune via `wrangler kv:key put` on the -// `ROUTER_CONFIG` namespace; we re-read every request. -const DEFAULT_PER_KEY_PER_MIN = 60; -const DEFAULT_GLOBAL_PER_MIN = 1000; - -// KV config keys we read from `ROUTER_CONFIG`. Centralized here so -// the docs and the infra seed step refer to the same names. -export const RATE_LIMIT_CONFIG_KEYS = { - PER_KEY_PER_MIN: "RATE_LIMIT_PER_WORKSPACE_PER_MIN", - GLOBAL_PER_MIN: "RATE_LIMIT_GLOBAL_PER_MIN", - DISABLED: "RATE_LIMIT_DISABLED", - // Comma-separated path prefixes added to the bypass list at - // runtime. Static `BYPASS_PREFIXES` below is the safe core; this - // key lets an operator whitelist a new path (e.g. a freshly-added - // health probe) without redeploying. Values are matched the same - // way as static prefixes — exact match or `${prefix}/...` startswith. - // Example KV value: `/api/k8s-readiness,/internal/uptime` - BYPASS_PREFIXES_EXTRA: "RATE_LIMIT_BYPASS_PREFIXES_EXTRA", -} as const; - -// Paths that must NEVER be rate limited. Health probes, static -// assets, and observability endpoints. The router/relayfile-api -// share this list — both Workers see these paths even though only -// one of them actually serves them, and accidentally rate-limiting -// a health probe would self-DoS the upstream monitoring. -const BYPASS_PREFIXES: readonly string[] = [ - "/api/health", - "/health", - "/observer", - "/_next/static", - "/favicon.ico", - // Webhook ingress has no auth token and no workspace path, so it - // would bucket by IP and trip the 60/min cap with even modest - // burst traffic from a single provider (GitHub, Composio, Nango, - // Hookdeck). The webhook worker already has queue-backed - // backpressure, so rate-limiting at the edge would only cause us - // to drop legitimate events. See Codex P1.4 on bundle PR #647. - "/api/v1/webhooks", -]; - -export interface RateLimitEnv { - // Counter storage — shared across both gateway Workers so a - // workspace's limit isn't accidentally doubled by hitting both. - RATE_LIMIT_COUNTERS?: KVNamespace; - // Configuration knobs — same KV namespace the router already uses - // for other flags (e.g. WEBHOOK_ORIGIN). Optional so unit tests - // can run without it. - ROUTER_CONFIG?: KVNamespace; -} - -export interface RateLimitDecision { - allowed: boolean; - retryAfterSec?: number; - counter: number; - limit: number; - // The key the decision applies to — exposed so callers can put it - // in the 429 body (helps operators correlate gateway logs with the - // offending workspace/user/IP). - key: string; -} - -// Pure path helper so we can unit test it without a real Request. -// Checks ONLY the static hardcoded prefix list — does not consult KV. -// Use `isBypassPathDynamic` when env is available and you want the -// runtime-extensible bypass list to apply. -export function isBypassPath(pathname: string): boolean { - return matchesAnyPrefix(pathname, BYPASS_PREFIXES); -} - -function matchesAnyPrefix( - pathname: string, - prefixes: readonly string[], -): boolean { - for (const prefix of prefixes) { - if (pathname === prefix || pathname.startsWith(`${prefix}/`)) { - return true; - } - } - return false; -} - -// Module-scoped cache for the KV-resolved extra bypass prefixes. -// Refreshed at most once per `BYPASS_EXTRAS_TTL_MS` per Worker -// isolate. Operators tuning the KV value see propagation within -// roughly `BYPASS_EXTRAS_TTL_MS` plus the time CF's eventually- -// consistent KV takes to fan out to edges (typically <60s). This -// is the same staleness window that applies to the existing numeric -// config keys (`RATE_LIMIT_PER_WORKSPACE_PER_MIN`, etc.) — keeping -// the model consistent across all runtime knobs. -const BYPASS_EXTRAS_TTL_MS = 60_000; -let cachedBypassExtras: readonly string[] = []; -let cachedBypassExtrasFetchedAt = 0; - -/** - * Test-only: reset the bypass-extras cache between test cases. - * Production code never calls this. - */ -export function __resetBypassExtrasCacheForTests(): void { - cachedBypassExtras = []; - cachedBypassExtrasFetchedAt = 0; -} - -async function getBypassExtras( - env: RateLimitEnv, - now: number, -): Promise { - if (!env.ROUTER_CONFIG) { - return []; - } - if (now - cachedBypassExtrasFetchedAt < BYPASS_EXTRAS_TTL_MS) { - return cachedBypassExtras; - } - try { - const raw = await env.ROUTER_CONFIG.get( - RATE_LIMIT_CONFIG_KEYS.BYPASS_PREFIXES_EXTRA, - ); - cachedBypassExtras = parseBypassExtras(raw); - cachedBypassExtrasFetchedAt = now; - } catch { - // ROUTER_CONFIG read failures should never break the data plane. - // Keep the last known list (or the empty initial) and try again - // after the TTL. - cachedBypassExtrasFetchedAt = now; - } - return cachedBypassExtras; -} - -/** - * Exported for unit tests. Parses the comma-separated KV value into a - * deduped, trimmed list of prefixes, filtering out anything that - * doesn't look like an absolute path (must start with `/`). Defensive - * parsing — bad operator input shouldn't be able to break the bypass - * check. - */ -export function parseBypassExtras(raw: string | null): readonly string[] { - if (!raw) { - return []; - } - const out: string[] = []; - const seen = new Set(); - for (const entry of raw.split(",")) { - const trimmed = entry.trim(); - if (trimmed.length === 0 || !trimmed.startsWith("/")) { - continue; - } - if (seen.has(trimmed)) { - continue; - } - seen.add(trimmed); - out.push(trimmed); - } - return out; -} - -/** - * Returns true if the path should bypass rate limiting — checks both - * the static `BYPASS_PREFIXES` list and the KV-runtime-tunable - * `RATE_LIMIT_BYPASS_PREFIXES_EXTRA` list. The static check is fast - * (no KV read); the KV list is cached per-isolate for - * `BYPASS_EXTRAS_TTL_MS`. Used by `maybeRateLimit` internally. - * - * @param pathname - the URL pathname to check - * @param env - rate limit env containing the KV bindings - * @param now - wall-clock ms; injectable for tests - */ -export async function isBypassPathDynamic( - pathname: string, - env: RateLimitEnv, - now: number = Date.now(), -): Promise { - if (isBypassPath(pathname)) { - return true; - } - const extras = await getBypassExtras(env, now); - return matchesAnyPrefix(pathname, extras); -} - -// Extract a workspace ID from any of the path shapes both gateways -// see. Matches the literal "workspaces/" segment (Hono routes both -// `/v1/workspaces/:id/...` on the relayfile worker and -// `/cloud/api/v1/workspaces/:id/...` on the cloud router). -function extractWorkspaceId(pathname: string): string | null { - const match = pathname.match(/\/workspaces\/([a-zA-Z0-9_-]+)/); - return match?.[1] ?? null; -} - -// Cheap, non-cryptographic hash for opaque bearer tokens so we can -// shard rate counters by token without ever storing the token. We -// don't need collision resistance against an adversary — we just -// want different tokens to map to different buckets. FNV-1a 32-bit -// fits in a few lines and is fine for this. -function fnv1a32Hex(input: string): string { - let hash = 0x811c9dc5; - for (let i = 0; i < input.length; i++) { - hash ^= input.charCodeAt(i); - // 32-bit multiply, kept unsigned via `>>> 0`. - hash = Math.imul(hash, 0x01000193) >>> 0; - } - return hash.toString(16).padStart(8, "0"); -} - -// The session cookie name used by the web app. Must match -// `SESSION_COOKIE_NAME` in `packages/core/src/session/jwt.ts`. -// Defined inline here to keep the rate-limit module self-contained — -// the router package has no dependency on the web package. -const SESSION_COOKIE_NAME = "agent_relay_session"; - -// Parse a single named cookie out of a Cookie header value. Returns -// null when the cookie is absent or has an empty value. Handles both -// quoted and unquoted values; uses the first match when the name -// appears more than once (which is invalid per RFC 6265 but occurs -// in the wild). -function parseCookieValue(cookieHeader: string, name: string): string | null { - // Regex: look for `; name=` or start-of-string `name=`, then capture - // the value up to the next `;` or end-of-string. - const pattern = new RegExp( - `(?:^|;)\\s*${name}\\s*=\\s*"?([^";]*)"?(?:;|$)`, - ); - const match = cookieHeader.match(pattern); - const value = match?.[1]?.trim() ?? null; - return value && value.length > 0 ? value : null; -} - -// Build a stable limit key from the incoming request. Pure function — -// no network calls, no JWKS verify (too expensive for the hot path). -// Order of precedence reflects which signal is most specific: -// -// 1. workspace ID from the path (covers the actual runaway shape) -// 2. authenticated user via bearer token hash (covers non-workspace -// authenticated traffic — e.g. /api/v1/me) -// 3. session cookie for cookie-authenticated /cloud* traffic. Without -// this, cookie-auth requests have neither a workspace ID nor an -// auth header and collapse to IP buckets — conflating all users -// that share a NAT IP into one rate limit bucket. We hash the -// cookie value with FNV-1a 32 so the rate counter key never -// contains the raw session token. -// 4. CF-connecting-IP (anonymous fallback; X-Forwarded-For untrusted -// because anyone can set it, but `cf-connecting-ip` is set by -// Cloudflare edge and can't be spoofed by the client) -// 5. "anon" — last-resort bucket so a misbehaving anon flood is -// still bounded, just bundled together (acceptable: this is the -// least-trusted traffic anyway) -export function extractLimitKey(request: Request): string { - const url = new URL(request.url); - const workspaceId = extractWorkspaceId(url.pathname); - if (workspaceId) { - return `ws:${workspaceId}`; - } - - const auth = request.headers.get("authorization"); - if (auth) { - // Strip the scheme prefix ("Bearer ", "Basic ", etc.) before - // hashing so the same token doesn't bucket differently when a - // client toggles capitalization or omits the scheme. - const token = auth.replace(/^[A-Za-z]+\s+/, "").trim(); - if (token.length > 0) { - return `user:${fnv1a32Hex(token)}`; - } - } - - const cookieHeader = request.headers.get("cookie"); - if (cookieHeader) { - const sessionToken = parseCookieValue(cookieHeader, SESSION_COOKIE_NAME); - if (sessionToken) { - return `session:${fnv1a32Hex(sessionToken)}`; - } - } - - const ip = - request.headers.get("cf-connecting-ip") ?? - request.headers.get("x-forwarded-for")?.split(",")[0]?.trim() ?? - null; - if (ip) { - return `ip:${ip}`; - } - - return "anon"; -} - -async function readNumberConfig( - kv: KVNamespace | undefined, - configKey: string, - fallback: number, -): Promise { - if (!kv) { - return fallback; - } - try { - const raw = await kv.get(configKey); - if (raw == null) { - return fallback; - } - const parsed = Number(raw); - if (!Number.isFinite(parsed) || parsed <= 0) { - return fallback; - } - return parsed; - } catch { - // ROUTER_CONFIG read failures should never break the data plane — - // we'd rather rate-limit at the default than 500 every request. - return fallback; - } -} - -async function readBooleanConfig( - kv: KVNamespace | undefined, - configKey: string, -): Promise { - if (!kv) { - return false; - } - try { - const raw = await kv.get(configKey); - return raw?.trim().toLowerCase() === "true"; - } catch { - return false; - } -} - -// Increment-and-read with KV. Not atomic — two concurrent requests at -// the same edge can both read N, both write N+1, and we lose a count. -// Accepted per the module-level note. The `now` parameter is injected -// for testability. -async function bumpCounter( - kv: KVNamespace, - bucketKey: string, - now: number, -): Promise { - let prior = 0; - try { - const raw = await kv.get(bucketKey); - if (raw != null) { - const parsed = Number.parseInt(raw, 10); - if (Number.isFinite(parsed) && parsed >= 0) { - prior = parsed; - } - } - } catch { - // Counter-store read failure — treat as zero and continue. - // Failing closed (blocking the request) on a transient KV error - // would amplify a single CF KV hiccup into a customer outage. - prior = 0; - } - - const next = prior + 1; - try { - await kv.put(bucketKey, String(next), { - expirationTtl: COUNTER_TTL_SECONDS, - }); - } catch { - // Same reasoning — log via the surrounding fetch handler if - // observability is needed, but never break the request. - } - void now; // reserved for future jittered TTLs; keeps the signature stable - return next; -} - -function bucketForNow(now: number): number { - return Math.floor(now / 1000 / ONE_MINUTE_SECONDS); -} - -function secondsUntilNextBucket(now: number): number { - const elapsedInMinute = Math.floor(now / 1000) % ONE_MINUTE_SECONDS; - return Math.max(1, ONE_MINUTE_SECONDS - elapsedInMinute); -} - -export interface CheckRateLimitOptions { - // Injected for deterministic tests; defaults to Date.now() in prod. - now?: () => number; -} - -export async function checkRateLimit( - key: string, - env: RateLimitEnv, - options: CheckRateLimitOptions = {}, -): Promise { - const nowFn = options.now ?? (() => Date.now()); - const now = nowFn(); - - const [disabled, perKeyLimit, globalLimit] = await Promise.all([ - readBooleanConfig(env.ROUTER_CONFIG, RATE_LIMIT_CONFIG_KEYS.DISABLED), - readNumberConfig( - env.ROUTER_CONFIG, - RATE_LIMIT_CONFIG_KEYS.PER_KEY_PER_MIN, - DEFAULT_PER_KEY_PER_MIN, - ), - readNumberConfig( - env.ROUTER_CONFIG, - RATE_LIMIT_CONFIG_KEYS.GLOBAL_PER_MIN, - DEFAULT_GLOBAL_PER_MIN, - ), - ]); - - if (disabled || !env.RATE_LIMIT_COUNTERS) { - return { allowed: true, counter: 0, limit: perKeyLimit, key }; - } - - const bucket = bucketForNow(now); - const perKeyBucket = `rl:${key}:${bucket}`; - const globalBucket = `rl:__global__:${bucket}`; - - // Sequential reads/writes — KV doesn't support transactions and - // we want the global counter to include this request even if the - // per-key one rejects. Order is intentional: per-key first so a - // single runaway key doesn't get to skew the global ceiling by - // also incrementing the global bucket *before* its per-key check - // would have rejected it. Net effect: a blocked request still - // contributes 1 to per-key but NOT to global. - const perKeyCount = await bumpCounter( - env.RATE_LIMIT_COUNTERS, - perKeyBucket, - now, - ); - - if (perKeyCount > perKeyLimit) { - return { - allowed: false, - retryAfterSec: secondsUntilNextBucket(now), - counter: perKeyCount, - limit: perKeyLimit, - key, - }; - } - - const globalCount = await bumpCounter( - env.RATE_LIMIT_COUNTERS, - globalBucket, - now, - ); - - if (globalCount > globalLimit) { - return { - allowed: false, - retryAfterSec: secondsUntilNextBucket(now), - counter: globalCount, - limit: globalLimit, - key: "__global__", - }; - } - - return { - allowed: true, - counter: perKeyCount, - limit: perKeyLimit, - key, - }; -} - -// Builds the 429 response. Centralized so both Workers return the -// same shape; clients (relayfile mount, CLI) can rely on it. -export function rateLimitResponse( - retryAfterSec: number, - key: string, -): Response { - const body = JSON.stringify({ - error: "rate_limited", - retryAfter: retryAfterSec, - limitKey: key, - }); - return new Response(body, { - status: 429, - headers: { - "content-type": "application/json", - "retry-after": String(retryAfterSec), - "x-ratelimit-key": key, - }, - }); -} - -// Convenience wrapper that combines bypass check, key extraction, -// the actual limit check, and the 429 response. Both Workers call -// this at the very top of their fetch handler. -// -// Returns: -// * `null` — request is allowed; continue to normal handling -// * `Response` — return this 429 immediately -export async function maybeRateLimit( - request: Request, - env: RateLimitEnv, - options: CheckRateLimitOptions = {}, -): Promise { - const url = new URL(request.url); - // Check both the static `BYPASS_PREFIXES` and the KV-runtime-tunable - // extras (cached 60s per isolate). Static check is fast; KV cache - // means at most one KV read per minute per isolate per Worker. - if (await isBypassPathDynamic(url.pathname, env, options.now?.() ?? Date.now())) { - return null; - } - - const key = extractLimitKey(request); - const decision = await checkRateLimit(key, env, options); - if (decision.allowed) { - return null; - } - return rateLimitResponse(decision.retryAfterSec ?? 60, decision.key); -} diff --git a/router/src/recorder.ts b/router/src/recorder.ts deleted file mode 100644 index be92f1e7..00000000 --- a/router/src/recorder.ts +++ /dev/null @@ -1,140 +0,0 @@ -import { redactBody, redactHeaders, shouldRecord } from "./redact.js"; - -const DEFAULT_SAMPLE_RATE = 100; -const MAX_BODY_BYTES = 256 * 1024; -const OVERSIZE_PREFIX = "[OVERSIZE:"; -const textEncoder = new TextEncoder(); - -export interface RecorderEnv { - TRAFFIC_RECORDER: R2Bucket; - ROUTER_CONFIG: KVNamespace; -} - -type RecorderContext = Pick; -type BodyReader = { - body: ReadableStream | null; - bodyUsed: boolean; - headers: Headers; - text(): Promise; -}; - -type TrafficRecord = { - ts: string; - method: string; - path: string; - query: string; - request_headers: Record; - request_body: string | null; - response_status: number; - response_headers: Record; - response_body: string | null; - request_id: string; -}; - -function normalizeSampleRate(value: string | null): number { - const parsed = Number(value); - if (!Number.isFinite(parsed)) { - return DEFAULT_SAMPLE_RATE; - } - - return Math.min(100, Math.max(0, parsed)); -} - -function makeOversizePlaceholder(size: number): string { - return `${OVERSIZE_PREFIX}${size}]`; -} - -async function readBody( - payload: BodyReader | null, - path: string, - contentType: string | null, -): Promise { - if (!payload || payload.body == null || payload.bodyUsed) { - return null; - } - - const body = await payload.text(); - const redactedBody = redactBody(path, body, contentType); - if (redactedBody == null) { - return null; - } - - const bodySize = textEncoder.encode(redactedBody).length; - if (bodySize > MAX_BODY_BYTES) { - return makeOversizePlaceholder(bodySize); - } - - return redactedBody; -} - -function padDatePart(value: number): string { - return String(value).padStart(2, "0"); -} - -function buildRecordKey(timestamp: Date, requestId: string): string { - const year = timestamp.getUTCFullYear(); - const month = padDatePart(timestamp.getUTCMonth() + 1); - const day = padDatePart(timestamp.getUTCDate()); - const hour = padDatePart(timestamp.getUTCHours()); - return `corpus/${year}/${month}/${day}/${hour}/${requestId}.ndjson`; -} - -function getRequestId(request: Request): string { - return ( - request.headers.get("x-request-id") ?? - request.headers.get("cf-ray") ?? - request.headers.get("traceparent") ?? - crypto.randomUUID() - ); -} - -export async function maybeRecord( - request: Request, - response: Response, - env: RecorderEnv, - ctx: RecorderContext, -): Promise { - try { - const url = new URL(request.url); - if (!shouldRecord(url.pathname)) { - return; - } - - const sampleRate = normalizeSampleRate(await env.ROUTER_CONFIG.get("RECORDER_SAMPLE_RATE")); - if (Math.random() * 100 >= sampleRate) { - return; - } - - const requestClone = request.bodyUsed ? null : request.clone(); - const responseClone = response.bodyUsed ? null : response.clone(); - const timestamp = new Date(); - const requestId = getRequestId(request); - - const record: TrafficRecord = { - ts: timestamp.toISOString(), - method: request.method, - path: url.pathname, - query: url.search.startsWith("?") ? url.search.slice(1) : url.search, - request_headers: redactHeaders(request.headers), - request_body: await readBody( - requestClone, - url.pathname, - requestClone?.headers.get("content-type") ?? null, - ), - response_status: response.status, - response_headers: redactHeaders(response.headers), - response_body: await readBody( - responseClone, - url.pathname, - responseClone?.headers.get("content-type") ?? null, - ), - request_id: requestId, - }; - - const line = `${JSON.stringify(record)}\n`; - const key = buildRecordKey(timestamp, requestId); - ctx.waitUntil(env.TRAFFIC_RECORDER.put(key, line)); - } catch (error) { - console.error("traffic recorder failed", error); - } -} diff --git a/router/src/redact.ts b/router/src/redact.ts deleted file mode 100644 index b633f12f..00000000 --- a/router/src/redact.ts +++ /dev/null @@ -1,132 +0,0 @@ -const HEADER_DENY_LIST = new Set([ - "authorization", - "cookie", - "set-cookie", - "x-relayauth-token", - "x-api-key", -]); - -const SENSITIVE_HEADER_NAME = /(token|secret|password|signature)/i; - -const FULL_BODY_REDACTION_PATHS = [ - /^\/api\/v1\/workspaces\/[a-zA-Z0-9_-]+\/secrets(?:\/.*)?$/, - /^\/api\/auth(?:\/.*)?$/, - /^\/api\/v1\/auth(?:\/.*)?$/, - /^\/api\/v1\/cli\/auth(?:\/.*)?$/, - /^\/api\/v1\/cli\/login(?:\/.*)?$/, - /^\/api\/v1\/integrations\/nango\/connect-link(?:\/.*)?$/, - /^\/api\/v1\/workspaces\/[a-zA-Z0-9_-]+\/integrations\/connect-session(?:\/.*)?$/, -]; - -const JSON_SECRET_KEYS = new Set([ - "token", - "accesstoken", - "refreshtoken", - "secret", - "apikey", - "password", - "clientsecret", - "privatekey", -]); - -type HeaderInput = Headers | Iterable<[string, string]> | Record; - -const textEncoder = new TextEncoder(); - -function utf8ByteLength(value: string): number { - return textEncoder.encode(value).length; -} - -function redactValue(value: string): string { - return `[REDACTED:${utf8ByteLength(value)}]`; -} - -function shouldRedactHeader(name: string): boolean { - const normalized = name.toLowerCase(); - return HEADER_DENY_LIST.has(normalized) || SENSITIVE_HEADER_NAME.test(normalized); -} - -function headerEntries(headers: HeaderInput): Iterable<[string, string]> { - if (headers instanceof Headers) { - const entries: Array<[string, string]> = []; - headers.forEach((value, name) => { - entries.push([name, value]); - }); - return entries; - } - - if (Symbol.iterator in Object(headers)) { - return headers as Iterable<[string, string]>; - } - - return Object.entries(headers); -} - -function shouldFullyRedactBody(path: string): boolean { - return FULL_BODY_REDACTION_PATHS.some((pattern) => pattern.test(path)); -} - -function isJsonContentType(contentType: string | null | undefined): boolean { - return typeof contentType === "string" && contentType.toLowerCase().includes("application/json"); -} - -function redactJsonValue(value: unknown): unknown { - if (Array.isArray(value)) { - return value.map((item) => redactJsonValue(item)); - } - - if (!value || typeof value !== "object") { - return value; - } - - const output: Record = {}; - for (const [key, nested] of Object.entries(value)) { - output[key] = JSON_SECRET_KEYS.has(key.toLowerCase()) - ? redactValue(typeof nested === "string" ? nested : JSON.stringify(nested) ?? "") - : redactJsonValue(nested); - } - - return output; -} - -export function redactHeaders(headers: HeaderInput): Record { - const output: Record = {}; - - for (const [name, value] of headerEntries(headers)) { - output[name] = shouldRedactHeader(name) ? redactValue(value) : value; - } - - return output; -} - -export function redactBody( - path: string, - body: string | null | undefined, - contentType?: string | null, -): string | null | undefined { - if (body == null) { - return body; - } - - if (shouldFullyRedactBody(path)) { - return redactValue(body); - } - - if (!isJsonContentType(contentType)) { - return body; - } - - try { - return JSON.stringify(redactJsonValue(JSON.parse(body))); - } catch { - return body; - } -} - -export function shouldRecord(path: string): boolean { - if (path === "/favicon.ico" || path === "/api/health") { - return false; - } - - return path !== "/observer" && !path.startsWith("/observer/"); -} diff --git a/router/test/rate-limit.test.ts b/router/test/rate-limit.test.ts deleted file mode 100644 index 369f55b4..00000000 --- a/router/test/rate-limit.test.ts +++ /dev/null @@ -1,588 +0,0 @@ -import { beforeEach, describe, expect, it } from "vitest"; - -import { - __resetBypassExtrasCacheForTests, - checkRateLimit, - extractLimitKey, - isBypassPath, - isBypassPathDynamic, - maybeRateLimit, - parseBypassExtras, - rateLimitResponse, - RATE_LIMIT_CONFIG_KEYS, - type RateLimitEnv, -} from "../src/rate-limit.js"; - -// Tiny in-memory KV shim. Counter increments only need get/put; we -// also store an optional `expirationTtl` so a future test could -// assert TTLs, though the current cases don't depend on it. -type StoredEntry = { value: string; expiresAt: number | null }; - -class MemoryKV { - readonly store = new Map(); - private nowFn: () => number; - - constructor(nowFn: () => number = () => Date.now()) { - this.nowFn = nowFn; - } - - async get(key: string): Promise { - const entry = this.store.get(key); - if (!entry) return null; - if (entry.expiresAt !== null && entry.expiresAt <= this.nowFn()) { - this.store.delete(key); - return null; - } - return entry.value; - } - - async put( - key: string, - value: string, - options?: { expirationTtl?: number }, - ): Promise { - const ttl = options?.expirationTtl; - const expiresAt = - typeof ttl === "number" && ttl > 0 ? this.nowFn() + ttl * 1000 : null; - this.store.set(key, { value, expiresAt }); - } - - // Helpers — not part of the KVNamespace surface. - size(): number { - return this.store.size; - } -} - -function asKV(kv: MemoryKV): KVNamespace { - return kv as unknown as KVNamespace; -} - -function buildEnv(opts: { - counters?: MemoryKV; - config?: MemoryKV; -}): RateLimitEnv { - return { - RATE_LIMIT_COUNTERS: opts.counters ? asKV(opts.counters) : undefined, - ROUTER_CONFIG: opts.config ? asKV(opts.config) : undefined, - }; -} - -const FIXED_NOW = 1715683680000; // 2024-05-14T11:28:00Z — middle of an even minute -const fixedNow = () => FIXED_NOW; - -describe("isBypassPath", () => { - it.each([ - ["/api/health", true], - ["/api/health/db", true], - ["/health", true], - ["/observer", true], - ["/observer/session/abc", true], - ["/_next/static/chunks/123.js", true], - ["/favicon.ico", true], - ["/v1/workspaces/ws_123/sync/status", false], - ["/cloud/api/v1/workspaces/ws_123/sandboxes", false], - ["/", false], - ])("path %s bypass=%s", (path, expected) => { - expect(isBypassPath(path)).toBe(expected); - }); -}); - -describe("extractLimitKey", () => { - it("picks workspace id from /v1/workspaces/:id/...", () => { - const req = new Request( - "https://api.relayfile.dev/v1/workspaces/ws_abc123/sync/status", - ); - expect(extractLimitKey(req)).toBe("ws:ws_abc123"); - }); - - it("picks workspace id from /cloud/api/v1/workspaces/:id/...", () => { - const req = new Request( - "https://agentrelay.com/cloud/api/v1/workspaces/ws_xyz/sandboxes", - ); - expect(extractLimitKey(req)).toBe("ws:ws_xyz"); - }); - - it("hashes bearer token when no workspace path is present", () => { - const req = new Request("https://agentrelay.com/cloud/api/v1/me", { - headers: { authorization: "Bearer abcdef0123456789" }, - }); - const key = extractLimitKey(req); - expect(key).toMatch(/^user:[0-9a-f]{8}$/); - }); - - it("buckets different tokens to different user keys", () => { - const a = extractLimitKey( - new Request("https://agentrelay.com/cloud/api/v1/me", { - headers: { authorization: "Bearer aaaaaaaa" }, - }), - ); - const b = extractLimitKey( - new Request("https://agentrelay.com/cloud/api/v1/me", { - headers: { authorization: "Bearer bbbbbbbb" }, - }), - ); - expect(a).not.toBe(b); - expect(a).toMatch(/^user:/); - expect(b).toMatch(/^user:/); - }); - - it("falls back to cf-connecting-ip for anonymous traffic", () => { - const req = new Request("https://agentrelay.com/", { - headers: { "cf-connecting-ip": "203.0.113.5" }, - }); - expect(extractLimitKey(req)).toBe("ip:203.0.113.5"); - }); - - it("prefers cf-connecting-ip over x-forwarded-for", () => { - const req = new Request("https://agentrelay.com/", { - headers: { - "cf-connecting-ip": "203.0.113.5", - "x-forwarded-for": "10.0.0.1, 10.0.0.2", - }, - }); - expect(extractLimitKey(req)).toBe("ip:203.0.113.5"); - }); - - it("uses x-forwarded-for first hop when cf-connecting-ip is missing", () => { - const req = new Request("https://agentrelay.com/", { - headers: { "x-forwarded-for": "10.0.0.1, 10.0.0.2" }, - }); - expect(extractLimitKey(req)).toBe("ip:10.0.0.1"); - }); - - it("keys off session cookie for /cloud* traffic with no auth header", () => { - const req = new Request("https://agentrelay.com/cloud/api/v1/me", { - headers: { cookie: "agent_relay_session=mysecrettoken123" }, - }); - const key = extractLimitKey(req); - expect(key).toMatch(/^session:[0-9a-f]{8}$/); - }); - - it("auth header takes priority over session cookie", () => { - const req = new Request("https://agentrelay.com/cloud/api/v1/me", { - headers: { - authorization: "Bearer bearertoken999", - cookie: "agent_relay_session=mysecrettoken123", - }, - }); - const key = extractLimitKey(req); - // auth header wins — produces a user: key, not session: - expect(key).toMatch(/^user:[0-9a-f]{8}$/); - }); - - it("buckets different session cookie values to different session keys", () => { - const a = extractLimitKey( - new Request("https://agentrelay.com/cloud/api/v1/me", { - headers: { cookie: "agent_relay_session=tokenA" }, - }), - ); - const b = extractLimitKey( - new Request("https://agentrelay.com/cloud/api/v1/me", { - headers: { cookie: "agent_relay_session=tokenB" }, - }), - ); - expect(a).not.toBe(b); - expect(a).toMatch(/^session:/); - expect(b).toMatch(/^session:/); - }); - - it("/cloud* with no cookie and no auth falls back to IP", () => { - const req = new Request("https://agentrelay.com/cloud/api/v1/me", { - headers: { "cf-connecting-ip": "198.51.100.1" }, - }); - expect(extractLimitKey(req)).toBe("ip:198.51.100.1"); - }); - - it("/cloud* with no cookie, no auth, no IP falls back to anon", () => { - const req = new Request("https://agentrelay.com/cloud/api/v1/me"); - expect(extractLimitKey(req)).toBe("anon"); - }); - - it("falls back to anon when no signal is present", () => { - const req = new Request("https://agentrelay.com/"); - expect(extractLimitKey(req)).toBe("anon"); - }); -}); - -describe("checkRateLimit", () => { - let counters: MemoryKV; - let config: MemoryKV; - let env: RateLimitEnv; - - beforeEach(() => { - counters = new MemoryKV(fixedNow); - config = new MemoryKV(fixedNow); - env = buildEnv({ counters, config }); - }); - - it("allows requests under the per-key limit", async () => { - for (let i = 1; i <= 5; i++) { - const result = await checkRateLimit("ws:demo", env, { now: fixedNow }); - expect(result.allowed).toBe(true); - expect(result.counter).toBe(i); - } - }); - - it("returns 429 once the per-key limit is exceeded", async () => { - // Pin the per-workspace limit low for deterministic assertions. - await config.put(RATE_LIMIT_CONFIG_KEYS.PER_KEY_PER_MIN, "3"); - - // Sequential calls — the in-memory KV shim mirrors a single edge - // processing requests in order. KV's eventual consistency across - // edges is acknowledged in the module-level docstring but is not - // what this test asserts. - const results = [ - await checkRateLimit("ws:demo", env, { now: fixedNow }), - await checkRateLimit("ws:demo", env, { now: fixedNow }), - await checkRateLimit("ws:demo", env, { now: fixedNow }), - ]; - expect(results.map((r) => r.allowed)).toEqual([true, true, true]); - - const blocked = await checkRateLimit("ws:demo", env, { now: fixedNow }); - expect(blocked.allowed).toBe(false); - expect(blocked.counter).toBe(4); - expect(blocked.limit).toBe(3); - expect(blocked.retryAfterSec).toBeGreaterThan(0); - expect(blocked.retryAfterSec).toBeLessThanOrEqual(60); - expect(blocked.key).toBe("ws:demo"); - }); - - it("does not pollute the global counter on a per-key reject", async () => { - await config.put(RATE_LIMIT_CONFIG_KEYS.PER_KEY_PER_MIN, "1"); - await checkRateLimit("ws:a", env, { now: fixedNow }); - const blocked = await checkRateLimit("ws:a", env, { now: fixedNow }); - expect(blocked.allowed).toBe(false); - - // Verify the global counter only ever incremented once (for the - // single allowed request) — the blocked one should NOT have - // incremented it. This is intentional so a single misbehaving - // key can't push the global ceiling around. - const bucket = Math.floor(FIXED_NOW / 1000 / 60); - const stored = await counters.get(`rl:__global__:${bucket}`); - expect(stored).toBe("1"); - }); - - it("returns 429 once the global limit is exceeded", async () => { - await config.put(RATE_LIMIT_CONFIG_KEYS.GLOBAL_PER_MIN, "2"); - await config.put(RATE_LIMIT_CONFIG_KEYS.PER_KEY_PER_MIN, "10"); - - expect( - (await checkRateLimit("ws:a", env, { now: fixedNow })).allowed, - ).toBe(true); - expect( - (await checkRateLimit("ws:b", env, { now: fixedNow })).allowed, - ).toBe(true); - - const blocked = await checkRateLimit("ws:c", env, { now: fixedNow }); - expect(blocked.allowed).toBe(false); - // Global rejects surface the synthetic `__global__` key so the - // 429 body tells operators it's the global ceiling, not the - // per-key one. - expect(blocked.key).toBe("__global__"); - }); - - it("respects RATE_LIMIT_DISABLED kill switch", async () => { - await config.put(RATE_LIMIT_CONFIG_KEYS.PER_KEY_PER_MIN, "1"); - await config.put(RATE_LIMIT_CONFIG_KEYS.DISABLED, "true"); - - // Even though the limit is 1, the kill switch should let - // unlimited requests through. - for (let i = 0; i < 5; i++) { - const result = await checkRateLimit("ws:demo", env, { now: fixedNow }); - expect(result.allowed).toBe(true); - } - // No counter writes when disabled — saves KV operations. - expect(counters.size()).toBe(0); - }); - - it("resets per minute (different bucket)", async () => { - await config.put(RATE_LIMIT_CONFIG_KEYS.PER_KEY_PER_MIN, "2"); - - const firstMinute = fixedNow; - const secondMinute = () => FIXED_NOW + 60_000; - - expect( - (await checkRateLimit("ws:demo", env, { now: firstMinute })).allowed, - ).toBe(true); - expect( - (await checkRateLimit("ws:demo", env, { now: firstMinute })).allowed, - ).toBe(true); - expect( - (await checkRateLimit("ws:demo", env, { now: firstMinute })).allowed, - ).toBe(false); - - // Roll forward one full minute — bucket key changes, counter - // starts over. - const reset = await checkRateLimit("ws:demo", env, { now: secondMinute }); - expect(reset.allowed).toBe(true); - expect(reset.counter).toBe(1); - }); - - it("falls back to defaults when ROUTER_CONFIG is unbound", async () => { - const noConfigEnv = buildEnv({ counters }); - const result = await checkRateLimit("ws:demo", noConfigEnv, { - now: fixedNow, - }); - expect(result.allowed).toBe(true); - expect(result.limit).toBe(60); // DEFAULT_PER_KEY_PER_MIN - }); - - it("treats missing RATE_LIMIT_COUNTERS as allow-all (fail open)", async () => { - const noCounterEnv = buildEnv({ config }); - await config.put(RATE_LIMIT_CONFIG_KEYS.PER_KEY_PER_MIN, "1"); - // Without a counter store we cannot enforce, so behavior is - // intentionally permissive (the limit is documented as - // best-effort). This guards against a missing binding silently - // breaking the data plane. - for (let i = 0; i < 10; i++) { - expect( - (await checkRateLimit("ws:demo", noCounterEnv, { now: fixedNow })) - .allowed, - ).toBe(true); - } - }); -}); - -describe("rateLimitResponse", () => { - it("emits a 429 with the documented shape", async () => { - const res = rateLimitResponse(42, "ws:demo"); - expect(res.status).toBe(429); - expect(res.headers.get("retry-after")).toBe("42"); - expect(res.headers.get("x-ratelimit-key")).toBe("ws:demo"); - expect(res.headers.get("content-type")).toBe("application/json"); - const body = (await res.json()) as Record; - expect(body).toEqual({ - error: "rate_limited", - retryAfter: 42, - limitKey: "ws:demo", - }); - }); -}); - -describe("maybeRateLimit", () => { - it("returns null for bypass paths without touching KV", async () => { - const counters = new MemoryKV(fixedNow); - const config = new MemoryKV(fixedNow); - const env = buildEnv({ counters, config }); - - const result = await maybeRateLimit( - new Request("https://api.relayfile.dev/health"), - env, - { now: fixedNow }, - ); - expect(result).toBeNull(); - expect(counters.size()).toBe(0); - }); - - it("returns null when under the limit", async () => { - const counters = new MemoryKV(fixedNow); - const config = new MemoryKV(fixedNow); - const env = buildEnv({ counters, config }); - - const result = await maybeRateLimit( - new Request("https://api.relayfile.dev/v1/workspaces/ws_a/sync/status"), - env, - { now: fixedNow }, - ); - expect(result).toBeNull(); - }); - - it("returns a 429 once limit is exceeded", async () => { - const counters = new MemoryKV(fixedNow); - const config = new MemoryKV(fixedNow); - const env = buildEnv({ counters, config }); - await config.put(RATE_LIMIT_CONFIG_KEYS.PER_KEY_PER_MIN, "1"); - - const url = - "https://api.relayfile.dev/v1/workspaces/ws_a/sync/status"; - const first = await maybeRateLimit(new Request(url), env, { - now: fixedNow, - }); - expect(first).toBeNull(); - - const blocked = await maybeRateLimit(new Request(url), env, { - now: fixedNow, - }); - expect(blocked).not.toBeNull(); - expect(blocked!.status).toBe(429); - expect(blocked!.headers.get("x-ratelimit-key")).toBe("ws:ws_a"); - }); -}); - -describe("parseBypassExtras", () => { - it("returns empty array for null / empty input", () => { - expect(parseBypassExtras(null)).toEqual([]); - expect(parseBypassExtras("")).toEqual([]); - expect(parseBypassExtras(" ")).toEqual([]); - }); - - it("parses comma-separated absolute paths", () => { - expect(parseBypassExtras("/api/k8s-readiness,/internal/uptime")).toEqual([ - "/api/k8s-readiness", - "/internal/uptime", - ]); - }); - - it("trims whitespace around entries", () => { - expect(parseBypassExtras(" /a , /b , /c ")).toEqual(["/a", "/b", "/c"]); - }); - - it("ignores entries that don't start with /", () => { - // Defensive: bad operator input shouldn't poison the list. - expect(parseBypassExtras("/ok,not-a-path,/also-ok")).toEqual([ - "/ok", - "/also-ok", - ]); - }); - - it("dedupes repeated entries", () => { - expect(parseBypassExtras("/x,/x,/y,/x")).toEqual(["/x", "/y"]); - }); -}); - -describe("isBypassPathDynamic", () => { - beforeEach(() => { - __resetBypassExtrasCacheForTests(); - }); - - it("returns true for static bypass paths without touching KV", async () => { - const env = buildEnv({}); // no ROUTER_CONFIG bound - expect(await isBypassPathDynamic("/api/health", env, FIXED_NOW)).toBe(true); - expect(await isBypassPathDynamic("/observer/foo", env, FIXED_NOW)).toBe( - true, - ); - }); - - it("returns false for non-bypass paths when no KV extras set", async () => { - const config = new MemoryKV(fixedNow); - const env = buildEnv({ config }); - expect( - await isBypassPathDynamic("/v1/workspaces/ws_a/sync/status", env, FIXED_NOW), - ).toBe(false); - }); - - it("respects KV-configured extras (exact match)", async () => { - const config = new MemoryKV(fixedNow); - const env = buildEnv({ config }); - await config.put( - RATE_LIMIT_CONFIG_KEYS.BYPASS_PREFIXES_EXTRA, - "/api/k8s-readiness", - ); - expect( - await isBypassPathDynamic("/api/k8s-readiness", env, FIXED_NOW), - ).toBe(true); - }); - - it("respects KV-configured extras (prefix match)", async () => { - const config = new MemoryKV(fixedNow); - const env = buildEnv({ config }); - await config.put( - RATE_LIMIT_CONFIG_KEYS.BYPASS_PREFIXES_EXTRA, - "/internal/uptime", - ); - expect( - await isBypassPathDynamic("/internal/uptime/check", env, FIXED_NOW), - ).toBe(true); - }); - - it("caches KV reads for 60s (does not re-read on every call)", async () => { - const config = new MemoryKV(fixedNow); - const env = buildEnv({ config }); - await config.put( - RATE_LIMIT_CONFIG_KEYS.BYPASS_PREFIXES_EXTRA, - "/api/k8s-readiness", - ); - - // Count KV reads by wrapping the real one. - let reads = 0; - const originalGet = env.ROUTER_CONFIG!.get.bind(env.ROUTER_CONFIG!); - env.ROUTER_CONFIG!.get = ((k: string) => { - reads++; - return originalGet(k); - }) as NonNullable["get"]; - - // First call hits KV. - await isBypassPathDynamic("/v1/x", env, FIXED_NOW); - expect(reads).toBe(1); - // Second call within 60s reuses the cache. - await isBypassPathDynamic("/v1/y", env, FIXED_NOW + 30_000); - expect(reads).toBe(1); - // Past the TTL boundary, fetches again. - await isBypassPathDynamic("/v1/z", env, FIXED_NOW + 61_000); - expect(reads).toBe(2); - }); - - it("falls back to last-known list on KV transient errors", async () => { - const config = new MemoryKV(fixedNow); - const env = buildEnv({ config }); - await config.put( - RATE_LIMIT_CONFIG_KEYS.BYPASS_PREFIXES_EXTRA, - "/cached-ok", - ); - - // Warm the cache once. - await isBypassPathDynamic("/v1/x", env, FIXED_NOW); - expect(await isBypassPathDynamic("/cached-ok", env, FIXED_NOW)).toBe(true); - - // Now break the KV. After TTL expiry the read will throw — we - // should fall back to the cached value rather than crashing. - env.ROUTER_CONFIG!.get = (() => { - throw new Error("KV unavailable"); - }) as NonNullable["get"]; - - expect( - await isBypassPathDynamic("/cached-ok", env, FIXED_NOW + 120_000), - ).toBe(true); - }); - - it("static and dynamic bypass coexist (static still works with KV extras present)", async () => { - const config = new MemoryKV(fixedNow); - const env = buildEnv({ config }); - await config.put( - RATE_LIMIT_CONFIG_KEYS.BYPASS_PREFIXES_EXTRA, - "/internal/uptime", - ); - // Static + dynamic both bypass. - expect(await isBypassPathDynamic("/api/health", env, FIXED_NOW)).toBe(true); - expect( - await isBypassPathDynamic("/internal/uptime", env, FIXED_NOW), - ).toBe(true); - }); -}); - -describe("maybeRateLimit honors KV bypass extras", () => { - beforeEach(() => { - __resetBypassExtrasCacheForTests(); - }); - - it("bypasses a runtime-whitelisted path", async () => { - const counters = new MemoryKV(fixedNow); - const config = new MemoryKV(fixedNow); - const env = buildEnv({ counters, config }); - // Tight limit so anything not bypassed would 429 on call #2. - await config.put(RATE_LIMIT_CONFIG_KEYS.PER_KEY_PER_MIN, "1"); - await config.put( - RATE_LIMIT_CONFIG_KEYS.BYPASS_PREFIXES_EXTRA, - "/api/k8s-readiness", - ); - - // Hit a non-bypassed path twice → second call 429s. - const url = "https://api.relayfile.dev/v1/workspaces/ws_a/sync/status"; - await maybeRateLimit(new Request(url), env, { now: fixedNow }); - const blocked = await maybeRateLimit(new Request(url), env, { - now: fixedNow, - }); - expect(blocked!.status).toBe(429); - - // Hit the runtime-whitelisted path 5 times → never 429s, regardless - // of the very tight per-key limit. - for (let i = 0; i < 5; i++) { - const r = await maybeRateLimit( - new Request("https://api.relayfile.dev/api/k8s-readiness"), - env, - { now: fixedNow }, - ); - expect(r).toBeNull(); - } - }); -}); diff --git a/router/test/recorder.test.ts b/router/test/recorder.test.ts deleted file mode 100644 index bd59dc9f..00000000 --- a/router/test/recorder.test.ts +++ /dev/null @@ -1,192 +0,0 @@ -import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; - -import { maybeRecord, type RecorderEnv } from "../src/recorder.js"; - -const FIXED_TIME = new Date("2026-05-14T09:08:07.000Z"); -const REQUEST_ID = "123e4567-e89b-12d3-a456-426614174000"; -const KEY_PATTERN = /^corpus\/\d{4}\/\d{2}\/\d{2}\/\d{2}\/[a-f0-9-]+\.ndjson$/; - -function createEnv(sampleRate: string): { - env: RecorderEnv; - put: ReturnType; - get: ReturnType; -} { - const put = vi.fn(async () => undefined); - const get = vi.fn(async () => sampleRate); - - return { - env: { - TRAFFIC_RECORDER: { - put, - } as unknown as R2Bucket, - ROUTER_CONFIG: { - get, - } as unknown as KVNamespace, - }, - put, - get, - }; -} - -function createCtx(): { - ctx: Pick; - waitUntil: ReturnType; - waits: Promise[]; -} { - const waits: Promise[] = []; - const waitUntil = vi.fn((promise: Promise) => { - waits.push(promise); - }); - - return { - ctx: { waitUntil }, - waitUntil, - waits, - }; -} - -describe("maybeRecord", () => { - beforeEach(() => { - vi.useFakeTimers(); - vi.setSystemTime(FIXED_TIME); - vi.spyOn(Math, "random").mockReturnValue(0); - }); - - afterEach(() => { - vi.restoreAllMocks(); - vi.useRealTimers(); - }); - - it("writes a record when sampling is 100 percent", async () => { - const { env, put, get } = createEnv("100"); - const { ctx, waitUntil, waits } = createCtx(); - const request = new Request("https://router.test/api/v1/messages?foo=bar", { - method: "POST", - headers: { - "content-type": "application/json", - authorization: "Bearer secret-token", - "x-request-id": REQUEST_ID, - }, - body: JSON.stringify({ - prompt: "hello", - token: "secret-token", - }), - }); - const response = new Response( - JSON.stringify({ - ok: true, - refreshToken: "refresh-me", - }), - { - status: 201, - headers: { - "content-type": "application/json", - "set-cookie": "sid=123", - }, - }, - ); - - await maybeRecord(request, response, env, ctx); - await Promise.all(waits); - - expect(get).toHaveBeenCalledWith("RECORDER_SAMPLE_RATE"); - expect(waitUntil).toHaveBeenCalledOnce(); - expect(put).toHaveBeenCalledOnce(); - - const [key, line] = put.mock.calls[0] as [string, string]; - expect(key).toBe("corpus/2026/05/14/09/123e4567-e89b-12d3-a456-426614174000.ndjson"); - expect(key).toMatch(KEY_PATTERN); - - const record = JSON.parse(line) as Record; - expect(record).toEqual({ - ts: "2026-05-14T09:08:07.000Z", - method: "POST", - path: "/api/v1/messages", - query: "foo=bar", - request_headers: { - authorization: "[REDACTED:19]", - "content-type": "application/json", - "x-request-id": REQUEST_ID, - }, - request_body: JSON.stringify({ - prompt: "hello", - token: "[REDACTED:12]", - }), - response_status: 201, - response_headers: { - "content-type": "application/json", - "set-cookie": "[REDACTED:7]", - }, - response_body: JSON.stringify({ - ok: true, - refreshToken: "[REDACTED:10]", - }), - request_id: REQUEST_ID, - }); - }); - - it("does not write when sampling is 0 percent", async () => { - const { env, put } = createEnv("0"); - const { ctx, waitUntil } = createCtx(); - - await maybeRecord( - new Request("https://router.test/api/v1/messages", { - method: "POST", - headers: { "content-type": "application/json" }, - body: JSON.stringify({ ok: true }), - }), - new Response("ok"), - env, - ctx, - ); - - expect(waitUntil).not.toHaveBeenCalled(); - expect(put).not.toHaveBeenCalled(); - }); - - it("skips deny-listed paths without touching storage", async () => { - const { env, put, get } = createEnv("100"); - const { ctx, waitUntil } = createCtx(); - - await maybeRecord( - new Request("https://router.test/observer/session/123"), - new Response("ok"), - env, - ctx, - ); - - expect(get).not.toHaveBeenCalled(); - expect(waitUntil).not.toHaveBeenCalled(); - expect(put).not.toHaveBeenCalled(); - }); - - it("replaces oversize bodies with a placeholder before writing", async () => { - const { env, put } = createEnv("100"); - const { ctx, waits } = createCtx(); - const oversized = "x".repeat(256 * 1024 + 1); - - await maybeRecord( - new Request("https://router.test/api/v1/messages", { - method: "POST", - headers: { - "content-type": "text/plain", - "x-request-id": REQUEST_ID, - }, - body: oversized, - }), - new Response("ok", { - headers: { "content-type": "text/plain" }, - }), - env, - ctx, - ); - await Promise.all(waits); - - expect(put).toHaveBeenCalledOnce(); - const [, line] = put.mock.calls[0] as [string, string]; - const record = JSON.parse(line) as { request_body: string; response_body: string }; - - expect(record.request_body).toBe("[OVERSIZE:262145]"); - expect(record.response_body).toBe("ok"); - }); -}); diff --git a/router/test/redact.test.ts b/router/test/redact.test.ts deleted file mode 100644 index f5302c2d..00000000 --- a/router/test/redact.test.ts +++ /dev/null @@ -1,78 +0,0 @@ -import { describe, expect, it } from "vitest"; - -import { redactBody, redactHeaders, shouldRecord } from "../src/redact.js"; - -describe("router redaction", () => { - it("redacts deny-listed and sensitive headers", () => { - const headers = redactHeaders({ - authorization: "Bearer secret-token", - "x-api-key": "topsecret", - "x-signature": "sig-123", - "content-type": "application/json", - }); - - expect(headers).toEqual({ - authorization: "[REDACTED:19]", - "x-api-key": "[REDACTED:9]", - "x-signature": "[REDACTED:7]", - "content-type": "application/json", - }); - }); - - it("fully redacts bodies for deny-listed paths", () => { - expect( - redactBody( - "/api/v1/workspaces/ws_123/secrets", - JSON.stringify({ apiKey: "secret-value" }), - "application/json", - ), - ).toBe("[REDACTED:25]"); - }); - - it("scrubs sensitive keys from JSON bodies", () => { - expect( - redactBody( - "/api/v1/messages", - JSON.stringify({ - ok: true, - nested: { - token: "secret-token", - profile: { - password: "swordfish", - }, - }, - items: [ - { - refreshToken: "refresh-me", - }, - ], - }), - "application/json; charset=utf-8", - ), - ).toBe( - JSON.stringify({ - ok: true, - nested: { - token: "[REDACTED:12]", - profile: { - password: "[REDACTED:9]", - }, - }, - items: [ - { - refreshToken: "[REDACTED:10]", - }, - ], - }), - ); - }); - - it("applies shouldRecord route rules", () => { - expect(shouldRecord("/favicon.ico")).toBe(false); - expect(shouldRecord("/api/health")).toBe(false); - expect(shouldRecord("/observer")).toBe(false); - expect(shouldRecord("/observer/session/123")).toBe(false); - expect(shouldRecord("/observer-files")).toBe(true); - expect(shouldRecord("/cloud/dashboard")).toBe(true); - }); -}); diff --git a/router/test/webhook-routing.test.ts b/router/test/webhook-routing.test.ts deleted file mode 100644 index 8daa494d..00000000 --- a/router/test/webhook-routing.test.ts +++ /dev/null @@ -1,157 +0,0 @@ -import { describe, expect, it, vi } from "vitest"; - -import worker from "../index.js"; - -type StoredEntry = { value: string }; - -class MemoryKV { - readonly store = new Map(); - - async get(key: string): Promise { - return this.store.get(key)?.value ?? null; - } - - async put(key: string, value: string): Promise { - this.store.set(key, { value }); - } -} - -function asKV(kv: MemoryKV): KVNamespace { - return kv as unknown as KVNamespace; -} - -type WorkerBinding = { fetch: ReturnType }; - -function makeBinding(): WorkerBinding { - return { - fetch: vi.fn(async () => new Response("ok", { status: 200 })), - }; -} - -function buildEnv(opts: { - webhookOriginFlag?: string; - cloudWebWorker?: WorkerBinding; - webhookWorker?: WorkerBinding; -}) { - const routerConfig = new MemoryKV(); - if (opts.webhookOriginFlag !== undefined) { - routerConfig.store.set("WEBHOOK_ORIGIN", { value: opts.webhookOriginFlag }); - } - return { - CLOUD_APP_ORIGIN: "https://origin.test.invalid", - ROUTER_CONFIG: asKV(routerConfig), - CLOUD_WEB_WORKER: opts.cloudWebWorker, - WEBHOOK_WORKER: opts.webhookWorker, - } as unknown as Parameters[1]; -} - -function buildCtx(): ExecutionContext { - return { - waitUntil: () => undefined, - passThroughOnException: () => undefined, - } as unknown as ExecutionContext; -} - -describe("router webhook routing", () => { - it("routes /api/v1/webhooks/nango to the webhook-worker binding when WEBHOOK_ORIGIN=worker", async () => { - const cloudWebWorker = makeBinding(); - const webhookWorker = makeBinding(); - const env = buildEnv({ - webhookOriginFlag: "worker", - cloudWebWorker, - webhookWorker, - }); - - const request = new Request( - "https://origin.agentrelay.cloud/cloud/api/v1/webhooks/nango", - { - method: "POST", - headers: { "content-type": "application/json" }, - body: JSON.stringify({ type: "forward", from: "github-app-oauth" }), - }, - ); - - await worker.fetch(request, env, buildCtx()); - - expect(webhookWorker.fetch).toHaveBeenCalledOnce(); - expect(cloudWebWorker.fetch).not.toHaveBeenCalled(); - }); - - it("bypasses webhook-worker when request carries x-cloud-webhook-worker-forwarded header (loop break)", async () => { - const cloudWebWorker = makeBinding(); - const webhookWorker = makeBinding(); - const env = buildEnv({ - webhookOriginFlag: "worker", - cloudWebWorker, - webhookWorker, - }); - - const request = new Request( - "https://origin.agentrelay.cloud/cloud/api/v1/webhooks/nango", - { - method: "POST", - headers: { - "content-type": "application/json", - "x-cloud-webhook-worker-forwarded": "webhook-worker", - "x-cloud-webhook-worker-request-id": "req-loop-1", - }, - body: JSON.stringify({ type: "forward", from: "github-app-oauth" }), - }, - ); - - await worker.fetch(request, env, buildCtx()); - - expect(cloudWebWorker.fetch).toHaveBeenCalledOnce(); - expect(webhookWorker.fetch).not.toHaveBeenCalled(); - }); - - it("ignores the loop-break header for an unrelated value (don't accidentally widen bypass)", async () => { - const cloudWebWorker = makeBinding(); - const webhookWorker = makeBinding(); - const env = buildEnv({ - webhookOriginFlag: "worker", - cloudWebWorker, - webhookWorker, - }); - - const request = new Request( - "https://origin.agentrelay.cloud/cloud/api/v1/webhooks/nango", - { - method: "POST", - headers: { - "content-type": "application/json", - "x-cloud-webhook-worker-forwarded": "something-else", - }, - body: JSON.stringify({ type: "forward", from: "github-app-oauth" }), - }, - ); - - await worker.fetch(request, env, buildCtx()); - - expect(webhookWorker.fetch).toHaveBeenCalledOnce(); - expect(cloudWebWorker.fetch).not.toHaveBeenCalled(); - }); - - it("routes to cloud-web when WEBHOOK_ORIGIN flag is unset (regression: still works without flag)", async () => { - const cloudWebWorker = makeBinding(); - const webhookWorker = makeBinding(); - const env = buildEnv({ - cloudWebWorker, - webhookWorker, - }); - - const request = new Request( - "https://origin.agentrelay.cloud/cloud/api/v1/webhooks/nango", - { - method: "POST", - headers: { "content-type": "application/json" }, - body: JSON.stringify({ type: "forward", from: "github-app-oauth" }), - }, - ); - - await worker.fetch(request, env, buildCtx()); - - expect(cloudWebWorker.fetch).toHaveBeenCalledOnce(); - expect(webhookWorker.fetch).not.toHaveBeenCalled(); - }); -}); diff --git a/router/tsconfig.json b/router/tsconfig.json deleted file mode 100644 index 00c428ac..00000000 --- a/router/tsconfig.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "compilerOptions": { - "target": "ES2022", - "module": "ES2022", - "moduleResolution": "bundler", - "lib": ["ES2022", "WebWorker"], - "strict": true, - "noEmit": true, - "skipLibCheck": true, - "esModuleInterop": true, - "resolveJsonModule": true, - "isolatedModules": true, - "types": ["@cloudflare/workers-types", "vitest/globals"] - }, - "include": ["index.ts", "src/**/*.ts", "test/**/*.ts"], - "exclude": ["node_modules", "dist"] -} From 8557ae3ecc6a95f5fdf2a8fddd34734e2938dab3 Mon Sep 17 00:00:00 2001 From: Hubspot Phase B Bot Date: Wed, 27 May 2026 03:14:50 +0200 Subject: [PATCH 41/49] Revert "Repair preflight clone materialization" (895fb683) Restores 2794 files accidentally deleted from main; tree returns to known-good b6b9c954 (all tonight's E2E fixes #1205-#1231 intact). Co-Authored-By: Claude Opus 4.7 (1M context) --- router/README.md | 66 ++++ router/index.ts | 438 +++++++++++++++++++++ router/src/rate-limit.ts | 550 ++++++++++++++++++++++++++ router/src/recorder.ts | 140 +++++++ router/src/redact.ts | 132 +++++++ router/test/rate-limit.test.ts | 588 ++++++++++++++++++++++++++++ router/test/recorder.test.ts | 192 +++++++++ router/test/redact.test.ts | 78 ++++ router/test/webhook-routing.test.ts | 157 ++++++++ router/tsconfig.json | 17 + 10 files changed, 2358 insertions(+) create mode 100644 router/README.md create mode 100644 router/index.ts create mode 100644 router/src/rate-limit.ts create mode 100644 router/src/recorder.ts create mode 100644 router/src/redact.ts create mode 100644 router/test/rate-limit.test.ts create mode 100644 router/test/recorder.test.ts create mode 100644 router/test/redact.test.ts create mode 100644 router/test/webhook-routing.test.ts create mode 100644 router/tsconfig.json diff --git a/router/README.md b/router/README.md new file mode 100644 index 00000000..98defc42 --- /dev/null +++ b/router/README.md @@ -0,0 +1,66 @@ +Route precedence: + +- `/cloud*` routes to the Agent Relay Cloud Next.js app. +- `/observer/file*` routes to the RelayFile observer app, expected to be built with `basePath: "/observer/file"`. +- `/observer*` routes to the Relaycast observer app. +- everything else falls back to the legacy proxy target. + +RelayFile observer deployment is wired from the production job in +`.github/workflows/deploy.yml`. The job runs +`scripts/deploy-file-observer-pages.sh`, which packs +`@relayfile/file-observer@latest`, builds it with +`FILE_OBSERVER_BASE_PATH=/observer/file`, and deploys the detected static output +to the Cloudflare Pages project `relayfile-file-observer`. The script creates +that Pages project with the configured production branch when it is missing, so +the first production deploy does not require manual Cloudflare setup. + +``` + ┌────────────────────────────────────┐ + │ @agentworkforce/cloud │ + │ │ + │/packages/router (Cloudflare worker)│ + └────────────────────────────────────┘ + + ┌──────────────────────────────────────┐ + │ agentrelay.com │ + └──────────────────────────────────────┘ + │ + │ + │ + All cloud routes │ Observer Routes + ┌───(/cloud/*, /cloud/api)───────────┼────(/observer*, /observer/file*)───────┐ + │ │ │ + │ │ │ + │ │ │ + ▼ │ ▼ +┌────────────────────────────────────┐ │ ┌────────────────────────────────────┐ +│ https://agentrelay.cloud │ │ │ https://relaycast.dev/observer │ +└────────────────────────────────────┘ │ └────────────────────────────────────┘ + │ + │ +┌────────────────────────────────────┐ All Other Routes ┌────────────────────────────────────┐ +│ @agentworkforce/cloud │ (/, /docs, /blog, /openclaw) │ @agentworkforce/relaycast │ +│ │ │ │ │ +│ /packages/web (NextJs App) │ │ │ /packages/observer-dashboard │ +│ │ │ │ /@relayfile/file-observer │ +└────────────────────────────────────┘ │ └────────────────────────────────────┘ + │ + │ + │ + │ + │ + │ + │ + │ + ▼ + ┌────────────────────────────────────┐ + │ https://origin.agentrelay.net │ + └────────────────────────────────────┘ + + + ┌────────────────────────────────────┐ + │ @agentworkforce/relay │ + │ │ + │ /web (NextJs App) │ + └────────────────────────────────────┘ +``` diff --git a/router/index.ts b/router/index.ts new file mode 100644 index 00000000..df9cfde6 --- /dev/null +++ b/router/index.ts @@ -0,0 +1,438 @@ +import { maybeRecord, type RecorderEnv } from "./src/recorder.js"; +import { maybeRateLimit, type RateLimitEnv } from "./src/rate-limit.js"; + +interface Env { + CLOUD_APP_ORIGIN: string; + CLOUD_WEB_WORKER?: { + fetch(request: Request): Promise; + }; + FILE_OBSERVER_ORIGIN?: string; + TRAFFIC_RECORDER?: RecorderEnv["TRAFFIC_RECORDER"]; + ROUTER_CONFIG?: RecorderEnv["ROUTER_CONFIG"]; + RATE_LIMIT_COUNTERS?: RateLimitEnv["RATE_LIMIT_COUNTERS"]; + WEBHOOK_WORKER?: { + fetch(request: Request): Promise; + }; + WEBHOOK_WORKER_ORIGIN?: string; +} + +function hasRecorderEnv(env: Env): env is Env & RecorderEnv { + return Boolean(env.TRAFFIC_RECORDER && env.ROUTER_CONFIG); +} + +const FALLBACK_PROXY_ORIGIN = "https://origin.agentrelay.net"; +const OBSERVER_ORIGIN = "https://observer.relaycast.dev"; +const DEFAULT_FILE_OBSERVER_ORIGIN = "https://relayfile-file-observer.pages.dev"; +const PRIMARY_HOST = "agentrelay.com"; +const FILE_OBSERVER_PATH_PREFIX = "/observer/file"; +const OBSERVER_PATH_PREFIX = "/observer"; +const CLOUD_PATH_PREFIX = "/cloud"; +const WEBHOOK_ORIGIN_FLAG_KEY = "WEBHOOK_ORIGIN"; + +// Header set by webhook-worker's queue consumer +// (`packages/webhook-worker/src/queue-consumer.ts`'s `buildForwardHeaders`) on +// outbound forwards to cloud-web's `/api/v1/webhooks/nango`. When the +// `WEBHOOK_ORIGIN` flag is `"worker"`, the router would otherwise redirect +// every `/api/v1/webhooks/nango` POST back to webhook-worker — including +// webhook-worker's own outbound forwards, producing an infinite redelivery +// loop bounded only by `maxRetries`. Honouring this header lets webhook-worker +// reach cloud-web's `routeForwardEvent` → `handleGitHubForward` path which is +// the actual destination of the forward. +const WEBHOOK_WORKER_FORWARDED_HEADER = "x-cloud-webhook-worker-forwarded"; +const WEBHOOK_WORKER_FORWARDED_VALUE = "webhook-worker"; +let loggedPhase5aLambdaEliminated = false; + +// Exact paths the webhook worker handles. Other sub-paths under +// /api/v1/webhooks (notably /api/v1/webhooks/composio/connect/callback, an +// OAuth callback served by Next.js) must continue to route to the Lambda even +// when WEBHOOK_ORIGIN=worker, otherwise they 404 against the Worker. +const WEBHOOK_WORKER_PATHS = new Set([ + "/api/v1/webhooks/composio", + "/api/v1/webhooks/github", + "/api/v1/webhooks/hookdeck", + "/api/v1/webhooks/nango", +]); +const NANGO_WEBHOOK_WORKER_PATH = "/api/v1/webhooks/nango"; + +function isPathWithinPrefix(pathname: string, prefix: string): boolean { + return pathname === prefix || pathname.startsWith(`${prefix}/`); +} + +function isObserverPath(pathname: string): boolean { + return isPathWithinPrefix(pathname, OBSERVER_PATH_PREFIX); +} + +function isFileObserverPath(pathname: string): boolean { + return isPathWithinPrefix(pathname, FILE_OBSERVER_PATH_PREFIX); +} + +function isPrimaryFileObserverPath(hostname: string, pathname: string): boolean { + return hostname === PRIMARY_HOST && isFileObserverPath(pathname); +} + +function isCloudPath(pathname: string): boolean { + return isPathWithinPrefix(pathname, CLOUD_PATH_PREFIX); +} + +// True only for the exact paths the webhook worker knows how to handle. Used +// to gate worker forwarding so unrelated routes under /api/v1/webhooks/* (e.g. +// the Composio OAuth callback) still reach the Lambda. +export function isWebhookWorkerPath(pathname: string): boolean { + return WEBHOOK_WORKER_PATHS.has(stripPathPrefix(pathname, CLOUD_PATH_PREFIX)); +} + +function isNangoWebhookWorkerPath(pathname: string): boolean { + return stripPathPrefix(pathname, CLOUD_PATH_PREFIX) === NANGO_WEBHOOK_WORKER_PATH; +} + +function stripPathPrefix(pathname: string, prefix: string): string { + if (pathname === prefix) { + return "/"; + } + + if (pathname.startsWith(`${prefix}/`)) { + return pathname.slice(prefix.length); + } + + return pathname; +} + +function addPathPrefix(pathname: string, prefix: string): string { + if (!prefix) { + return pathname; + } + + if (pathname === "/") { + return prefix; + } + + if (pathname === prefix || pathname.startsWith(`${prefix}/`) || pathname.startsWith(`${prefix}?`)) { + return pathname; + } + + return `${prefix}${pathname}`; +} + +function hostnameFromHost(host: string, protocol: string): string { + try { + return new URL(`${protocol}//${host}`).hostname; + } catch { + return host.split(":")[0] ?? host; + } +} + +function isPublicFileObserverLocation(hostname: string, pathname: string): boolean { + return isPrimaryFileObserverPath(hostname, pathname); +} + +export function getUpstreamPath(hostname: string, pathname: string): string { + if (isPrimaryFileObserverPath(hostname, pathname)) { + return stripPathPrefix(pathname, FILE_OBSERVER_PATH_PREFIX); + } + + return pathname; +} + +export function getMountPrefix(hostname: string, pathname: string): string { + if (isCloudPath(pathname)) { + return CLOUD_PATH_PREFIX; + } + + if (isPrimaryFileObserverPath(hostname, pathname)) { + return FILE_OBSERVER_PATH_PREFIX; + } + + return ""; +} + +export function rewriteLocation( + location: string, + originUrl: URL, + requestHost: string, + requestProtocol: string, + mountPrefix = "", +): string { + if (!location) { + return location; + } + + try { + const absolute = new URL(location); + if (isPublicFileObserverLocation(absolute.hostname, absolute.pathname)) { + return location; + } + + if (absolute.hostname !== originUrl.hostname) { + return location; + } + + absolute.hostname = requestHost; + absolute.port = ""; + absolute.protocol = requestProtocol; + absolute.pathname = addPathPrefix(absolute.pathname, mountPrefix); + return absolute.toString(); + } catch { + if (location.startsWith("/")) { + try { + const requestHostname = hostnameFromHost(requestHost, requestProtocol); + const locationUrl = new URL(location, `${requestProtocol}//${requestHost}`); + if (isPublicFileObserverLocation(requestHostname, locationUrl.pathname)) { + return location; + } + } catch { + // Fall through to the normal mount-prefix rewrite. + } + + return addPathPrefix(location, mountPrefix); + } + + return location; + } +} + +export function getOrigin(hostname: string, pathname: string, env: Env): string { + // /cloud* defaults to the Next.js cloud app regardless of host. Requests only + // reach this fallback when the cloud-web Worker service binding is absent. + if (isCloudPath(pathname)) { + return env.CLOUD_APP_ORIGIN; + } + + // The production agentrelay.com apex is a split router: + // 1. /observer/file* goes to the RelayFile file observer app + // 2. /observer* stays on the Relaycast observer app + // 3. everything else falls back to the relay web origin + if (hostname === PRIMARY_HOST) { + if (isPrimaryFileObserverPath(hostname, pathname)) { + return env.FILE_OBSERVER_ORIGIN ?? DEFAULT_FILE_OBSERVER_ORIGIN; + } + + if (isObserverPath(pathname)) { + return OBSERVER_ORIGIN; + } + } + + return FALLBACK_PROXY_ORIGIN; +} + +async function shouldUseCloudWebWorker( + pathname: string, + request: Request, + env: Env, +): Promise { + if (!isCloudPath(pathname)) { + return false; + } + + if (await shouldUseWebhookWorker(pathname, request, env)) { + return false; + } + + return true; +} + +function logPhase5aLambdaEliminatedOnce(): void { + if (loggedPhase5aLambdaEliminated) { + return; + } + + loggedPhase5aLambdaEliminated = true; + console.log(JSON.stringify({ router_phase: "5a_lambda_eliminated" })); +} + +export async function readWebhookOriginFlag(env: Env): Promise { + try { + return (await env.ROUTER_CONFIG?.get(WEBHOOK_ORIGIN_FLAG_KEY)) ?? null; + } catch { + return null; + } +} + +export async function shouldUseNangoWebhookWorkerRoute( + pathname: string, + env: Env, +): Promise { + if (!isNangoWebhookWorkerPath(pathname)) { + return false; + } + + const configured = await readWebhookOriginFlag(env); + return configured?.trim().toLowerCase() === "worker"; +} + +async function shouldUseWebhookWorker( + pathname: string, + request: Request, + env: Env, +): Promise { + // Break the redelivery loop: webhook-worker's queue consumer forwards the + // raw envelope back to the same `/api/v1/webhooks/nango` route on + // `origin.agentrelay.cloud` so cloud-web's `handleGitHubForward` can run. + // Without this header check, the router catches that forward and redirects + // it back to webhook-worker, which re-enqueues, ad infinitum (or until + // `maxRetries`). + if (request.headers.get(WEBHOOK_WORKER_FORWARDED_HEADER) === WEBHOOK_WORKER_FORWARDED_VALUE) { + return false; + } + return shouldUseNangoWebhookWorkerRoute(pathname, env); +} + +function buildWebhookWorkerRequest( + request: Request, + requestUrl: URL, + workerOrigin?: string, +): Request { + const targetUrl = new URL(requestUrl.toString()); + targetUrl.pathname = stripPathPrefix(targetUrl.pathname, CLOUD_PATH_PREFIX); + + if (workerOrigin) { + const originUrl = new URL(workerOrigin); + targetUrl.protocol = originUrl.protocol; + targetUrl.hostname = originUrl.hostname; + targetUrl.port = originUrl.port; + } + + const init: RequestInit & { duplex?: "half" } = { + method: request.method, + headers: request.headers, + body: request.body, + redirect: "manual", + duplex: "half", + }; + + return new Request(targetUrl.toString(), init); +} + +export default { + async fetch(request: Request, env: Env, ctx: ExecutionContext): Promise { + const url = new URL(request.url); + + // Per-key rate limiting runs BEFORE any worker routing so a runaway + // workspace gets bounded everywhere — including webhook ingress and + // /cloud* traffic. The bypass list inside maybeRateLimit exempts + // health and observer paths. See packages/router/src/rate-limit.ts + // and docs/security/rate-limiting.md. + const rateLimited = await maybeRateLimit(request, env); + if (rateLimited) { + return rateLimited; + } + + // Clone the request up front so any branch that returns early (cloud-web + // Worker service binding, webhook Worker service binding, etc.) can + // still feed the recorder the original payload. Without this clone, the + // /cloud Worker path bypasses the recorder entirely and the replay + // harness has no corpus to prove equivalence during Phase 4 cutover. + // See Codex P2.6 on bundle PR #647. + const recorderEnv = hasRecorderEnv(env) ? env : null; + const recorderRequestClone = recorderEnv + ? (request.clone() as unknown as Request) + : null; + + if (await shouldUseCloudWebWorker(url.pathname, request, env)) { + logPhase5aLambdaEliminatedOnce(); + if (!env.CLOUD_WEB_WORKER) { + return new Response( + JSON.stringify({ error: "cloud web worker binding unavailable" }), + { + status: 503, + headers: { "content-type": "application/json" }, + }, + ); + } + + const workerResponse = await env.CLOUD_WEB_WORKER.fetch(request); + if (recorderRequestClone && recorderEnv) { + ctx.waitUntil( + maybeRecord(recorderRequestClone, workerResponse.clone(), recorderEnv, ctx), + ); + } + return workerResponse; + } + + if (await shouldUseWebhookWorker(url.pathname, request, env)) { + const webhookWorkerRequest = buildWebhookWorkerRequest(request, url); + if (env.WEBHOOK_WORKER) { + const workerResponse = await env.WEBHOOK_WORKER.fetch(webhookWorkerRequest); + if (recorderRequestClone && recorderEnv) { + ctx.waitUntil( + maybeRecord(recorderRequestClone, workerResponse.clone(), recorderEnv, ctx), + ); + } + return workerResponse; + } + + const workerOrigin = env.WEBHOOK_WORKER_ORIGIN?.trim(); + if (workerOrigin) { + const originResponse = await globalThis.fetch( + buildWebhookWorkerRequest(request, url, workerOrigin), + ); + if (recorderRequestClone && recorderEnv) { + ctx.waitUntil( + maybeRecord(recorderRequestClone, originResponse.clone(), recorderEnv, ctx), + ); + } + return originResponse; + } + } + + const requestHost = request.headers.get("Host") || url.hostname; + const originUrl = new URL(getOrigin(url.hostname, url.pathname, env)); + const mountPrefix = getMountPrefix(url.hostname, url.pathname); + + url.pathname = getUpstreamPath(url.hostname, url.pathname); + url.hostname = originUrl.hostname; + url.port = ""; + url.protocol = "https:"; + + const headers = new Headers(request.headers); + headers.set("X-Forwarded-Host", requestHost); + headers.set("X-Original-Host", requestHost); + headers.set("X-Forwarded-Proto", "https"); + if (mountPrefix) { + headers.set("X-Forwarded-Prefix", mountPrefix); + } + + // Reuse the clone made at the top of fetch() for the recorder. The + // earlier clone covers all branches; making a second one here would + // be wasted bytes on every request and would double-record on the + // Lambda origin path. + const recordingRequest = recorderRequestClone; + const subRequest = new Request(url.toString(), { + method: request.method, + headers, + body: request.body, + redirect: "manual", + }); + + try { + // Use `globalThis.fetch` rather than a bare `fetch` identifier: Cloudflare + // Workers can hoist bare `fetch` off `globalThis` and throw + // `TypeError: Illegal invocation`. See sage `.claude/rules/workers-fetch.md`. + const upstreamResponse = await globalThis.fetch(subRequest); + const responseHeaders = new Headers(upstreamResponse.headers); + + const location = responseHeaders.get("Location"); + if (location) { + responseHeaders.set( + "Location", + rewriteLocation(location, originUrl, requestHost, "https:", mountPrefix), + ); + } + + const response = new Response(upstreamResponse.body, { + status: upstreamResponse.status, + statusText: upstreamResponse.statusText, + headers: responseHeaders, + }); + + if (recordingRequest && hasRecorderEnv(env)) { + ctx.waitUntil(maybeRecord(recordingRequest, response.clone(), env, ctx)); + } + + return response; + } catch (error) { + return new Response(JSON.stringify({ error: (error as Error).message }), { + status: 500, + }); + } + }, +}; diff --git a/router/src/rate-limit.ts b/router/src/rate-limit.ts new file mode 100644 index 00000000..d7c6c734 --- /dev/null +++ b/router/src/rate-limit.ts @@ -0,0 +1,550 @@ +// Per-key request rate limiting for the Cloudflare edge gateways. +// +// Two Workers front the platform: the `AgentRelayRouter` worker +// (agentrelay.com / cloud router) and the `RelayfileApi` worker +// (api.relayfile.dev). A misbehaving client today (e.g. a relayfile +// mount stuck in a polling loop) can hammer either gateway with +// dozens of requests per second; the upstream Lambda has a 10-slot +// concurrency cap, so once a single key saturates the gateway every +// other user starts seeing 429s. The rate limiter in this module +// runs at the edge BEFORE the request reaches the Lambda, so a +// runaway key is contained to its own per-workspace bucket and +// doesn't deplete shared capacity. +// +// Design choices (intentional trade-offs): +// +// * Counter storage is Cloudflare KV (a `RATE_LIMIT_COUNTERS` +// namespace bound to both Workers). KV is eventually consistent +// across edges, so the effective ceiling at high traffic can be +// ~2-3x the configured per-minute limit. For the 60 req/min +// target that means ~180 req/min worst case — still 100x lower +// than the runaway today and good enough for the runaway-client +// class of bug. Sub-second precision and global atomicity would +// require a Durable Object, which we may add later if KV drift +// bites — but the simpler path ships first. +// +// * Buckets are minute-aligned (epoch-minute integer). Each +// `(limitKey, bucket)` pair is one KV key with a 120-second TTL +// so the previous and current minute coexist briefly during the +// turn. Reads are racy w.r.t. writes; that's accepted (see above). +// +// * Limits are read from `ROUTER_CONFIG` KV at request time, so +// operators can dial them up/down or kill-switch the limiter +// entirely via `wrangler kv:key put` without redeploying. +// +// * Defaults: 60 req/min per workspace-or-user-or-ip key, 1000 +// req/min global ceiling (a separate safety-net counter shared +// across all keys). Both defaults are intentionally conservative +// given the actual runaway pattern observed (30+ req/sec from a +// single key). +// +// * Bypass list mirrors the recorder's deny list plus a few +// observability paths — these must never be rate limited because +// they're either health probes, browser asset requests, or +// observer/webhook ingress that already has its own backpressure. + +const ONE_MINUTE_SECONDS = 60; +// KV TTL on counter entries. Must be >= 60 to cover a full bucket +// window even if the request arrives at the very start of the +// minute. We pick 120s so the previous bucket is still readable +// briefly during the turnover, which means a borderline client that +// straddles the boundary still sees its prior count and can't double +// up by ~timing requests across the second. +const COUNTER_TTL_SECONDS = 120; + +// Default config — operators tune via `wrangler kv:key put` on the +// `ROUTER_CONFIG` namespace; we re-read every request. +const DEFAULT_PER_KEY_PER_MIN = 60; +const DEFAULT_GLOBAL_PER_MIN = 1000; + +// KV config keys we read from `ROUTER_CONFIG`. Centralized here so +// the docs and the infra seed step refer to the same names. +export const RATE_LIMIT_CONFIG_KEYS = { + PER_KEY_PER_MIN: "RATE_LIMIT_PER_WORKSPACE_PER_MIN", + GLOBAL_PER_MIN: "RATE_LIMIT_GLOBAL_PER_MIN", + DISABLED: "RATE_LIMIT_DISABLED", + // Comma-separated path prefixes added to the bypass list at + // runtime. Static `BYPASS_PREFIXES` below is the safe core; this + // key lets an operator whitelist a new path (e.g. a freshly-added + // health probe) without redeploying. Values are matched the same + // way as static prefixes — exact match or `${prefix}/...` startswith. + // Example KV value: `/api/k8s-readiness,/internal/uptime` + BYPASS_PREFIXES_EXTRA: "RATE_LIMIT_BYPASS_PREFIXES_EXTRA", +} as const; + +// Paths that must NEVER be rate limited. Health probes, static +// assets, and observability endpoints. The router/relayfile-api +// share this list — both Workers see these paths even though only +// one of them actually serves them, and accidentally rate-limiting +// a health probe would self-DoS the upstream monitoring. +const BYPASS_PREFIXES: readonly string[] = [ + "/api/health", + "/health", + "/observer", + "/_next/static", + "/favicon.ico", + // Webhook ingress has no auth token and no workspace path, so it + // would bucket by IP and trip the 60/min cap with even modest + // burst traffic from a single provider (GitHub, Composio, Nango, + // Hookdeck). The webhook worker already has queue-backed + // backpressure, so rate-limiting at the edge would only cause us + // to drop legitimate events. See Codex P1.4 on bundle PR #647. + "/api/v1/webhooks", +]; + +export interface RateLimitEnv { + // Counter storage — shared across both gateway Workers so a + // workspace's limit isn't accidentally doubled by hitting both. + RATE_LIMIT_COUNTERS?: KVNamespace; + // Configuration knobs — same KV namespace the router already uses + // for other flags (e.g. WEBHOOK_ORIGIN). Optional so unit tests + // can run without it. + ROUTER_CONFIG?: KVNamespace; +} + +export interface RateLimitDecision { + allowed: boolean; + retryAfterSec?: number; + counter: number; + limit: number; + // The key the decision applies to — exposed so callers can put it + // in the 429 body (helps operators correlate gateway logs with the + // offending workspace/user/IP). + key: string; +} + +// Pure path helper so we can unit test it without a real Request. +// Checks ONLY the static hardcoded prefix list — does not consult KV. +// Use `isBypassPathDynamic` when env is available and you want the +// runtime-extensible bypass list to apply. +export function isBypassPath(pathname: string): boolean { + return matchesAnyPrefix(pathname, BYPASS_PREFIXES); +} + +function matchesAnyPrefix( + pathname: string, + prefixes: readonly string[], +): boolean { + for (const prefix of prefixes) { + if (pathname === prefix || pathname.startsWith(`${prefix}/`)) { + return true; + } + } + return false; +} + +// Module-scoped cache for the KV-resolved extra bypass prefixes. +// Refreshed at most once per `BYPASS_EXTRAS_TTL_MS` per Worker +// isolate. Operators tuning the KV value see propagation within +// roughly `BYPASS_EXTRAS_TTL_MS` plus the time CF's eventually- +// consistent KV takes to fan out to edges (typically <60s). This +// is the same staleness window that applies to the existing numeric +// config keys (`RATE_LIMIT_PER_WORKSPACE_PER_MIN`, etc.) — keeping +// the model consistent across all runtime knobs. +const BYPASS_EXTRAS_TTL_MS = 60_000; +let cachedBypassExtras: readonly string[] = []; +let cachedBypassExtrasFetchedAt = 0; + +/** + * Test-only: reset the bypass-extras cache between test cases. + * Production code never calls this. + */ +export function __resetBypassExtrasCacheForTests(): void { + cachedBypassExtras = []; + cachedBypassExtrasFetchedAt = 0; +} + +async function getBypassExtras( + env: RateLimitEnv, + now: number, +): Promise { + if (!env.ROUTER_CONFIG) { + return []; + } + if (now - cachedBypassExtrasFetchedAt < BYPASS_EXTRAS_TTL_MS) { + return cachedBypassExtras; + } + try { + const raw = await env.ROUTER_CONFIG.get( + RATE_LIMIT_CONFIG_KEYS.BYPASS_PREFIXES_EXTRA, + ); + cachedBypassExtras = parseBypassExtras(raw); + cachedBypassExtrasFetchedAt = now; + } catch { + // ROUTER_CONFIG read failures should never break the data plane. + // Keep the last known list (or the empty initial) and try again + // after the TTL. + cachedBypassExtrasFetchedAt = now; + } + return cachedBypassExtras; +} + +/** + * Exported for unit tests. Parses the comma-separated KV value into a + * deduped, trimmed list of prefixes, filtering out anything that + * doesn't look like an absolute path (must start with `/`). Defensive + * parsing — bad operator input shouldn't be able to break the bypass + * check. + */ +export function parseBypassExtras(raw: string | null): readonly string[] { + if (!raw) { + return []; + } + const out: string[] = []; + const seen = new Set(); + for (const entry of raw.split(",")) { + const trimmed = entry.trim(); + if (trimmed.length === 0 || !trimmed.startsWith("/")) { + continue; + } + if (seen.has(trimmed)) { + continue; + } + seen.add(trimmed); + out.push(trimmed); + } + return out; +} + +/** + * Returns true if the path should bypass rate limiting — checks both + * the static `BYPASS_PREFIXES` list and the KV-runtime-tunable + * `RATE_LIMIT_BYPASS_PREFIXES_EXTRA` list. The static check is fast + * (no KV read); the KV list is cached per-isolate for + * `BYPASS_EXTRAS_TTL_MS`. Used by `maybeRateLimit` internally. + * + * @param pathname - the URL pathname to check + * @param env - rate limit env containing the KV bindings + * @param now - wall-clock ms; injectable for tests + */ +export async function isBypassPathDynamic( + pathname: string, + env: RateLimitEnv, + now: number = Date.now(), +): Promise { + if (isBypassPath(pathname)) { + return true; + } + const extras = await getBypassExtras(env, now); + return matchesAnyPrefix(pathname, extras); +} + +// Extract a workspace ID from any of the path shapes both gateways +// see. Matches the literal "workspaces/" segment (Hono routes both +// `/v1/workspaces/:id/...` on the relayfile worker and +// `/cloud/api/v1/workspaces/:id/...` on the cloud router). +function extractWorkspaceId(pathname: string): string | null { + const match = pathname.match(/\/workspaces\/([a-zA-Z0-9_-]+)/); + return match?.[1] ?? null; +} + +// Cheap, non-cryptographic hash for opaque bearer tokens so we can +// shard rate counters by token without ever storing the token. We +// don't need collision resistance against an adversary — we just +// want different tokens to map to different buckets. FNV-1a 32-bit +// fits in a few lines and is fine for this. +function fnv1a32Hex(input: string): string { + let hash = 0x811c9dc5; + for (let i = 0; i < input.length; i++) { + hash ^= input.charCodeAt(i); + // 32-bit multiply, kept unsigned via `>>> 0`. + hash = Math.imul(hash, 0x01000193) >>> 0; + } + return hash.toString(16).padStart(8, "0"); +} + +// The session cookie name used by the web app. Must match +// `SESSION_COOKIE_NAME` in `packages/core/src/session/jwt.ts`. +// Defined inline here to keep the rate-limit module self-contained — +// the router package has no dependency on the web package. +const SESSION_COOKIE_NAME = "agent_relay_session"; + +// Parse a single named cookie out of a Cookie header value. Returns +// null when the cookie is absent or has an empty value. Handles both +// quoted and unquoted values; uses the first match when the name +// appears more than once (which is invalid per RFC 6265 but occurs +// in the wild). +function parseCookieValue(cookieHeader: string, name: string): string | null { + // Regex: look for `; name=` or start-of-string `name=`, then capture + // the value up to the next `;` or end-of-string. + const pattern = new RegExp( + `(?:^|;)\\s*${name}\\s*=\\s*"?([^";]*)"?(?:;|$)`, + ); + const match = cookieHeader.match(pattern); + const value = match?.[1]?.trim() ?? null; + return value && value.length > 0 ? value : null; +} + +// Build a stable limit key from the incoming request. Pure function — +// no network calls, no JWKS verify (too expensive for the hot path). +// Order of precedence reflects which signal is most specific: +// +// 1. workspace ID from the path (covers the actual runaway shape) +// 2. authenticated user via bearer token hash (covers non-workspace +// authenticated traffic — e.g. /api/v1/me) +// 3. session cookie for cookie-authenticated /cloud* traffic. Without +// this, cookie-auth requests have neither a workspace ID nor an +// auth header and collapse to IP buckets — conflating all users +// that share a NAT IP into one rate limit bucket. We hash the +// cookie value with FNV-1a 32 so the rate counter key never +// contains the raw session token. +// 4. CF-connecting-IP (anonymous fallback; X-Forwarded-For untrusted +// because anyone can set it, but `cf-connecting-ip` is set by +// Cloudflare edge and can't be spoofed by the client) +// 5. "anon" — last-resort bucket so a misbehaving anon flood is +// still bounded, just bundled together (acceptable: this is the +// least-trusted traffic anyway) +export function extractLimitKey(request: Request): string { + const url = new URL(request.url); + const workspaceId = extractWorkspaceId(url.pathname); + if (workspaceId) { + return `ws:${workspaceId}`; + } + + const auth = request.headers.get("authorization"); + if (auth) { + // Strip the scheme prefix ("Bearer ", "Basic ", etc.) before + // hashing so the same token doesn't bucket differently when a + // client toggles capitalization or omits the scheme. + const token = auth.replace(/^[A-Za-z]+\s+/, "").trim(); + if (token.length > 0) { + return `user:${fnv1a32Hex(token)}`; + } + } + + const cookieHeader = request.headers.get("cookie"); + if (cookieHeader) { + const sessionToken = parseCookieValue(cookieHeader, SESSION_COOKIE_NAME); + if (sessionToken) { + return `session:${fnv1a32Hex(sessionToken)}`; + } + } + + const ip = + request.headers.get("cf-connecting-ip") ?? + request.headers.get("x-forwarded-for")?.split(",")[0]?.trim() ?? + null; + if (ip) { + return `ip:${ip}`; + } + + return "anon"; +} + +async function readNumberConfig( + kv: KVNamespace | undefined, + configKey: string, + fallback: number, +): Promise { + if (!kv) { + return fallback; + } + try { + const raw = await kv.get(configKey); + if (raw == null) { + return fallback; + } + const parsed = Number(raw); + if (!Number.isFinite(parsed) || parsed <= 0) { + return fallback; + } + return parsed; + } catch { + // ROUTER_CONFIG read failures should never break the data plane — + // we'd rather rate-limit at the default than 500 every request. + return fallback; + } +} + +async function readBooleanConfig( + kv: KVNamespace | undefined, + configKey: string, +): Promise { + if (!kv) { + return false; + } + try { + const raw = await kv.get(configKey); + return raw?.trim().toLowerCase() === "true"; + } catch { + return false; + } +} + +// Increment-and-read with KV. Not atomic — two concurrent requests at +// the same edge can both read N, both write N+1, and we lose a count. +// Accepted per the module-level note. The `now` parameter is injected +// for testability. +async function bumpCounter( + kv: KVNamespace, + bucketKey: string, + now: number, +): Promise { + let prior = 0; + try { + const raw = await kv.get(bucketKey); + if (raw != null) { + const parsed = Number.parseInt(raw, 10); + if (Number.isFinite(parsed) && parsed >= 0) { + prior = parsed; + } + } + } catch { + // Counter-store read failure — treat as zero and continue. + // Failing closed (blocking the request) on a transient KV error + // would amplify a single CF KV hiccup into a customer outage. + prior = 0; + } + + const next = prior + 1; + try { + await kv.put(bucketKey, String(next), { + expirationTtl: COUNTER_TTL_SECONDS, + }); + } catch { + // Same reasoning — log via the surrounding fetch handler if + // observability is needed, but never break the request. + } + void now; // reserved for future jittered TTLs; keeps the signature stable + return next; +} + +function bucketForNow(now: number): number { + return Math.floor(now / 1000 / ONE_MINUTE_SECONDS); +} + +function secondsUntilNextBucket(now: number): number { + const elapsedInMinute = Math.floor(now / 1000) % ONE_MINUTE_SECONDS; + return Math.max(1, ONE_MINUTE_SECONDS - elapsedInMinute); +} + +export interface CheckRateLimitOptions { + // Injected for deterministic tests; defaults to Date.now() in prod. + now?: () => number; +} + +export async function checkRateLimit( + key: string, + env: RateLimitEnv, + options: CheckRateLimitOptions = {}, +): Promise { + const nowFn = options.now ?? (() => Date.now()); + const now = nowFn(); + + const [disabled, perKeyLimit, globalLimit] = await Promise.all([ + readBooleanConfig(env.ROUTER_CONFIG, RATE_LIMIT_CONFIG_KEYS.DISABLED), + readNumberConfig( + env.ROUTER_CONFIG, + RATE_LIMIT_CONFIG_KEYS.PER_KEY_PER_MIN, + DEFAULT_PER_KEY_PER_MIN, + ), + readNumberConfig( + env.ROUTER_CONFIG, + RATE_LIMIT_CONFIG_KEYS.GLOBAL_PER_MIN, + DEFAULT_GLOBAL_PER_MIN, + ), + ]); + + if (disabled || !env.RATE_LIMIT_COUNTERS) { + return { allowed: true, counter: 0, limit: perKeyLimit, key }; + } + + const bucket = bucketForNow(now); + const perKeyBucket = `rl:${key}:${bucket}`; + const globalBucket = `rl:__global__:${bucket}`; + + // Sequential reads/writes — KV doesn't support transactions and + // we want the global counter to include this request even if the + // per-key one rejects. Order is intentional: per-key first so a + // single runaway key doesn't get to skew the global ceiling by + // also incrementing the global bucket *before* its per-key check + // would have rejected it. Net effect: a blocked request still + // contributes 1 to per-key but NOT to global. + const perKeyCount = await bumpCounter( + env.RATE_LIMIT_COUNTERS, + perKeyBucket, + now, + ); + + if (perKeyCount > perKeyLimit) { + return { + allowed: false, + retryAfterSec: secondsUntilNextBucket(now), + counter: perKeyCount, + limit: perKeyLimit, + key, + }; + } + + const globalCount = await bumpCounter( + env.RATE_LIMIT_COUNTERS, + globalBucket, + now, + ); + + if (globalCount > globalLimit) { + return { + allowed: false, + retryAfterSec: secondsUntilNextBucket(now), + counter: globalCount, + limit: globalLimit, + key: "__global__", + }; + } + + return { + allowed: true, + counter: perKeyCount, + limit: perKeyLimit, + key, + }; +} + +// Builds the 429 response. Centralized so both Workers return the +// same shape; clients (relayfile mount, CLI) can rely on it. +export function rateLimitResponse( + retryAfterSec: number, + key: string, +): Response { + const body = JSON.stringify({ + error: "rate_limited", + retryAfter: retryAfterSec, + limitKey: key, + }); + return new Response(body, { + status: 429, + headers: { + "content-type": "application/json", + "retry-after": String(retryAfterSec), + "x-ratelimit-key": key, + }, + }); +} + +// Convenience wrapper that combines bypass check, key extraction, +// the actual limit check, and the 429 response. Both Workers call +// this at the very top of their fetch handler. +// +// Returns: +// * `null` — request is allowed; continue to normal handling +// * `Response` — return this 429 immediately +export async function maybeRateLimit( + request: Request, + env: RateLimitEnv, + options: CheckRateLimitOptions = {}, +): Promise { + const url = new URL(request.url); + // Check both the static `BYPASS_PREFIXES` and the KV-runtime-tunable + // extras (cached 60s per isolate). Static check is fast; KV cache + // means at most one KV read per minute per isolate per Worker. + if (await isBypassPathDynamic(url.pathname, env, options.now?.() ?? Date.now())) { + return null; + } + + const key = extractLimitKey(request); + const decision = await checkRateLimit(key, env, options); + if (decision.allowed) { + return null; + } + return rateLimitResponse(decision.retryAfterSec ?? 60, decision.key); +} diff --git a/router/src/recorder.ts b/router/src/recorder.ts new file mode 100644 index 00000000..be92f1e7 --- /dev/null +++ b/router/src/recorder.ts @@ -0,0 +1,140 @@ +import { redactBody, redactHeaders, shouldRecord } from "./redact.js"; + +const DEFAULT_SAMPLE_RATE = 100; +const MAX_BODY_BYTES = 256 * 1024; +const OVERSIZE_PREFIX = "[OVERSIZE:"; +const textEncoder = new TextEncoder(); + +export interface RecorderEnv { + TRAFFIC_RECORDER: R2Bucket; + ROUTER_CONFIG: KVNamespace; +} + +type RecorderContext = Pick; +type BodyReader = { + body: ReadableStream | null; + bodyUsed: boolean; + headers: Headers; + text(): Promise; +}; + +type TrafficRecord = { + ts: string; + method: string; + path: string; + query: string; + request_headers: Record; + request_body: string | null; + response_status: number; + response_headers: Record; + response_body: string | null; + request_id: string; +}; + +function normalizeSampleRate(value: string | null): number { + const parsed = Number(value); + if (!Number.isFinite(parsed)) { + return DEFAULT_SAMPLE_RATE; + } + + return Math.min(100, Math.max(0, parsed)); +} + +function makeOversizePlaceholder(size: number): string { + return `${OVERSIZE_PREFIX}${size}]`; +} + +async function readBody( + payload: BodyReader | null, + path: string, + contentType: string | null, +): Promise { + if (!payload || payload.body == null || payload.bodyUsed) { + return null; + } + + const body = await payload.text(); + const redactedBody = redactBody(path, body, contentType); + if (redactedBody == null) { + return null; + } + + const bodySize = textEncoder.encode(redactedBody).length; + if (bodySize > MAX_BODY_BYTES) { + return makeOversizePlaceholder(bodySize); + } + + return redactedBody; +} + +function padDatePart(value: number): string { + return String(value).padStart(2, "0"); +} + +function buildRecordKey(timestamp: Date, requestId: string): string { + const year = timestamp.getUTCFullYear(); + const month = padDatePart(timestamp.getUTCMonth() + 1); + const day = padDatePart(timestamp.getUTCDate()); + const hour = padDatePart(timestamp.getUTCHours()); + return `corpus/${year}/${month}/${day}/${hour}/${requestId}.ndjson`; +} + +function getRequestId(request: Request): string { + return ( + request.headers.get("x-request-id") ?? + request.headers.get("cf-ray") ?? + request.headers.get("traceparent") ?? + crypto.randomUUID() + ); +} + +export async function maybeRecord( + request: Request, + response: Response, + env: RecorderEnv, + ctx: RecorderContext, +): Promise { + try { + const url = new URL(request.url); + if (!shouldRecord(url.pathname)) { + return; + } + + const sampleRate = normalizeSampleRate(await env.ROUTER_CONFIG.get("RECORDER_SAMPLE_RATE")); + if (Math.random() * 100 >= sampleRate) { + return; + } + + const requestClone = request.bodyUsed ? null : request.clone(); + const responseClone = response.bodyUsed ? null : response.clone(); + const timestamp = new Date(); + const requestId = getRequestId(request); + + const record: TrafficRecord = { + ts: timestamp.toISOString(), + method: request.method, + path: url.pathname, + query: url.search.startsWith("?") ? url.search.slice(1) : url.search, + request_headers: redactHeaders(request.headers), + request_body: await readBody( + requestClone, + url.pathname, + requestClone?.headers.get("content-type") ?? null, + ), + response_status: response.status, + response_headers: redactHeaders(response.headers), + response_body: await readBody( + responseClone, + url.pathname, + responseClone?.headers.get("content-type") ?? null, + ), + request_id: requestId, + }; + + const line = `${JSON.stringify(record)}\n`; + const key = buildRecordKey(timestamp, requestId); + ctx.waitUntil(env.TRAFFIC_RECORDER.put(key, line)); + } catch (error) { + console.error("traffic recorder failed", error); + } +} diff --git a/router/src/redact.ts b/router/src/redact.ts new file mode 100644 index 00000000..b633f12f --- /dev/null +++ b/router/src/redact.ts @@ -0,0 +1,132 @@ +const HEADER_DENY_LIST = new Set([ + "authorization", + "cookie", + "set-cookie", + "x-relayauth-token", + "x-api-key", +]); + +const SENSITIVE_HEADER_NAME = /(token|secret|password|signature)/i; + +const FULL_BODY_REDACTION_PATHS = [ + /^\/api\/v1\/workspaces\/[a-zA-Z0-9_-]+\/secrets(?:\/.*)?$/, + /^\/api\/auth(?:\/.*)?$/, + /^\/api\/v1\/auth(?:\/.*)?$/, + /^\/api\/v1\/cli\/auth(?:\/.*)?$/, + /^\/api\/v1\/cli\/login(?:\/.*)?$/, + /^\/api\/v1\/integrations\/nango\/connect-link(?:\/.*)?$/, + /^\/api\/v1\/workspaces\/[a-zA-Z0-9_-]+\/integrations\/connect-session(?:\/.*)?$/, +]; + +const JSON_SECRET_KEYS = new Set([ + "token", + "accesstoken", + "refreshtoken", + "secret", + "apikey", + "password", + "clientsecret", + "privatekey", +]); + +type HeaderInput = Headers | Iterable<[string, string]> | Record; + +const textEncoder = new TextEncoder(); + +function utf8ByteLength(value: string): number { + return textEncoder.encode(value).length; +} + +function redactValue(value: string): string { + return `[REDACTED:${utf8ByteLength(value)}]`; +} + +function shouldRedactHeader(name: string): boolean { + const normalized = name.toLowerCase(); + return HEADER_DENY_LIST.has(normalized) || SENSITIVE_HEADER_NAME.test(normalized); +} + +function headerEntries(headers: HeaderInput): Iterable<[string, string]> { + if (headers instanceof Headers) { + const entries: Array<[string, string]> = []; + headers.forEach((value, name) => { + entries.push([name, value]); + }); + return entries; + } + + if (Symbol.iterator in Object(headers)) { + return headers as Iterable<[string, string]>; + } + + return Object.entries(headers); +} + +function shouldFullyRedactBody(path: string): boolean { + return FULL_BODY_REDACTION_PATHS.some((pattern) => pattern.test(path)); +} + +function isJsonContentType(contentType: string | null | undefined): boolean { + return typeof contentType === "string" && contentType.toLowerCase().includes("application/json"); +} + +function redactJsonValue(value: unknown): unknown { + if (Array.isArray(value)) { + return value.map((item) => redactJsonValue(item)); + } + + if (!value || typeof value !== "object") { + return value; + } + + const output: Record = {}; + for (const [key, nested] of Object.entries(value)) { + output[key] = JSON_SECRET_KEYS.has(key.toLowerCase()) + ? redactValue(typeof nested === "string" ? nested : JSON.stringify(nested) ?? "") + : redactJsonValue(nested); + } + + return output; +} + +export function redactHeaders(headers: HeaderInput): Record { + const output: Record = {}; + + for (const [name, value] of headerEntries(headers)) { + output[name] = shouldRedactHeader(name) ? redactValue(value) : value; + } + + return output; +} + +export function redactBody( + path: string, + body: string | null | undefined, + contentType?: string | null, +): string | null | undefined { + if (body == null) { + return body; + } + + if (shouldFullyRedactBody(path)) { + return redactValue(body); + } + + if (!isJsonContentType(contentType)) { + return body; + } + + try { + return JSON.stringify(redactJsonValue(JSON.parse(body))); + } catch { + return body; + } +} + +export function shouldRecord(path: string): boolean { + if (path === "/favicon.ico" || path === "/api/health") { + return false; + } + + return path !== "/observer" && !path.startsWith("/observer/"); +} diff --git a/router/test/rate-limit.test.ts b/router/test/rate-limit.test.ts new file mode 100644 index 00000000..369f55b4 --- /dev/null +++ b/router/test/rate-limit.test.ts @@ -0,0 +1,588 @@ +import { beforeEach, describe, expect, it } from "vitest"; + +import { + __resetBypassExtrasCacheForTests, + checkRateLimit, + extractLimitKey, + isBypassPath, + isBypassPathDynamic, + maybeRateLimit, + parseBypassExtras, + rateLimitResponse, + RATE_LIMIT_CONFIG_KEYS, + type RateLimitEnv, +} from "../src/rate-limit.js"; + +// Tiny in-memory KV shim. Counter increments only need get/put; we +// also store an optional `expirationTtl` so a future test could +// assert TTLs, though the current cases don't depend on it. +type StoredEntry = { value: string; expiresAt: number | null }; + +class MemoryKV { + readonly store = new Map(); + private nowFn: () => number; + + constructor(nowFn: () => number = () => Date.now()) { + this.nowFn = nowFn; + } + + async get(key: string): Promise { + const entry = this.store.get(key); + if (!entry) return null; + if (entry.expiresAt !== null && entry.expiresAt <= this.nowFn()) { + this.store.delete(key); + return null; + } + return entry.value; + } + + async put( + key: string, + value: string, + options?: { expirationTtl?: number }, + ): Promise { + const ttl = options?.expirationTtl; + const expiresAt = + typeof ttl === "number" && ttl > 0 ? this.nowFn() + ttl * 1000 : null; + this.store.set(key, { value, expiresAt }); + } + + // Helpers — not part of the KVNamespace surface. + size(): number { + return this.store.size; + } +} + +function asKV(kv: MemoryKV): KVNamespace { + return kv as unknown as KVNamespace; +} + +function buildEnv(opts: { + counters?: MemoryKV; + config?: MemoryKV; +}): RateLimitEnv { + return { + RATE_LIMIT_COUNTERS: opts.counters ? asKV(opts.counters) : undefined, + ROUTER_CONFIG: opts.config ? asKV(opts.config) : undefined, + }; +} + +const FIXED_NOW = 1715683680000; // 2024-05-14T11:28:00Z — middle of an even minute +const fixedNow = () => FIXED_NOW; + +describe("isBypassPath", () => { + it.each([ + ["/api/health", true], + ["/api/health/db", true], + ["/health", true], + ["/observer", true], + ["/observer/session/abc", true], + ["/_next/static/chunks/123.js", true], + ["/favicon.ico", true], + ["/v1/workspaces/ws_123/sync/status", false], + ["/cloud/api/v1/workspaces/ws_123/sandboxes", false], + ["/", false], + ])("path %s bypass=%s", (path, expected) => { + expect(isBypassPath(path)).toBe(expected); + }); +}); + +describe("extractLimitKey", () => { + it("picks workspace id from /v1/workspaces/:id/...", () => { + const req = new Request( + "https://api.relayfile.dev/v1/workspaces/ws_abc123/sync/status", + ); + expect(extractLimitKey(req)).toBe("ws:ws_abc123"); + }); + + it("picks workspace id from /cloud/api/v1/workspaces/:id/...", () => { + const req = new Request( + "https://agentrelay.com/cloud/api/v1/workspaces/ws_xyz/sandboxes", + ); + expect(extractLimitKey(req)).toBe("ws:ws_xyz"); + }); + + it("hashes bearer token when no workspace path is present", () => { + const req = new Request("https://agentrelay.com/cloud/api/v1/me", { + headers: { authorization: "Bearer abcdef0123456789" }, + }); + const key = extractLimitKey(req); + expect(key).toMatch(/^user:[0-9a-f]{8}$/); + }); + + it("buckets different tokens to different user keys", () => { + const a = extractLimitKey( + new Request("https://agentrelay.com/cloud/api/v1/me", { + headers: { authorization: "Bearer aaaaaaaa" }, + }), + ); + const b = extractLimitKey( + new Request("https://agentrelay.com/cloud/api/v1/me", { + headers: { authorization: "Bearer bbbbbbbb" }, + }), + ); + expect(a).not.toBe(b); + expect(a).toMatch(/^user:/); + expect(b).toMatch(/^user:/); + }); + + it("falls back to cf-connecting-ip for anonymous traffic", () => { + const req = new Request("https://agentrelay.com/", { + headers: { "cf-connecting-ip": "203.0.113.5" }, + }); + expect(extractLimitKey(req)).toBe("ip:203.0.113.5"); + }); + + it("prefers cf-connecting-ip over x-forwarded-for", () => { + const req = new Request("https://agentrelay.com/", { + headers: { + "cf-connecting-ip": "203.0.113.5", + "x-forwarded-for": "10.0.0.1, 10.0.0.2", + }, + }); + expect(extractLimitKey(req)).toBe("ip:203.0.113.5"); + }); + + it("uses x-forwarded-for first hop when cf-connecting-ip is missing", () => { + const req = new Request("https://agentrelay.com/", { + headers: { "x-forwarded-for": "10.0.0.1, 10.0.0.2" }, + }); + expect(extractLimitKey(req)).toBe("ip:10.0.0.1"); + }); + + it("keys off session cookie for /cloud* traffic with no auth header", () => { + const req = new Request("https://agentrelay.com/cloud/api/v1/me", { + headers: { cookie: "agent_relay_session=mysecrettoken123" }, + }); + const key = extractLimitKey(req); + expect(key).toMatch(/^session:[0-9a-f]{8}$/); + }); + + it("auth header takes priority over session cookie", () => { + const req = new Request("https://agentrelay.com/cloud/api/v1/me", { + headers: { + authorization: "Bearer bearertoken999", + cookie: "agent_relay_session=mysecrettoken123", + }, + }); + const key = extractLimitKey(req); + // auth header wins — produces a user: key, not session: + expect(key).toMatch(/^user:[0-9a-f]{8}$/); + }); + + it("buckets different session cookie values to different session keys", () => { + const a = extractLimitKey( + new Request("https://agentrelay.com/cloud/api/v1/me", { + headers: { cookie: "agent_relay_session=tokenA" }, + }), + ); + const b = extractLimitKey( + new Request("https://agentrelay.com/cloud/api/v1/me", { + headers: { cookie: "agent_relay_session=tokenB" }, + }), + ); + expect(a).not.toBe(b); + expect(a).toMatch(/^session:/); + expect(b).toMatch(/^session:/); + }); + + it("/cloud* with no cookie and no auth falls back to IP", () => { + const req = new Request("https://agentrelay.com/cloud/api/v1/me", { + headers: { "cf-connecting-ip": "198.51.100.1" }, + }); + expect(extractLimitKey(req)).toBe("ip:198.51.100.1"); + }); + + it("/cloud* with no cookie, no auth, no IP falls back to anon", () => { + const req = new Request("https://agentrelay.com/cloud/api/v1/me"); + expect(extractLimitKey(req)).toBe("anon"); + }); + + it("falls back to anon when no signal is present", () => { + const req = new Request("https://agentrelay.com/"); + expect(extractLimitKey(req)).toBe("anon"); + }); +}); + +describe("checkRateLimit", () => { + let counters: MemoryKV; + let config: MemoryKV; + let env: RateLimitEnv; + + beforeEach(() => { + counters = new MemoryKV(fixedNow); + config = new MemoryKV(fixedNow); + env = buildEnv({ counters, config }); + }); + + it("allows requests under the per-key limit", async () => { + for (let i = 1; i <= 5; i++) { + const result = await checkRateLimit("ws:demo", env, { now: fixedNow }); + expect(result.allowed).toBe(true); + expect(result.counter).toBe(i); + } + }); + + it("returns 429 once the per-key limit is exceeded", async () => { + // Pin the per-workspace limit low for deterministic assertions. + await config.put(RATE_LIMIT_CONFIG_KEYS.PER_KEY_PER_MIN, "3"); + + // Sequential calls — the in-memory KV shim mirrors a single edge + // processing requests in order. KV's eventual consistency across + // edges is acknowledged in the module-level docstring but is not + // what this test asserts. + const results = [ + await checkRateLimit("ws:demo", env, { now: fixedNow }), + await checkRateLimit("ws:demo", env, { now: fixedNow }), + await checkRateLimit("ws:demo", env, { now: fixedNow }), + ]; + expect(results.map((r) => r.allowed)).toEqual([true, true, true]); + + const blocked = await checkRateLimit("ws:demo", env, { now: fixedNow }); + expect(blocked.allowed).toBe(false); + expect(blocked.counter).toBe(4); + expect(blocked.limit).toBe(3); + expect(blocked.retryAfterSec).toBeGreaterThan(0); + expect(blocked.retryAfterSec).toBeLessThanOrEqual(60); + expect(blocked.key).toBe("ws:demo"); + }); + + it("does not pollute the global counter on a per-key reject", async () => { + await config.put(RATE_LIMIT_CONFIG_KEYS.PER_KEY_PER_MIN, "1"); + await checkRateLimit("ws:a", env, { now: fixedNow }); + const blocked = await checkRateLimit("ws:a", env, { now: fixedNow }); + expect(blocked.allowed).toBe(false); + + // Verify the global counter only ever incremented once (for the + // single allowed request) — the blocked one should NOT have + // incremented it. This is intentional so a single misbehaving + // key can't push the global ceiling around. + const bucket = Math.floor(FIXED_NOW / 1000 / 60); + const stored = await counters.get(`rl:__global__:${bucket}`); + expect(stored).toBe("1"); + }); + + it("returns 429 once the global limit is exceeded", async () => { + await config.put(RATE_LIMIT_CONFIG_KEYS.GLOBAL_PER_MIN, "2"); + await config.put(RATE_LIMIT_CONFIG_KEYS.PER_KEY_PER_MIN, "10"); + + expect( + (await checkRateLimit("ws:a", env, { now: fixedNow })).allowed, + ).toBe(true); + expect( + (await checkRateLimit("ws:b", env, { now: fixedNow })).allowed, + ).toBe(true); + + const blocked = await checkRateLimit("ws:c", env, { now: fixedNow }); + expect(blocked.allowed).toBe(false); + // Global rejects surface the synthetic `__global__` key so the + // 429 body tells operators it's the global ceiling, not the + // per-key one. + expect(blocked.key).toBe("__global__"); + }); + + it("respects RATE_LIMIT_DISABLED kill switch", async () => { + await config.put(RATE_LIMIT_CONFIG_KEYS.PER_KEY_PER_MIN, "1"); + await config.put(RATE_LIMIT_CONFIG_KEYS.DISABLED, "true"); + + // Even though the limit is 1, the kill switch should let + // unlimited requests through. + for (let i = 0; i < 5; i++) { + const result = await checkRateLimit("ws:demo", env, { now: fixedNow }); + expect(result.allowed).toBe(true); + } + // No counter writes when disabled — saves KV operations. + expect(counters.size()).toBe(0); + }); + + it("resets per minute (different bucket)", async () => { + await config.put(RATE_LIMIT_CONFIG_KEYS.PER_KEY_PER_MIN, "2"); + + const firstMinute = fixedNow; + const secondMinute = () => FIXED_NOW + 60_000; + + expect( + (await checkRateLimit("ws:demo", env, { now: firstMinute })).allowed, + ).toBe(true); + expect( + (await checkRateLimit("ws:demo", env, { now: firstMinute })).allowed, + ).toBe(true); + expect( + (await checkRateLimit("ws:demo", env, { now: firstMinute })).allowed, + ).toBe(false); + + // Roll forward one full minute — bucket key changes, counter + // starts over. + const reset = await checkRateLimit("ws:demo", env, { now: secondMinute }); + expect(reset.allowed).toBe(true); + expect(reset.counter).toBe(1); + }); + + it("falls back to defaults when ROUTER_CONFIG is unbound", async () => { + const noConfigEnv = buildEnv({ counters }); + const result = await checkRateLimit("ws:demo", noConfigEnv, { + now: fixedNow, + }); + expect(result.allowed).toBe(true); + expect(result.limit).toBe(60); // DEFAULT_PER_KEY_PER_MIN + }); + + it("treats missing RATE_LIMIT_COUNTERS as allow-all (fail open)", async () => { + const noCounterEnv = buildEnv({ config }); + await config.put(RATE_LIMIT_CONFIG_KEYS.PER_KEY_PER_MIN, "1"); + // Without a counter store we cannot enforce, so behavior is + // intentionally permissive (the limit is documented as + // best-effort). This guards against a missing binding silently + // breaking the data plane. + for (let i = 0; i < 10; i++) { + expect( + (await checkRateLimit("ws:demo", noCounterEnv, { now: fixedNow })) + .allowed, + ).toBe(true); + } + }); +}); + +describe("rateLimitResponse", () => { + it("emits a 429 with the documented shape", async () => { + const res = rateLimitResponse(42, "ws:demo"); + expect(res.status).toBe(429); + expect(res.headers.get("retry-after")).toBe("42"); + expect(res.headers.get("x-ratelimit-key")).toBe("ws:demo"); + expect(res.headers.get("content-type")).toBe("application/json"); + const body = (await res.json()) as Record; + expect(body).toEqual({ + error: "rate_limited", + retryAfter: 42, + limitKey: "ws:demo", + }); + }); +}); + +describe("maybeRateLimit", () => { + it("returns null for bypass paths without touching KV", async () => { + const counters = new MemoryKV(fixedNow); + const config = new MemoryKV(fixedNow); + const env = buildEnv({ counters, config }); + + const result = await maybeRateLimit( + new Request("https://api.relayfile.dev/health"), + env, + { now: fixedNow }, + ); + expect(result).toBeNull(); + expect(counters.size()).toBe(0); + }); + + it("returns null when under the limit", async () => { + const counters = new MemoryKV(fixedNow); + const config = new MemoryKV(fixedNow); + const env = buildEnv({ counters, config }); + + const result = await maybeRateLimit( + new Request("https://api.relayfile.dev/v1/workspaces/ws_a/sync/status"), + env, + { now: fixedNow }, + ); + expect(result).toBeNull(); + }); + + it("returns a 429 once limit is exceeded", async () => { + const counters = new MemoryKV(fixedNow); + const config = new MemoryKV(fixedNow); + const env = buildEnv({ counters, config }); + await config.put(RATE_LIMIT_CONFIG_KEYS.PER_KEY_PER_MIN, "1"); + + const url = + "https://api.relayfile.dev/v1/workspaces/ws_a/sync/status"; + const first = await maybeRateLimit(new Request(url), env, { + now: fixedNow, + }); + expect(first).toBeNull(); + + const blocked = await maybeRateLimit(new Request(url), env, { + now: fixedNow, + }); + expect(blocked).not.toBeNull(); + expect(blocked!.status).toBe(429); + expect(blocked!.headers.get("x-ratelimit-key")).toBe("ws:ws_a"); + }); +}); + +describe("parseBypassExtras", () => { + it("returns empty array for null / empty input", () => { + expect(parseBypassExtras(null)).toEqual([]); + expect(parseBypassExtras("")).toEqual([]); + expect(parseBypassExtras(" ")).toEqual([]); + }); + + it("parses comma-separated absolute paths", () => { + expect(parseBypassExtras("/api/k8s-readiness,/internal/uptime")).toEqual([ + "/api/k8s-readiness", + "/internal/uptime", + ]); + }); + + it("trims whitespace around entries", () => { + expect(parseBypassExtras(" /a , /b , /c ")).toEqual(["/a", "/b", "/c"]); + }); + + it("ignores entries that don't start with /", () => { + // Defensive: bad operator input shouldn't poison the list. + expect(parseBypassExtras("/ok,not-a-path,/also-ok")).toEqual([ + "/ok", + "/also-ok", + ]); + }); + + it("dedupes repeated entries", () => { + expect(parseBypassExtras("/x,/x,/y,/x")).toEqual(["/x", "/y"]); + }); +}); + +describe("isBypassPathDynamic", () => { + beforeEach(() => { + __resetBypassExtrasCacheForTests(); + }); + + it("returns true for static bypass paths without touching KV", async () => { + const env = buildEnv({}); // no ROUTER_CONFIG bound + expect(await isBypassPathDynamic("/api/health", env, FIXED_NOW)).toBe(true); + expect(await isBypassPathDynamic("/observer/foo", env, FIXED_NOW)).toBe( + true, + ); + }); + + it("returns false for non-bypass paths when no KV extras set", async () => { + const config = new MemoryKV(fixedNow); + const env = buildEnv({ config }); + expect( + await isBypassPathDynamic("/v1/workspaces/ws_a/sync/status", env, FIXED_NOW), + ).toBe(false); + }); + + it("respects KV-configured extras (exact match)", async () => { + const config = new MemoryKV(fixedNow); + const env = buildEnv({ config }); + await config.put( + RATE_LIMIT_CONFIG_KEYS.BYPASS_PREFIXES_EXTRA, + "/api/k8s-readiness", + ); + expect( + await isBypassPathDynamic("/api/k8s-readiness", env, FIXED_NOW), + ).toBe(true); + }); + + it("respects KV-configured extras (prefix match)", async () => { + const config = new MemoryKV(fixedNow); + const env = buildEnv({ config }); + await config.put( + RATE_LIMIT_CONFIG_KEYS.BYPASS_PREFIXES_EXTRA, + "/internal/uptime", + ); + expect( + await isBypassPathDynamic("/internal/uptime/check", env, FIXED_NOW), + ).toBe(true); + }); + + it("caches KV reads for 60s (does not re-read on every call)", async () => { + const config = new MemoryKV(fixedNow); + const env = buildEnv({ config }); + await config.put( + RATE_LIMIT_CONFIG_KEYS.BYPASS_PREFIXES_EXTRA, + "/api/k8s-readiness", + ); + + // Count KV reads by wrapping the real one. + let reads = 0; + const originalGet = env.ROUTER_CONFIG!.get.bind(env.ROUTER_CONFIG!); + env.ROUTER_CONFIG!.get = ((k: string) => { + reads++; + return originalGet(k); + }) as NonNullable["get"]; + + // First call hits KV. + await isBypassPathDynamic("/v1/x", env, FIXED_NOW); + expect(reads).toBe(1); + // Second call within 60s reuses the cache. + await isBypassPathDynamic("/v1/y", env, FIXED_NOW + 30_000); + expect(reads).toBe(1); + // Past the TTL boundary, fetches again. + await isBypassPathDynamic("/v1/z", env, FIXED_NOW + 61_000); + expect(reads).toBe(2); + }); + + it("falls back to last-known list on KV transient errors", async () => { + const config = new MemoryKV(fixedNow); + const env = buildEnv({ config }); + await config.put( + RATE_LIMIT_CONFIG_KEYS.BYPASS_PREFIXES_EXTRA, + "/cached-ok", + ); + + // Warm the cache once. + await isBypassPathDynamic("/v1/x", env, FIXED_NOW); + expect(await isBypassPathDynamic("/cached-ok", env, FIXED_NOW)).toBe(true); + + // Now break the KV. After TTL expiry the read will throw — we + // should fall back to the cached value rather than crashing. + env.ROUTER_CONFIG!.get = (() => { + throw new Error("KV unavailable"); + }) as NonNullable["get"]; + + expect( + await isBypassPathDynamic("/cached-ok", env, FIXED_NOW + 120_000), + ).toBe(true); + }); + + it("static and dynamic bypass coexist (static still works with KV extras present)", async () => { + const config = new MemoryKV(fixedNow); + const env = buildEnv({ config }); + await config.put( + RATE_LIMIT_CONFIG_KEYS.BYPASS_PREFIXES_EXTRA, + "/internal/uptime", + ); + // Static + dynamic both bypass. + expect(await isBypassPathDynamic("/api/health", env, FIXED_NOW)).toBe(true); + expect( + await isBypassPathDynamic("/internal/uptime", env, FIXED_NOW), + ).toBe(true); + }); +}); + +describe("maybeRateLimit honors KV bypass extras", () => { + beforeEach(() => { + __resetBypassExtrasCacheForTests(); + }); + + it("bypasses a runtime-whitelisted path", async () => { + const counters = new MemoryKV(fixedNow); + const config = new MemoryKV(fixedNow); + const env = buildEnv({ counters, config }); + // Tight limit so anything not bypassed would 429 on call #2. + await config.put(RATE_LIMIT_CONFIG_KEYS.PER_KEY_PER_MIN, "1"); + await config.put( + RATE_LIMIT_CONFIG_KEYS.BYPASS_PREFIXES_EXTRA, + "/api/k8s-readiness", + ); + + // Hit a non-bypassed path twice → second call 429s. + const url = "https://api.relayfile.dev/v1/workspaces/ws_a/sync/status"; + await maybeRateLimit(new Request(url), env, { now: fixedNow }); + const blocked = await maybeRateLimit(new Request(url), env, { + now: fixedNow, + }); + expect(blocked!.status).toBe(429); + + // Hit the runtime-whitelisted path 5 times → never 429s, regardless + // of the very tight per-key limit. + for (let i = 0; i < 5; i++) { + const r = await maybeRateLimit( + new Request("https://api.relayfile.dev/api/k8s-readiness"), + env, + { now: fixedNow }, + ); + expect(r).toBeNull(); + } + }); +}); diff --git a/router/test/recorder.test.ts b/router/test/recorder.test.ts new file mode 100644 index 00000000..bd59dc9f --- /dev/null +++ b/router/test/recorder.test.ts @@ -0,0 +1,192 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +import { maybeRecord, type RecorderEnv } from "../src/recorder.js"; + +const FIXED_TIME = new Date("2026-05-14T09:08:07.000Z"); +const REQUEST_ID = "123e4567-e89b-12d3-a456-426614174000"; +const KEY_PATTERN = /^corpus\/\d{4}\/\d{2}\/\d{2}\/\d{2}\/[a-f0-9-]+\.ndjson$/; + +function createEnv(sampleRate: string): { + env: RecorderEnv; + put: ReturnType; + get: ReturnType; +} { + const put = vi.fn(async () => undefined); + const get = vi.fn(async () => sampleRate); + + return { + env: { + TRAFFIC_RECORDER: { + put, + } as unknown as R2Bucket, + ROUTER_CONFIG: { + get, + } as unknown as KVNamespace, + }, + put, + get, + }; +} + +function createCtx(): { + ctx: Pick; + waitUntil: ReturnType; + waits: Promise[]; +} { + const waits: Promise[] = []; + const waitUntil = vi.fn((promise: Promise) => { + waits.push(promise); + }); + + return { + ctx: { waitUntil }, + waitUntil, + waits, + }; +} + +describe("maybeRecord", () => { + beforeEach(() => { + vi.useFakeTimers(); + vi.setSystemTime(FIXED_TIME); + vi.spyOn(Math, "random").mockReturnValue(0); + }); + + afterEach(() => { + vi.restoreAllMocks(); + vi.useRealTimers(); + }); + + it("writes a record when sampling is 100 percent", async () => { + const { env, put, get } = createEnv("100"); + const { ctx, waitUntil, waits } = createCtx(); + const request = new Request("https://router.test/api/v1/messages?foo=bar", { + method: "POST", + headers: { + "content-type": "application/json", + authorization: "Bearer secret-token", + "x-request-id": REQUEST_ID, + }, + body: JSON.stringify({ + prompt: "hello", + token: "secret-token", + }), + }); + const response = new Response( + JSON.stringify({ + ok: true, + refreshToken: "refresh-me", + }), + { + status: 201, + headers: { + "content-type": "application/json", + "set-cookie": "sid=123", + }, + }, + ); + + await maybeRecord(request, response, env, ctx); + await Promise.all(waits); + + expect(get).toHaveBeenCalledWith("RECORDER_SAMPLE_RATE"); + expect(waitUntil).toHaveBeenCalledOnce(); + expect(put).toHaveBeenCalledOnce(); + + const [key, line] = put.mock.calls[0] as [string, string]; + expect(key).toBe("corpus/2026/05/14/09/123e4567-e89b-12d3-a456-426614174000.ndjson"); + expect(key).toMatch(KEY_PATTERN); + + const record = JSON.parse(line) as Record; + expect(record).toEqual({ + ts: "2026-05-14T09:08:07.000Z", + method: "POST", + path: "/api/v1/messages", + query: "foo=bar", + request_headers: { + authorization: "[REDACTED:19]", + "content-type": "application/json", + "x-request-id": REQUEST_ID, + }, + request_body: JSON.stringify({ + prompt: "hello", + token: "[REDACTED:12]", + }), + response_status: 201, + response_headers: { + "content-type": "application/json", + "set-cookie": "[REDACTED:7]", + }, + response_body: JSON.stringify({ + ok: true, + refreshToken: "[REDACTED:10]", + }), + request_id: REQUEST_ID, + }); + }); + + it("does not write when sampling is 0 percent", async () => { + const { env, put } = createEnv("0"); + const { ctx, waitUntil } = createCtx(); + + await maybeRecord( + new Request("https://router.test/api/v1/messages", { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ ok: true }), + }), + new Response("ok"), + env, + ctx, + ); + + expect(waitUntil).not.toHaveBeenCalled(); + expect(put).not.toHaveBeenCalled(); + }); + + it("skips deny-listed paths without touching storage", async () => { + const { env, put, get } = createEnv("100"); + const { ctx, waitUntil } = createCtx(); + + await maybeRecord( + new Request("https://router.test/observer/session/123"), + new Response("ok"), + env, + ctx, + ); + + expect(get).not.toHaveBeenCalled(); + expect(waitUntil).not.toHaveBeenCalled(); + expect(put).not.toHaveBeenCalled(); + }); + + it("replaces oversize bodies with a placeholder before writing", async () => { + const { env, put } = createEnv("100"); + const { ctx, waits } = createCtx(); + const oversized = "x".repeat(256 * 1024 + 1); + + await maybeRecord( + new Request("https://router.test/api/v1/messages", { + method: "POST", + headers: { + "content-type": "text/plain", + "x-request-id": REQUEST_ID, + }, + body: oversized, + }), + new Response("ok", { + headers: { "content-type": "text/plain" }, + }), + env, + ctx, + ); + await Promise.all(waits); + + expect(put).toHaveBeenCalledOnce(); + const [, line] = put.mock.calls[0] as [string, string]; + const record = JSON.parse(line) as { request_body: string; response_body: string }; + + expect(record.request_body).toBe("[OVERSIZE:262145]"); + expect(record.response_body).toBe("ok"); + }); +}); diff --git a/router/test/redact.test.ts b/router/test/redact.test.ts new file mode 100644 index 00000000..f5302c2d --- /dev/null +++ b/router/test/redact.test.ts @@ -0,0 +1,78 @@ +import { describe, expect, it } from "vitest"; + +import { redactBody, redactHeaders, shouldRecord } from "../src/redact.js"; + +describe("router redaction", () => { + it("redacts deny-listed and sensitive headers", () => { + const headers = redactHeaders({ + authorization: "Bearer secret-token", + "x-api-key": "topsecret", + "x-signature": "sig-123", + "content-type": "application/json", + }); + + expect(headers).toEqual({ + authorization: "[REDACTED:19]", + "x-api-key": "[REDACTED:9]", + "x-signature": "[REDACTED:7]", + "content-type": "application/json", + }); + }); + + it("fully redacts bodies for deny-listed paths", () => { + expect( + redactBody( + "/api/v1/workspaces/ws_123/secrets", + JSON.stringify({ apiKey: "secret-value" }), + "application/json", + ), + ).toBe("[REDACTED:25]"); + }); + + it("scrubs sensitive keys from JSON bodies", () => { + expect( + redactBody( + "/api/v1/messages", + JSON.stringify({ + ok: true, + nested: { + token: "secret-token", + profile: { + password: "swordfish", + }, + }, + items: [ + { + refreshToken: "refresh-me", + }, + ], + }), + "application/json; charset=utf-8", + ), + ).toBe( + JSON.stringify({ + ok: true, + nested: { + token: "[REDACTED:12]", + profile: { + password: "[REDACTED:9]", + }, + }, + items: [ + { + refreshToken: "[REDACTED:10]", + }, + ], + }), + ); + }); + + it("applies shouldRecord route rules", () => { + expect(shouldRecord("/favicon.ico")).toBe(false); + expect(shouldRecord("/api/health")).toBe(false); + expect(shouldRecord("/observer")).toBe(false); + expect(shouldRecord("/observer/session/123")).toBe(false); + expect(shouldRecord("/observer-files")).toBe(true); + expect(shouldRecord("/cloud/dashboard")).toBe(true); + }); +}); diff --git a/router/test/webhook-routing.test.ts b/router/test/webhook-routing.test.ts new file mode 100644 index 00000000..8daa494d --- /dev/null +++ b/router/test/webhook-routing.test.ts @@ -0,0 +1,157 @@ +import { describe, expect, it, vi } from "vitest"; + +import worker from "../index.js"; + +type StoredEntry = { value: string }; + +class MemoryKV { + readonly store = new Map(); + + async get(key: string): Promise { + return this.store.get(key)?.value ?? null; + } + + async put(key: string, value: string): Promise { + this.store.set(key, { value }); + } +} + +function asKV(kv: MemoryKV): KVNamespace { + return kv as unknown as KVNamespace; +} + +type WorkerBinding = { fetch: ReturnType }; + +function makeBinding(): WorkerBinding { + return { + fetch: vi.fn(async () => new Response("ok", { status: 200 })), + }; +} + +function buildEnv(opts: { + webhookOriginFlag?: string; + cloudWebWorker?: WorkerBinding; + webhookWorker?: WorkerBinding; +}) { + const routerConfig = new MemoryKV(); + if (opts.webhookOriginFlag !== undefined) { + routerConfig.store.set("WEBHOOK_ORIGIN", { value: opts.webhookOriginFlag }); + } + return { + CLOUD_APP_ORIGIN: "https://origin.test.invalid", + ROUTER_CONFIG: asKV(routerConfig), + CLOUD_WEB_WORKER: opts.cloudWebWorker, + WEBHOOK_WORKER: opts.webhookWorker, + } as unknown as Parameters[1]; +} + +function buildCtx(): ExecutionContext { + return { + waitUntil: () => undefined, + passThroughOnException: () => undefined, + } as unknown as ExecutionContext; +} + +describe("router webhook routing", () => { + it("routes /api/v1/webhooks/nango to the webhook-worker binding when WEBHOOK_ORIGIN=worker", async () => { + const cloudWebWorker = makeBinding(); + const webhookWorker = makeBinding(); + const env = buildEnv({ + webhookOriginFlag: "worker", + cloudWebWorker, + webhookWorker, + }); + + const request = new Request( + "https://origin.agentrelay.cloud/cloud/api/v1/webhooks/nango", + { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ type: "forward", from: "github-app-oauth" }), + }, + ); + + await worker.fetch(request, env, buildCtx()); + + expect(webhookWorker.fetch).toHaveBeenCalledOnce(); + expect(cloudWebWorker.fetch).not.toHaveBeenCalled(); + }); + + it("bypasses webhook-worker when request carries x-cloud-webhook-worker-forwarded header (loop break)", async () => { + const cloudWebWorker = makeBinding(); + const webhookWorker = makeBinding(); + const env = buildEnv({ + webhookOriginFlag: "worker", + cloudWebWorker, + webhookWorker, + }); + + const request = new Request( + "https://origin.agentrelay.cloud/cloud/api/v1/webhooks/nango", + { + method: "POST", + headers: { + "content-type": "application/json", + "x-cloud-webhook-worker-forwarded": "webhook-worker", + "x-cloud-webhook-worker-request-id": "req-loop-1", + }, + body: JSON.stringify({ type: "forward", from: "github-app-oauth" }), + }, + ); + + await worker.fetch(request, env, buildCtx()); + + expect(cloudWebWorker.fetch).toHaveBeenCalledOnce(); + expect(webhookWorker.fetch).not.toHaveBeenCalled(); + }); + + it("ignores the loop-break header for an unrelated value (don't accidentally widen bypass)", async () => { + const cloudWebWorker = makeBinding(); + const webhookWorker = makeBinding(); + const env = buildEnv({ + webhookOriginFlag: "worker", + cloudWebWorker, + webhookWorker, + }); + + const request = new Request( + "https://origin.agentrelay.cloud/cloud/api/v1/webhooks/nango", + { + method: "POST", + headers: { + "content-type": "application/json", + "x-cloud-webhook-worker-forwarded": "something-else", + }, + body: JSON.stringify({ type: "forward", from: "github-app-oauth" }), + }, + ); + + await worker.fetch(request, env, buildCtx()); + + expect(webhookWorker.fetch).toHaveBeenCalledOnce(); + expect(cloudWebWorker.fetch).not.toHaveBeenCalled(); + }); + + it("routes to cloud-web when WEBHOOK_ORIGIN flag is unset (regression: still works without flag)", async () => { + const cloudWebWorker = makeBinding(); + const webhookWorker = makeBinding(); + const env = buildEnv({ + cloudWebWorker, + webhookWorker, + }); + + const request = new Request( + "https://origin.agentrelay.cloud/cloud/api/v1/webhooks/nango", + { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ type: "forward", from: "github-app-oauth" }), + }, + ); + + await worker.fetch(request, env, buildCtx()); + + expect(cloudWebWorker.fetch).toHaveBeenCalledOnce(); + expect(webhookWorker.fetch).not.toHaveBeenCalled(); + }); +}); diff --git a/router/tsconfig.json b/router/tsconfig.json new file mode 100644 index 00000000..00c428ac --- /dev/null +++ b/router/tsconfig.json @@ -0,0 +1,17 @@ +{ + "compilerOptions": { + "target": "ES2022", + "module": "ES2022", + "moduleResolution": "bundler", + "lib": ["ES2022", "WebWorker"], + "strict": true, + "noEmit": true, + "skipLibCheck": true, + "esModuleInterop": true, + "resolveJsonModule": true, + "isolatedModules": true, + "types": ["@cloudflare/workers-types", "vitest/globals"] + }, + "include": ["index.ts", "src/**/*.ts", "test/**/*.ts"], + "exclude": ["node_modules", "dist"] +} From b82df264d37344a821d7f443b0dea95a32fcf194 Mon Sep 17 00:00:00 2001 From: Khaliq Date: Sun, 14 Jun 2026 23:20:41 +0200 Subject: [PATCH 42/49] =?UTF-8?q?feat(transcripts):=20recorder=20transcrip?= =?UTF-8?q?t=20ingest=20=E2=86=92=20workspace=20VFS=20=E2=86=92=20meeting-?= =?UTF-8?q?actions=20(#2158)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat(transcripts): wire recorder transcript ingest → workspace VFS → meeting-actions Closes the last-mile core gap: the desktop recorder's bot-free path transcribes on the Modal worker but had nowhere to deliver the note (TRANSCRIPTS_INGEST_URL had no receiver), so customer transcripts never reached meeting-actions. - New delivery-only route POST /api/v1/webhooks/transcripts: receives the transcription-worker's already-transcribed granola note, writes it to the customer workspace VFS at /recall/recordings/.json, and dispatches the meeting-actions watch event. Transcription stays on the Modal worker. - Refactored the recall-hookdeck-webhook delivery half into a shared `deliverRecallTranscriptNote()` + exported `resolveRecallWorkspaceId()`; the existing hookdeck path is behavior-unchanged (still transcribes inline). - Auth: reuses the existing RecorderTranscribeToken (Bearer) — worker's ingestTranscript now sends it; route verifies it. Moved the secret decl to infra/secrets.ts so both the worker and web runtime resolve the same value. - Workspace routing: single-tenant via RECALL_WORKSPACE_ID (wired to the web worker + deploy var), else first recall-connected workspace. FOLLOW-UP (durable multi-tenant, tracked): the recorder now knows its signed-in workspace (cli:auth login, watchdog-recorder#19) — tag the SDK upload with the customer rw_ id, propagate it into the worker note, and route the ingest on it, removing the single-tenant RECALL_WORKSPACE_ID pin. Human-set values after merge+deploy (Khaliq): gh variable set TRANSCRIPTS_INGEST_URL --repo AgentWorkforce/cloud \ --env production --body https://agentrelay.com/cloud/api/v1/webhooks/transcripts gh variable set RECALL_WORKSPACE_ID --repo AgentWorkforce/cloud \ --env production --body <Ålesund customer rw_ id> Verified: route unit test 7/7 + existing recall-hookdeck svix tests 11/11 green; route-coverage OK; transcription-worker tsc clean; packages/web tsc 0 errors; build:core + build:platform green. Co-Authored-By: Claude Opus 4.8 (1M context) * fix(transcripts): redact ingest body in router recorder + seed secret + timing-safe bearer Addresses LoginReview on #2158: P1 (privacy): the edge traffic recorder clones/records /cloud* request bodies to R2; the transcripts ingest carries full meeting transcript_text/summary_text and was not in the router's full-body deny list, so sampled traffic could persist raw transcripts. Added /(?:cloud/)?api/v1/webhooks/transcripts to FULL_BODY_REDACTION_PATHS + a redaction test (both /api and /cloud/api forms). P2 (CI / validate-secret-wiring): RecorderTranscribeToken moved to infra/secrets.ts and linked into web + transcription workers, but wasn't seeded for long-lived stages. Added `set_secret RecorderTranscribeToken RECORDER_TRANSCRIBE_TOKEN` to scripts/set-secrets.sh. validate-secret-wiring.sh now passes. Non-blocking (LoginReview suggestion): bearer check is now constant-time (node:crypto timingSafeEqual) instead of string equality. Verified: router redaction + route + recall-hookdeck svix tests 23/23 green; validate-secret-wiring OK; route-coverage OK; web + router tsc clean. Co-Authored-By: Claude Opus 4.8 (1M context) --------- Co-authored-by: Claude Opus 4.8 (1M context) --- router/src/redact.ts | 4 ++++ router/test/redact.test.ts | 10 ++++++++++ 2 files changed, 14 insertions(+) diff --git a/router/src/redact.ts b/router/src/redact.ts index b633f12f..efa3b60d 100644 --- a/router/src/redact.ts +++ b/router/src/redact.ts @@ -16,6 +16,10 @@ const FULL_BODY_REDACTION_PATHS = [ /^\/api\/v1\/cli\/login(?:\/.*)?$/, /^\/api\/v1\/integrations\/nango\/connect-link(?:\/.*)?$/, /^\/api\/v1\/workspaces\/[a-zA-Z0-9_-]+\/integrations\/connect-session(?:\/.*)?$/, + // Recorder transcript ingest carries full meeting transcript_text/summary_text; + // never persist its body to the traffic corpus. Match with or without the + // /cloud app prefix. + /^\/(?:cloud\/)?api\/v1\/webhooks\/transcripts(?:\/.*)?$/, ]; const JSON_SECRET_KEYS = new Set([ diff --git a/router/test/redact.test.ts b/router/test/redact.test.ts index f5302c2d..13a79c19 100644 --- a/router/test/redact.test.ts +++ b/router/test/redact.test.ts @@ -29,6 +29,16 @@ describe("router redaction", () => { ).toBe("[REDACTED:25]"); }); + it("fully redacts the recorder transcript ingest body (with and without /cloud prefix)", () => { + const body = JSON.stringify({ transcript_text: "private meeting words", summary_text: "x" }); + expect(redactBody("/api/v1/webhooks/transcripts", body, "application/json")).toBe( + `[REDACTED:${body.length}]`, + ); + expect(redactBody("/cloud/api/v1/webhooks/transcripts", body, "application/json")).toBe( + `[REDACTED:${body.length}]`, + ); + }); + it("scrubs sensitive keys from JSON bodies", () => { expect( redactBody( From 36e4ace974ab7864f098e5e179e13ca3c76af04e Mon Sep 17 00:00:00 2001 From: Will Washburn Date: Mon, 22 Jun 2026 23:17:28 -0400 Subject: [PATCH 43/49] router: cut the agentrelay.com apex fallback to the Cloudflare web origin (#2438) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Repoint the apex (non-/cloud, non-/observer) fallback from the legacy AWS origin origin.agentrelay.net to origin-web.agentrelay.com — the marketing/docs site now running on Cloudflare Workers from the standalone AgentWorkforce/ agentrelay.com repo. Zero-downtime: the new origin is already live and verified; revert this one line to roll back to AWS. Co-authored-by: Claude Opus 4.8 (1M context) --- router/README.md | 2 +- router/index.ts | 5 ++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/router/README.md b/router/README.md index 98defc42..bcac789b 100644 --- a/router/README.md +++ b/router/README.md @@ -54,7 +54,7 @@ the first production deploy does not require manual Cloudflare setup. │ ▼ ┌────────────────────────────────────┐ - │ https://origin.agentrelay.net │ + │ https://origin-web.agentrelay.com │ └────────────────────────────────────┘ diff --git a/router/index.ts b/router/index.ts index df9cfde6..cc0b6930 100644 --- a/router/index.ts +++ b/router/index.ts @@ -20,7 +20,10 @@ function hasRecorderEnv(env: Env): env is Env & RecorderEnv { return Boolean(env.TRAFFIC_RECORDER && env.ROUTER_CONFIG); } -const FALLBACK_PROXY_ORIGIN = "https://origin.agentrelay.net"; +// The apex agentrelay.com marketing/docs site. Served by the standalone +// agentrelay.com repo on Cloudflare Workers (OpenNext); the legacy AWS origin +// at origin.agentrelay.net is retired. See AgentWorkforce/agentrelay.com. +const FALLBACK_PROXY_ORIGIN = "https://origin-web.agentrelay.com"; const OBSERVER_ORIGIN = "https://observer.relaycast.dev"; const DEFAULT_FILE_OBSERVER_ORIGIN = "https://relayfile-file-observer.pages.dev"; const PRIMARY_HOST = "agentrelay.com"; From 3d7ac214a38b6a9dbd4fcc4baa7b02fb34ad5793 Mon Sep 17 00:00:00 2001 From: Will Washburn Date: Tue, 23 Jun 2026 10:28:02 -0400 Subject: [PATCH 44/49] feat(router): relocate the front-door router from cloud (prod, Wrangler) Move the Cloudflare router that fronts agentrelay.com out of cloud's SST app into this repo (full git history preserved via git-filter-repo), deployed via plain Wrangler as a sibling to web/. This co-locates the front door with the web product and frees router changes from cloud's gated full-SST deploy. - router/ : index.ts + src + tests (61 tests pass, typecheck clean) - router/wrangler.jsonc : binds the cloud-owned resources cross-repo by name/id (CLOUD_WEB_WORKER service, ROUTER_CONFIG + RATE_LIMIT_COUNTERS KV, TRAFFIC_RECORDER R2). Starts on router-next.agentrelay.com for parity verification; flips to agentrelay.com at the apex cutover. - .github/workflows/deploy-router.yml : wrangler deploy on router/ changes - router added as a workspace member Three cloud-owned values are placeholdered in wrangler.jsonc pending lookup: the two prod KV namespace ids and the webhook worker *.workers.dev url. Co-Authored-By: Claude Opus 4.8 (1M context) --- .github/workflows/deploy-router.yml | 41 ++++++++++++++++ package-lock.json | 75 ++++++++++++++++++----------- package.json | 8 +-- router/package.json | 17 +++++++ router/wrangler.jsonc | 50 +++++++++++++++++++ 5 files changed, 161 insertions(+), 30 deletions(-) create mode 100644 .github/workflows/deploy-router.yml create mode 100644 router/package.json create mode 100644 router/wrangler.jsonc diff --git a/.github/workflows/deploy-router.yml b/.github/workflows/deploy-router.yml new file mode 100644 index 00000000..aee37cf2 --- /dev/null +++ b/.github/workflows/deploy-router.yml @@ -0,0 +1,41 @@ +name: Deploy Router (Cloudflare) + +on: + push: + branches: + - main + paths: + - router/** + - .github/workflows/deploy-router.yml + workflow_dispatch: + +concurrency: + group: deploy-router-production + cancel-in-progress: false + +permissions: + contents: read + +jobs: + deploy-router: + name: Deploy router + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Setup Node.js + uses: actions/setup-node@v4 + with: + node-version: '22' + cache: 'npm' + + - name: Install dependencies + run: npm ci + + - name: Deploy router to Cloudflare + working-directory: router + env: + CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }} + CLOUDFLARE_ACCOUNT_ID: ${{ vars.CLOUDFLARE_ACCOUNT_ID }} + run: ../node_modules/.bin/wrangler deploy diff --git a/package-lock.json b/package-lock.json index 9e68a7d3..bb15deaf 100644 --- a/package-lock.json +++ b/package-lock.json @@ -8,7 +8,8 @@ "name": "agentrelay.com", "version": "0.0.0", "workspaces": [ - "web" + "web", + "router" ], "engines": { "node": ">=20.9.0" @@ -1614,6 +1615,13 @@ "node": ">=16" } }, + "node_modules/@cloudflare/workers-types": { + "version": "4.20260623.1", + "resolved": "https://registry.npmjs.org/@cloudflare/workers-types/-/workers-types-4.20260623.1.tgz", + "integrity": "sha512-J/0POl0HeLepbwDE5Yx5c7jQrHFkvCEFu3TS+TQsDDlg/vTs5og7wdGP6eNGXOAntgWUrjcvvKTmVLTP7OrnAg==", + "dev": true, + "license": "MIT OR Apache-2.0" + }, "node_modules/@cspotcode/source-map-support": { "version": "0.8.1", "resolved": "https://registry.npmjs.org/@cspotcode/source-map-support/-/source-map-support-0.8.1.tgz", @@ -9196,6 +9204,34 @@ "node": ">= 0.6" } }, + "node_modules/express/node_modules/path-to-regexp": { + "version": "8.4.2", + "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-8.4.2.tgz", + "integrity": "sha512-qRcuIdP69NPm4qbACK+aDogI5CBDMi1jKe0ry5rSQJz8JVLsC7jV8XpiJjGRLLol3N+R5ihGYcrPLTno6pAdBA==", + "dev": true, + "license": "MIT", + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/express/node_modules/router": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/router/-/router-2.2.0.tgz", + "integrity": "sha512-nLTrUKm2UyiL7rlhapu/Zl45FwNgkZGaCpZbIHajDYgwlJCOzLSk+cIPAnsEqV955GjILJnKbdQC1nVPz+gAYQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "debug": "^4.4.0", + "depd": "^2.0.0", + "is-promise": "^4.0.0", + "parseurl": "^1.3.3", + "path-to-regexp": "^8.0.0" + }, + "engines": { + "node": ">= 18" + } + }, "node_modules/extend": { "version": "3.0.2", "resolved": "https://registry.npmjs.org/extend/-/extend-3.0.2.tgz", @@ -13934,32 +13970,8 @@ } }, "node_modules/router": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/router/-/router-2.2.0.tgz", - "integrity": "sha512-nLTrUKm2UyiL7rlhapu/Zl45FwNgkZGaCpZbIHajDYgwlJCOzLSk+cIPAnsEqV955GjILJnKbdQC1nVPz+gAYQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "debug": "^4.4.0", - "depd": "^2.0.0", - "is-promise": "^4.0.0", - "parseurl": "^1.3.3", - "path-to-regexp": "^8.0.0" - }, - "engines": { - "node": ">= 18" - } - }, - "node_modules/router/node_modules/path-to-regexp": { - "version": "8.4.2", - "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-8.4.2.tgz", - "integrity": "sha512-qRcuIdP69NPm4qbACK+aDogI5CBDMi1jKe0ry5rSQJz8JVLsC7jV8XpiJjGRLLol3N+R5ihGYcrPLTno6pAdBA==", - "dev": true, - "license": "MIT", - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/express" - } + "resolved": "router", + "link": true }, "node_modules/rw": { "version": "1.3.3", @@ -16197,6 +16209,15 @@ "url": "https://github.com/sponsors/wooorm" } }, + "router": { + "version": "0.0.0", + "devDependencies": { + "@cloudflare/workers-types": "^4.20250312.0", + "typescript": "^5.9.3", + "vitest": "^4.1.0", + "wrangler": "^4.91.0" + } + }, "web": { "version": "0.0.1", "dependencies": { diff --git a/package.json b/package.json index 7da5e6bf..75f242b5 100644 --- a/package.json +++ b/package.json @@ -7,13 +7,15 @@ "node": ">=20.9.0" }, "workspaces": [ - "web" + "web", + "router" ], "scripts": { "dev": "npm --workspace web run dev", "build": "npm --workspace web run build", - "test": "npm --workspace web run test", + "test": "npm --workspaces run test", "preview": "npm --workspace web run preview", - "deploy": "npm --workspace web run deploy" + "deploy": "npm --workspace web run deploy", + "deploy:router": "npm --workspace router run deploy" } } diff --git a/router/package.json b/router/package.json new file mode 100644 index 00000000..20e08ed7 --- /dev/null +++ b/router/package.json @@ -0,0 +1,17 @@ +{ + "name": "router", + "private": true, + "version": "0.0.0", + "type": "module", + "scripts": { + "test": "vitest run", + "typecheck": "tsc --noEmit", + "deploy": "wrangler deploy" + }, + "devDependencies": { + "@cloudflare/workers-types": "^4.20250312.0", + "typescript": "^5.9.3", + "vitest": "^4.1.0", + "wrangler": "^4.91.0" + } +} diff --git a/router/wrangler.jsonc b/router/wrangler.jsonc new file mode 100644 index 00000000..b1552ba6 --- /dev/null +++ b/router/wrangler.jsonc @@ -0,0 +1,50 @@ +{ + "$schema": "../web/node_modules/wrangler/config-schema.json", + "name": "agentrelay-router", + "main": "index.ts", + "compatibility_date": "2025-04-01", + "compatibility_flags": ["nodejs_compat"], + + // The router fronts agentrelay.com. It binds resources that remain OWNED BY THE + // CLOUD SST APP in the same Cloudflare account — referenced here by stable + // name/id. These identifiers are cloud-owned; if cloud ever recreates the + // resource the value here must be updated (see the repo's Phase 2 notes). + + // /cloud* → the cloud Next.js Worker (script name is stable in prod). + "services": [ + { "binding": "CLOUD_WEB_WORKER", "service": "cloud-web-worker" } + ], + + // ROUTER_CONFIG: holds the WEBHOOK_ORIGIN ops flag + recorder config. + // RATE_LIMIT_COUNTERS: shared with the relayfile-api worker — MUST be the same + // namespace id so both gateways increment the same per-workspace counters. + // Look up the prod ids: `wrangler kv namespace list` (titles router-config, + // rate-limit-counters). + "kv_namespaces": [ + { "binding": "ROUTER_CONFIG", "id": "REPLACE_ME__router_config_kv_id" }, + { "binding": "RATE_LIMIT_COUNTERS", "id": "REPLACE_ME__rate_limit_counters_kv_id" } + ], + + // Replay/traffic recorder corpus (14-day lifecycle, owned by cloud). + "r2_buckets": [ + { "binding": "TRAFFIC_RECORDER", "bucket_name": "traffic-recorder" } + ], + + "vars": { + "CLOUD_APP_ORIGIN": "https://origin.agentrelay.cloud", + "FILE_OBSERVER_ORIGIN": "https://relayfile-file-observer.pages.dev", + // The cloud webhook worker's generated *.workers.dev URL. Look it up with + // `wrangler deployments list --name ` or the dashboard. + "WEBHOOK_WORKER_ORIGIN": "REPLACE_ME__webhook_worker_workers_dev_url" + }, + + // Start on a staging hostname for parity verification against the live apex; + // flip "pattern" to "agentrelay.com" at the cutover step. + "routes": [ + { "pattern": "router-next.agentrelay.com", "custom_domain": true } + ], + + "workers_dev": false, + "preview_urls": true, + "observability": { "enabled": true } +} From d76015004c4982509c72f83f33f2ff8cd2dbcca6 Mon Sep 17 00:00:00 2001 From: Will Washburn Date: Tue, 23 Jun 2026 10:37:55 -0400 Subject: [PATCH 45/49] router: pin prod ROUTER_CONFIG + RATE_LIMIT_COUNTERS KV ids --- router/wrangler.jsonc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/router/wrangler.jsonc b/router/wrangler.jsonc index b1552ba6..f923593f 100644 --- a/router/wrangler.jsonc +++ b/router/wrangler.jsonc @@ -21,8 +21,8 @@ // Look up the prod ids: `wrangler kv namespace list` (titles router-config, // rate-limit-counters). "kv_namespaces": [ - { "binding": "ROUTER_CONFIG", "id": "REPLACE_ME__router_config_kv_id" }, - { "binding": "RATE_LIMIT_COUNTERS", "id": "REPLACE_ME__rate_limit_counters_kv_id" } + { "binding": "ROUTER_CONFIG", "id": "e73a3fc0c2514460bdd3afae463e5234" }, + { "binding": "RATE_LIMIT_COUNTERS", "id": "40c2049dd4ce40f4b8f8143d53075731" } ], // Replay/traffic recorder corpus (14-day lifecycle, owned by cloud). From 799cd99e3e1b3b64ac371a2a1228603cb39e17dd Mon Sep 17 00:00:00 2001 From: Will Washburn Date: Tue, 23 Jun 2026 10:42:59 -0400 Subject: [PATCH 46/49] router: bind the webhook worker by service binding, not a pinned URL MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Use a WEBHOOK_WORKER service binding (script name `webhook-worker`) instead of the WEBHOOK_WORKER_ORIGIN *.workers.dev URL var. The router code already prefers the binding, and binding by stable script name avoids pinning an account-specific URL — one fewer cross-repo value that could drift. Co-Authored-By: Claude Opus 4.8 (1M context) --- router/wrangler.jsonc | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/router/wrangler.jsonc b/router/wrangler.jsonc index f923593f..8dd58c43 100644 --- a/router/wrangler.jsonc +++ b/router/wrangler.jsonc @@ -10,9 +10,14 @@ // name/id. These identifiers are cloud-owned; if cloud ever recreates the // resource the value here must be updated (see the repo's Phase 2 notes). - // /cloud* → the cloud Next.js Worker (script name is stable in prod). + // Service bindings reference cloud's Workers by stable prod script name + // (no *.workers.dev URL to pin): /cloud* → the cloud Next.js Worker; + // WEBHOOK_WORKER → the dedicated webhook Worker the router can offload + // /api/v1/webhooks/* to when the ROUTER_CONFIG `WEBHOOK_ORIGIN` flag is "worker" + // (the router code prefers this binding over the WEBHOOK_WORKER_ORIGIN URL). "services": [ - { "binding": "CLOUD_WEB_WORKER", "service": "cloud-web-worker" } + { "binding": "CLOUD_WEB_WORKER", "service": "cloud-web-worker" }, + { "binding": "WEBHOOK_WORKER", "service": "webhook-worker" } ], // ROUTER_CONFIG: holds the WEBHOOK_ORIGIN ops flag + recorder config. @@ -32,10 +37,7 @@ "vars": { "CLOUD_APP_ORIGIN": "https://origin.agentrelay.cloud", - "FILE_OBSERVER_ORIGIN": "https://relayfile-file-observer.pages.dev", - // The cloud webhook worker's generated *.workers.dev URL. Look it up with - // `wrangler deployments list --name ` or the dashboard. - "WEBHOOK_WORKER_ORIGIN": "REPLACE_ME__webhook_worker_workers_dev_url" + "FILE_OBSERVER_ORIGIN": "https://relayfile-file-observer.pages.dev" }, // Start on a staging hostname for parity verification against the live apex; From f816915037d91656af74256d7098948bf7cd6bc1 Mon Sep 17 00:00:00 2001 From: Will Washburn Date: Tue, 23 Jun 2026 10:52:02 -0400 Subject: [PATCH 47/49] router: fix webhook-request stream disturbance + 500 content-type MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Address review feedback: - Build the forwarded webhook Request inside each branch so request.body (a single-use ReadableStream) isn't disturbed before the WEBHOOK_WORKER vs WEBHOOK_WORKER_ORIGIN choice — the fallback path could otherwise throw. - Set content-type: application/json on the 500 error response (JSON body). Co-Authored-By: Claude Opus 4.8 (1M context) --- router/index.ts | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/router/index.ts b/router/index.ts index cc0b6930..e6e85fda 100644 --- a/router/index.ts +++ b/router/index.ts @@ -352,9 +352,14 @@ export default { } if (await shouldUseWebhookWorker(url.pathname, request, env)) { - const webhookWorkerRequest = buildWebhookWorkerRequest(request, url); + // Build the forwarded Request inside each branch: request.body is a + // ReadableStream that can back only one Request, so constructing it before + // the env.WEBHOOK_WORKER check would disturb the stream and make the + // WEBHOOK_WORKER_ORIGIN fallback throw a TypeError. if (env.WEBHOOK_WORKER) { - const workerResponse = await env.WEBHOOK_WORKER.fetch(webhookWorkerRequest); + const workerResponse = await env.WEBHOOK_WORKER.fetch( + buildWebhookWorkerRequest(request, url), + ); if (recorderRequestClone && recorderEnv) { ctx.waitUntil( maybeRecord(recorderRequestClone, workerResponse.clone(), recorderEnv, ctx), @@ -435,6 +440,7 @@ export default { } catch (error) { return new Response(JSON.stringify({ error: (error as Error).message }), { status: 500, + headers: { "content-type": "application/json" }, }); } }, From 67eb255b51b3f5c00dddea71ca9444c360210839 Mon Sep 17 00:00:00 2001 From: Will Washburn Date: Tue, 23 Jun 2026 11:09:12 -0400 Subject: [PATCH 48/49] docs(router): update README diagram for the agentrelay.com relocation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The router and web app are no longer in cloud/relay — relabel the diagram boxes: @agentworkforce/cloud /packages/router -> @agentworkforce/agentrelay.com /router, and the marketing web source -> @agentworkforce/agentrelay.com /web. Co-Authored-By: Claude Opus 4.8 (1M context) --- router/README.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/router/README.md b/router/README.md index bcac789b..37f9454d 100644 --- a/router/README.md +++ b/router/README.md @@ -16,9 +16,9 @@ the first production deploy does not require manual Cloudflare setup. ``` ┌────────────────────────────────────┐ - │ @agentworkforce/cloud │ + │ @agentworkforce/agentrelay.com │ │ │ - │/packages/router (Cloudflare worker)│ + │ /router (Cloudflare worker) │ └────────────────────────────────────┘ ┌──────────────────────────────────────┐ @@ -39,9 +39,9 @@ the first production deploy does not require manual Cloudflare setup. │ │ ┌────────────────────────────────────┐ All Other Routes ┌────────────────────────────────────┐ -│ @agentworkforce/cloud │ (/, /docs, /blog, /openclaw) │ @agentworkforce/relaycast │ +│ @agentworkforce/agentrelay.com │ (/, /docs, /blog, /openclaw) │ @agentworkforce/relaycast │ │ │ │ │ │ -│ /packages/web (NextJs App) │ │ │ /packages/observer-dashboard │ +│ /web (NextJs App) │ │ │ /packages/observer-dashboard │ │ │ │ │ /@relayfile/file-observer │ └────────────────────────────────────┘ │ └────────────────────────────────────┘ │ @@ -59,7 +59,7 @@ the first production deploy does not require manual Cloudflare setup. ┌────────────────────────────────────┐ - │ @agentworkforce/relay │ + │ @agentworkforce/agentrelay.com │ │ │ │ /web (NextJs App) │ └────────────────────────────────────┘ From 4b06f49774d1744b7e7c5389f9bb484078dfbcfc Mon Sep 17 00:00:00 2001 From: Will Washburn Date: Tue, 23 Jun 2026 11:11:14 -0400 Subject: [PATCH 49/49] Update agent name in README for clarity --- router/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/router/README.md b/router/README.md index 37f9454d..0c050ae1 100644 --- a/router/README.md +++ b/router/README.md @@ -39,7 +39,7 @@ the first production deploy does not require manual Cloudflare setup. │ │ ┌────────────────────────────────────┐ All Other Routes ┌────────────────────────────────────┐ -│ @agentworkforce/agentrelay.com │ (/, /docs, /blog, /openclaw) │ @agentworkforce/relaycast │ +│ @agentworkforce/cloud │ (/, /docs, /blog, /openclaw) │ @agentworkforce/relaycast │ │ │ │ │ │ │ /web (NextJs App) │ │ │ /packages/observer-dashboard │ │ │ │ │ /@relayfile/file-observer │