From 4fee3879d7533cefbf3890145f844a6171b36bfb Mon Sep 17 00:00:00 2001 From: Yingdi Shan <5491399+yingdi-shan@users.noreply.github.com> Date: Sat, 15 Aug 2026 09:09:55 +0000 Subject: [PATCH 1/7] feat: add single-tenant API key authentication --- deploy/docker-compose.yml | 8 +- deploy/k8s/base/agentenv-daemonset.yaml | 7 +- deploy/k8s/base/gateway-deployment.yaml | 5 + deploy/k8s/base/kustomization.yaml | 5 + deploy/k8s/run.sh | 41 +++ docs/src/SUMMARY.md | 1 + docs/src/concepts/sandboxes.md | 2 +- docs/src/configuration/authentication.md | 121 +++++++ docs/src/configuration/env-vars.md | 13 +- docs/src/deployment/docker-compose.md | 38 ++ docs/src/deployment/docker.md | 12 + docs/src/deployment/kubernetes.md | 13 + docs/src/deployment/manual-compile.md | 13 +- docs/src/deployment/pvm.md | 5 +- docs/src/deployment/static-multi-node.md | 39 +- docs/src/getting-started/aenv-cli.md | 2 +- docs/src/getting-started/quickstart.md | 17 +- docs/src/integration/e2b.md | 10 +- docs/src/internals/architecture.md | 3 +- docs/src/internals/services.md | 4 +- docs/src/security/secure-sandboxes.md | 14 +- scripts/install.sh | 4 +- scripts/tests/e2e/lib/helpers.sh | 11 +- scripts/tests/e2e/lib/runtime.sh | 12 +- scripts/tests/e2e/lib/server.sh | 3 +- scripts/tests/e2e/suites/08_auth.sh | 19 + scripts/tests/e2e/suites/09_e2b_compat.sh | 3 +- .../tests/e2e/suites/14_code_interpreter.sh | 2 +- services/README.md | 9 +- services/gateway/cmd/main.go | 48 +++ services/gateway/cmd/main_test.go | 77 ++++ services/gateway/internal/server.go | 105 +++++- services/gateway/internal/server_test.go | 178 ++++++++- src/api/impls/auth.rs | 179 +++++++-- src/api/impls/mod.rs | 5 +- src/api/impls/sandbox.rs | 3 + src/api/proxy.rs | 340 ++++++++++++++++-- src/api/server.rs | 8 +- src/api_key.rs | 195 ++++++++++ src/bin/server.rs | 3 + src/lib.rs | 1 + src/sandbox/backend.rs | 4 + src/sandbox/firecracker/sandbox.rs | 4 + 43 files changed, 1458 insertions(+), 128 deletions(-) create mode 100644 docs/src/configuration/authentication.md create mode 100644 services/gateway/cmd/main_test.go create mode 100644 src/api_key.rs diff --git a/deploy/docker-compose.yml b/deploy/docker-compose.yml index 52130b7c8..4a6bfb2da 100644 --- a/deploy/docker-compose.yml +++ b/deploy/docker-compose.yml @@ -16,8 +16,9 @@ x-agentenv-base: &agentenv-base - /dev:/dev - ${CONFIG_PATH:-../config/default.toml}:/workspace/config/default.toml:ro # Runtime assets are baked into the image by `server --setup-only`; compose - # persists only committed snapshots across container restarts. + # persists committed snapshots and the deployment API key across restarts. - agentenv-snapshot-store:/workspace/env/snapshot-store + - agentenv-auth:/workspace/env/secrets devices: - /dev/kvm:/dev/kvm privileged: true @@ -67,7 +68,9 @@ services: depends_on: scheduler: condition: service_healthy - volumes: *control-plane-config-volume + volumes: + - ./docker/config/default.json:/config/default.json:ro + - agentenv-auth:/run/secrets:ro environment: GATEWAY_HTTP_LISTEN_ADDR: :8080 GATEWAY_SCHEDULER_ADDR: scheduler:9090 @@ -96,4 +99,5 @@ services: AENV_NODE_ID: node-b volumes: + agentenv-auth: agentenv-snapshot-store: diff --git a/deploy/k8s/base/agentenv-daemonset.yaml b/deploy/k8s/base/agentenv-daemonset.yaml index 2f9c43f4a..470b2a9cb 100644 --- a/deploy/k8s/base/agentenv-daemonset.yaml +++ b/deploy/k8s/base/agentenv-daemonset.yaml @@ -21,6 +21,11 @@ spec: image: agentenv-runtime:latest imagePullPolicy: IfNotPresent env: + - name: AENV_API_KEY + valueFrom: + secretKeyRef: + name: agentenv-auth + key: AENV_API_KEY - name: AENV_CONFIG_PATH value: /workspace/config/agentenv.toml - name: AENV_SANDBOX_ACCESS_TOKEN_HASH_SEED @@ -76,7 +81,7 @@ spec: - | echo "preStop: waiting for sandboxes to drain..." while true; do - count=$(curl -sf -H 'X-API-Key: preStop' http://localhost:8000/sandboxes | jq 'length') || count="" + count=$(curl -sf -H "X-API-Key: ${AENV_API_KEY}" http://localhost:8000/sandboxes | jq 'length') || count="" if [ -z "$count" ]; then echo "preStop: failed to query sandbox count, retrying..." sleep 3 diff --git a/deploy/k8s/base/gateway-deployment.yaml b/deploy/k8s/base/gateway-deployment.yaml index 81f222c86..08536fc0e 100644 --- a/deploy/k8s/base/gateway-deployment.yaml +++ b/deploy/k8s/base/gateway-deployment.yaml @@ -27,6 +27,11 @@ spec: - name: http containerPort: 8080 env: + - name: AENV_API_KEY + valueFrom: + secretKeyRef: + name: agentenv-auth + key: AENV_API_KEY - name: GATEWAY_SANDBOX_PROXY_DOMAINS valueFrom: configMapKeyRef: diff --git a/deploy/k8s/base/kustomization.yaml b/deploy/k8s/base/kustomization.yaml index 797a3bb74..556f0d700 100644 --- a/deploy/k8s/base/kustomization.yaml +++ b/deploy/k8s/base/kustomization.yaml @@ -30,6 +30,11 @@ configMapGenerator: literals: - SANDBOX_PROXY_DOMAINS= +secretGenerator: + - name: agentenv-auth + literals: + - AENV_API_KEY= + images: - name: agentenv-gateway newName: agentenv-gateway diff --git a/deploy/k8s/run.sh b/deploy/k8s/run.sh index d0322c011..4d39680ba 100644 --- a/deploy/k8s/run.sh +++ b/deploy/k8s/run.sh @@ -10,6 +10,7 @@ MODE="$1" shift KUBECTL_BIN="${KUBECTL:-kubectl}" OVERLAY_NAME="${K8S_OVERLAY:-default}" +NAMESPACE="${K8S_NAMESPACE:-agentenv-system}" SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" REPO_ROOT="$(cd "${SCRIPT_DIR}/../.." && pwd)" @@ -28,6 +29,43 @@ sed_in_place() { cp -R "${SCRIPT_DIR}" "${TEMP_DIR}/k8s" cp "${REPO_ROOT}/config/default.toml" "${TEMP_DIR}/k8s/base/config/agentenv.toml" +if [[ "${MODE}" != "delete" ]]; then + API_KEY_VALUE="" + if [[ "${AENV_API_KEY+x}" == "x" ]]; then + API_KEY_VALUE="${AENV_API_KEY}" + elif [[ "${MODE}" == "apply" ]]; then + encoded_key="" + if ! namespace_name="$("${KUBECTL_BIN}" get namespace "${NAMESPACE}" --ignore-not-found -o name)"; then + echo "failed to check namespace ${NAMESPACE}" >&2 + exit 1 + fi + if [[ -n "${namespace_name}" ]]; then + if ! encoded_key="$("${KUBECTL_BIN}" -n "${NAMESPACE}" get secret agentenv-auth \ + --ignore-not-found -o jsonpath='{.data.AENV_API_KEY}')"; then + echo "failed to read existing Secret ${NAMESPACE}/agentenv-auth" >&2 + exit 1 + fi + fi + if [[ -n "${encoded_key}" ]]; then + if ! API_KEY_VALUE="$(printf '%s' "${encoded_key}" | base64 -d)"; then + echo "failed to decode the existing agentenv-auth Secret" >&2 + exit 1 + fi + fi + fi + + if [[ -z "${API_KEY_VALUE}" ]]; then + API_KEY_VALUE="e2b_$(od -An -N32 -tx1 /dev/urandom | tr -d '[:space:]')" + fi + if [[ ! "${API_KEY_VALUE}" =~ ^[A-Za-z0-9._~-]{32,}$ ]]; then + echo "AENV_API_KEY must contain at least 32 URL-safe characters" >&2 + exit 1 + fi + + sed_in_place \ + "s#- AENV_API_KEY=.*#- AENV_API_KEY=${API_KEY_VALUE}#" \ + "${TEMP_DIR}/k8s/base/kustomization.yaml" +fi if [[ "${SANDBOX_PROXY_DOMAINS+x}" == "x" ]]; then ESCAPED_SANDBOX_PROXY_DOMAINS="${SANDBOX_PROXY_DOMAINS//\\/\\\\}" @@ -65,6 +103,9 @@ case "${MODE}" in ;; apply) "${KUBECTL_BIN}" apply -k "${OVERLAY_PATH}" "$@" + echo "AgentENV API key stored in Secret ${NAMESPACE}/agentenv-auth." >&2 + echo "Read it with:" >&2 + echo " ${KUBECTL_BIN} -n ${NAMESPACE} get secret agentenv-auth -o go-template='{{index .data \"AENV_API_KEY\" | base64decode}}{{\"\\n\"}}'" >&2 ;; delete) "${KUBECTL_BIN}" delete --ignore-not-found -k "${OVERLAY_PATH}" "$@" diff --git a/docs/src/SUMMARY.md b/docs/src/SUMMARY.md index f1d8d120a..74795cb26 100644 --- a/docs/src/SUMMARY.md +++ b/docs/src/SUMMARY.md @@ -18,6 +18,7 @@ # Configuration +- [Authentication](./configuration/authentication.md) - [Configuration Reference](./configuration/reference.md) - [Environment Variables](./configuration/env-vars.md) diff --git a/docs/src/concepts/sandboxes.md b/docs/src/concepts/sandboxes.md index ab6be6ea1..31f91e5b6 100644 --- a/docs/src/concepts/sandboxes.md +++ b/docs/src/concepts/sandboxes.md @@ -59,7 +59,7 @@ aenv start --cold ubuntu:24.04 The cold-start API accepts an optional `diskSizeMB` field to set the root filesystem's virtual size in MiB. Explicit values must be at least 1024 MiB and divisible by 1024 because the current resize tool operates at 1 GiB granularity. Growth is allowed by default; shrinking below the source image size requires `ublk.overlaybd.allow_shrink = true`. If omitted, the image's built-in virtual size is used. Resizing applies only when creating a fresh writable root filesystem, not to read-only images, images with an existing upper, or snapshot resume. Sandbox responses also report disk size as `diskSizeMB`. -Use `aenv start --secure` with either warm or cold starts to require an envd access token for command and file operations. The CLI obtains and sends the token automatically. Secure mode protects the envd control port only; it does not add authentication to application ports. Each fork derives a distinct envd token from the child sandbox ID. +Use `aenv start --secure` with either warm or cold starts to require an envd access token for command and file operations. The CLI obtains and sends the token automatically. Secure mode protects the envd control port, while application proxy requests use the independent sandbox-scoped traffic token. Each fork derives distinct envd and traffic credentials from the child sandbox ID. --- diff --git a/docs/src/configuration/authentication.md b/docs/src/configuration/authentication.md new file mode 100644 index 000000000..b57dac1e9 --- /dev/null +++ b/docs/src/configuration/authentication.md @@ -0,0 +1,121 @@ +# Authentication + +AgentENV uses one shared API key for a single-tenant deployment. The gateway +and every runtime node in a cluster must resolve the same key. + +Clients authenticate API requests with: + +```text +X-API-Key: +``` + +`Authorization`, `X-Admin-Token`, and `X-Team-ID` do not authenticate +AgentENV. The `Authorization` header is left unchanged when a request is +proxied into a sandbox, so applications inside a sandbox can use it normally. +`GET /health` is public for load balancer and container health checks. + +E2B SDK users set `E2B_API_KEY` to the same value. Sandbox create responses +include an independent `trafficAccessToken`; send it as +`e2b-traffic-access-token` on application proxy requests. The token is scoped to +the sandbox and is not accepted for control-plane API calls. + +For secure sandboxes, `envdAccessToken` is a separate credential for envd +control traffic and must be sent as `X-Access-Token` only when targeting the +envd control-plane port. It is absent for insecure sandboxes. + +## Key Resolution + +On normal startup, a runtime node uses the first available source: + +1. `AENV_API_KEY` +2. `/run/secrets/api-key` +3. `$AENV_HOME/secrets/api-key` + +If neither an environment value nor an external secret exists, the server +generates a 256-bit key and atomically stores it in the managed path with +`0600` permissions. It reuses that key on later starts. Dependency and host +setup modes do not create a key. + +The gateway uses `AENV_API_KEY` or `/run/secrets/api-key`; it never generates a +key because every gateway and runtime node in a cluster must share one. + +## Installation Methods + +For a native installation, start the service once and read the managed key: + +```bash +sudo cat /var/lib/aenv/secrets/api-key +``` + +When upgrading an installation that already has `AENV_API_KEY` in +`/etc/default/aenv`, the installer preserves that entry and the server keeps +using it. Fresh installations leave key creation to the server. + +For a single Docker container, no auth volume is required. The server creates +the key in its writable container layer: + +```bash +docker exec aenv-server cat /workspace/env/secrets/api-key +``` + +Removing the container removes this generated key. Supply an explicit key or +mount a secret at `/run/secrets/api-key` when it must remain stable across +container replacements. + +The checked-in Compose deployment mounts one named volume read-write on both +runtime nodes and read-only at `/run/secrets` on the gateway. Concurrent node +startup is safe: atomic creation makes both nodes converge on the same key. +Read it with: + +```bash +docker compose -f deploy/docker-compose.yml exec -T agentenv-a \ + cat /workspace/env/secrets/api-key +``` + +`docker compose down` preserves the key. `docker compose down -v` removes the +auth volume, so the next startup generates a new key. + +`make k8s-apply` creates `Secret/agentenv-auth` on the first apply and reuses +the existing key on later applies. Read it with: + +```bash +kubectl -n agentenv-system get secret agentenv-auth \ + -o go-template='{{index .data "AENV_API_KEY" | base64decode}}{{"\n"}}' +``` + +For a single-node manual build, start the server and read +`$AENV_HOME/secrets/api-key`. To provide your own key instead, export it before +startup: + +```bash +export AENV_API_KEY="e2b_$(openssl rand -hex 32)" +make start-server +``` + +Custom keys must contain at least 32 URL-safe characters. In a multi-node +deployment, use exactly the same value for the gateway and every runtime node. +The generated keys use `e2b_` followed by hexadecimal characters so they pass +the E2B SDK default API-key validation. Use that format for custom keys when +you need E2B SDK compatibility. + +Docker Compose secrets can supply a pre-existing key without another AgentENV +configuration variable. In an override file, define a file-backed secret and +mount it with `target: api-key` on the gateway and every runtime node. Compose +then exposes the standard `/run/secrets/api-key` path. Compose secret sources +must already exist, so the named-volume setup remains the zero-configuration +default that allows Rust to generate the key during startup. + +## Transport Security + +API key authentication does not encrypt HTTP traffic. Do not send the key over +an untrusted plaintext network. Keep AgentENV on loopback or a trusted private +network, use a VPN, or terminate HTTPS at a reverse proxy or load balancer. + +## Rotation + +Set a new `AENV_API_KEY` on the gateway and every runtime node, or replace the +shared secret file, then restart them. Existing clients must switch to the new +value. Previously issued +`trafficAccessToken` values stop working when the key changes. +`envdAccessToken` values are unaffected and rotate only when the optional envd +seed changes. diff --git a/docs/src/configuration/env-vars.md b/docs/src/configuration/env-vars.md index 8581458da..fc70ccaac 100644 --- a/docs/src/configuration/env-vars.md +++ b/docs/src/configuration/env-vars.md @@ -12,6 +12,7 @@ These variables are consumed by the repository's Docker Compose and Kubernetes h | Variable | Default | Description | |----------|---------|-------------| +| `AENV_API_KEY` | generated under `$AENV_HOME/secrets/api-key` | Optional API-key override. Runtime nodes also check `/run/secrets/api-key` before creating a managed key. Use one shared value or secret in multi-node deployments. | | `API_ADDR` | `0.0.0.0:8000` | Address and port the API server listens on | | `AENV_CONFIG_PATH` | `config/default.toml` | Path to the TOML configuration file | | `AENV_LOG_FORMAT` | `compact` | Server log output format: `compact`, `pretty`, or `json` | @@ -46,8 +47,7 @@ These variables configure the E2B SDK and CLI to point at an AgentENV server. Va |----------|-------------| | `E2B_API_URL` | AgentENV server API base URL | | `E2B_SANDBOX_URL` | Sandbox proxy URL (for WebSocket and process interaction) | -| `E2B_API_KEY` | API key for authentication | -| `E2B_ACCESS_TOKEN` | Access token (used by `e2b template` commands) | +| `E2B_API_KEY` | Set to the deployment's `AENV_API_KEY` | ### Values by Deployment Mode @@ -56,8 +56,7 @@ These variables configure the E2B SDK and CLI to point at an AgentENV server. Va ```bash export E2B_API_URL=http://127.0.0.1:8000 export E2B_SANDBOX_URL=${E2B_API_URL} -export E2B_API_KEY=e2b_000000 -export E2B_ACCESS_TOKEN=dummy +export E2B_API_KEY=${AENV_API_KEY} ``` **Docker Compose / Kubernetes (multi-node)**: @@ -65,15 +64,14 @@ export E2B_ACCESS_TOKEN=dummy ```bash export E2B_API_URL=http://127.0.0.1:8080 export E2B_SANDBOX_URL=${E2B_API_URL} -export E2B_API_KEY=e2b_000000 -export E2B_ACCESS_TOKEN=dummy +export E2B_API_KEY=${AENV_API_KEY} ``` > In both modes, sandbox data-plane requests can use routing headers with > `E2B_SANDBOX_URL=${E2B_API_URL}`. The explicit `/proxy` prefix > (`${E2B_API_URL}/proxy`) is still accepted for back-compat. -> For local development, any non-empty value works for `E2B_API_KEY` and `E2B_ACCESS_TOKEN` because the server only checks that the auth header is present. +See [Authentication](./authentication.md) for key generation and storage. ## Gateway and Scheduler @@ -88,6 +86,7 @@ These variables apply to both the gateway and scheduler processes. | Variable | Default | Description | |----------|---------|-------------| +| `AENV_API_KEY` | unset | Shared single-tenant API key. The gateway uses the environment value when set, otherwise it reads `/run/secrets/api-key`. | | `GATEWAY_HTTP_LISTEN_ADDR` | `:8080` | HTTP listen address | | `GATEWAY_METRICS_LISTEN_ADDR` | `:9102` | Prometheus metrics listen address | | `GATEWAY_SCHEDULER_ADDR` | `127.0.0.1:9090` | Scheduler gRPC address for routing and node lookup | diff --git a/docs/src/deployment/docker-compose.md b/docs/src/deployment/docker-compose.md index 5b0bf38b5..1417c4d91 100644 --- a/docs/src/deployment/docker-compose.md +++ b/docs/src/deployment/docker-compose.md @@ -38,6 +38,11 @@ make deploy-up The Gateway is available at `http://127.0.0.1:8000` and forwards requests to the backend nodes. +On first startup, the runtime nodes atomically generate one API key in the +shared `agentenv-auth` volume. The gateway mounts that volume read-only at +`/run/secrets`, so all three services use the same key. Normal +`make deploy-down` calls preserve the volume and key. + To enable host-based sandbox data-plane URLs, set the shared sandbox proxy domain variable when starting the stack: @@ -58,6 +63,14 @@ curl http://127.0.0.1:8000/health # Cluster node snapshots via gateway curl http://127.0.0.1:8000/nodes + +# Authenticated cluster node snapshots via gateway +export AENV_API_KEY="$(docker compose -f deploy/docker-compose.yml exec -T agentenv-a \ + cat /workspace/env/secrets/api-key)" +curl -H "X-API-Key: ${AENV_API_KEY}" http://127.0.0.1:8080/nodes + +# Direct health check on a backend node +curl http://127.0.0.1:8001/health ``` ## Management Commands @@ -68,6 +81,31 @@ make deploy-logs # Stream logs from all services make deploy-down # Tear down the cluster ``` +Removing Compose volumes with `docker compose down -v` also removes the API +key. The next startup generates a new key and existing clients must be updated. + +To provide an existing key through Docker Compose secrets, add a file-backed +secret in an override file and mount it with `target: api-key` on the gateway +and both runtime nodes. AgentENV automatically reads `/run/secrets/api-key`; +no file-path environment variable is needed. + +```yaml +services: + gateway: + secrets: [api-key] + agentenv-a: + secrets: [api-key] + agentenv-b: + secrets: [api-key] + +secrets: + api-key: + file: ./api-key +``` + +The secret name is also its default target filename, so this mounts the key at +`/run/secrets/api-key` in each service. + ## Configuration Container deployments use `deploy/docker/config/default.json`. Scheduler and backend node endpoints are configured for the Docker network. diff --git a/docs/src/deployment/docker.md b/docs/src/deployment/docker.md index 84d015781..b7c5ffbf0 100644 --- a/docs/src/deployment/docker.md +++ b/docs/src/deployment/docker.md @@ -44,6 +44,7 @@ docker build \ ```bash docker run --rm -it \ + --name aenv-server \ --device /dev/kvm --privileged -v /dev:/dev \ -p 8000:8000 \ ghcr.io/kvcache-ai/aenv-server:latest # or aenv:latest if built from source @@ -51,6 +52,17 @@ docker run --rm -it \ The `--privileged` flag is required for Firecracker's network namespace operations (veth pairs, iptables). The server auto-downloads runtime assets on first start and is accessible at `http://127.0.0.1:8000` once ready. +On normal startup, the server generates the API key inside the container at +`/workspace/env/secrets/api-key`. Read it while the container is running with: + +```bash +docker exec aenv-server cat /workspace/env/secrets/api-key +``` + +Removing the container also removes this generated key. Supply an explicit +`AENV_API_KEY` or a secret at `/run/secrets/api-key` when the key must remain +stable across container replacements. + ## Verify ```bash diff --git a/docs/src/deployment/kubernetes.md b/docs/src/deployment/kubernetes.md index e622b5325..412a63538 100644 --- a/docs/src/deployment/kubernetes.md +++ b/docs/src/deployment/kubernetes.md @@ -61,6 +61,19 @@ make k8s-render make k8s-apply ``` +`make k8s-apply` generates a 256-bit API key on the first deployment and +stores it in `Secret/agentenv-auth`. Later applies reuse that key. Read it +locally when configuring clients: + +```bash +kubectl -n agentenv-system get secret agentenv-auth \ + -o go-template='{{index .data "AENV_API_KEY" | base64decode}}{{"\n"}}' +``` + +Set `AENV_API_KEY` when applying to supply your own key instead. A standalone +`make k8s-render` uses a temporary generated value because it does not modify +or read cluster state. + To enable host-based sandbox data-plane URLs, set the shared sandbox proxy domain variable when rendering or applying manifests: diff --git a/docs/src/deployment/manual-compile.md b/docs/src/deployment/manual-compile.md index 6f1c447ea..9b287e0bd 100644 --- a/docs/src/deployment/manual-compile.md +++ b/docs/src/deployment/manual-compile.md @@ -33,6 +33,9 @@ make release ## Start the Server +Start the server. On first normal startup it generates an API key under +`$AENV_HOME/secrets/api-key` and reuses it on later starts: + ```bash # Debug build API_ADDR=0.0.0.0:8000 make start-server @@ -41,14 +44,22 @@ API_ADDR=0.0.0.0:8000 make start-server API_ADDR=0.0.0.0:8000 make start-server-release ``` -The server auto-downloads runtime assets (Firecracker binary, kernel, rootfs) on first start. Once ready, it listens at `http://127.0.0.1:8000`. +The server auto-downloads runtime assets (Firecracker binary, kernel, rootfs) on first start. Once ready, it listens at `http://127.0.0.1:8000`. Read the generated key before making authenticated requests: + +```bash +export AENV_API_KEY="$(cat "${AENV_HOME_PATH:-/var/lib/aenv}/secrets/api-key")" +``` ## Verify ```bash curl http://127.0.0.1:8000/health +curl -H "X-API-Key: ${AENV_API_KEY}" http://127.0.0.1:8000/sandboxes ``` +HTTP does not protect the key in transit. Use a trusted network, VPN, or +TLS-terminating reverse proxy for remote clients. + ## Configuration The server reads `config/default.toml` by default. Override with: diff --git a/docs/src/deployment/pvm.md b/docs/src/deployment/pvm.md index 47781d78c..1b896a655 100644 --- a/docs/src/deployment/pvm.md +++ b/docs/src/deployment/pvm.md @@ -197,7 +197,7 @@ Use the dedicated PVM image: ```bash docker pull ghcr.io/kvcache-ai/aenv-server:latest-pvm -docker run --rm -it \ +docker run --rm -it --name aenv-server \ --device /dev/kvm \ --privileged \ -v /dev:/dev \ @@ -218,6 +218,9 @@ cargo run --bin server -- --setup-only make start-server ``` +The server generates and persists the API key under +`$AENV_HOME/secrets/api-key` on its first normal startup. + You can also set the mode in the TOML configuration: ```toml diff --git a/docs/src/deployment/static-multi-node.md b/docs/src/deployment/static-multi-node.md index 5ecc6c3d9..a67abc97f 100644 --- a/docs/src/deployment/static-multi-node.md +++ b/docs/src/deployment/static-multi-node.md @@ -21,9 +21,9 @@ AgentENV runtime nodes: | Runtime node A | `10.0.0.21:8000` | Runs Firecracker sandboxes as `node-a` | | Runtime node B | `10.0.0.22:8000` | Runs Firecracker sandboxes as `node-b` | -Use private addresses or an otherwise trusted network. AgentENV does not -currently provide an authentication boundary suitable for exposing these -services directly to the public Internet. +AgentENV authenticates HTTP requests but does not encrypt them. Use private +addresses, a VPN, or TLS termination before traffic crosses an untrusted +network. ## Prerequisites @@ -55,14 +55,24 @@ an external metrics collector needs them. ## 1. Install the runtime nodes +Generate one API key, deliver it through your normal secret-management channel, +and use the same value on every runtime node and the Gateway: + +```bash +export AENV_API_KEY="e2b_$(openssl rand -hex 32)" +``` + Run the installation on each runtime node: ```bash -curl -fsSL https://raw.githubusercontent.com/kvcache-ai/AgentENV/main/scripts/install.sh \ - | sudo bash +curl -fsSL https://raw.githubusercontent.com/kvcache-ai/AgentENV/main/scripts/install.sh | sudo bash ``` -Edit `/etc/default/aenv` on each machine without removing the paths written by the installer. +Edit `/etc/default/aenv` on each machine without removing the paths written by +the installer, and add `AENV_API_KEY=` before starting the +services. A multi-node deployment must not let each node generate an +independent managed key. + See [Secure Sandboxes](../security/secure-sandboxes.md) if the deployment needs future cross-node sandbox recovery. Node A uses: @@ -107,6 +117,17 @@ sudo useradd --system --no-create-home --shell /usr/sbin/nologin agentenv-contro sudo install -d -o root -g agentenv-control -m 0750 /etc/agentenv ``` +Create `/etc/agentenv/auth.env` with the same key used on the runtime nodes: + +```bash +sudo install -o root -g agentenv-control -m 0640 /dev/null /etc/agentenv/auth.env +sudoedit /etc/agentenv/auth.env +``` + +```text +AENV_API_KEY= +``` + If the `agentenv-control` account already exists, the `useradd` command reports that fact and can be skipped. @@ -192,6 +213,7 @@ After=network-online.target agentenv-scheduler.service [Service] User=agentenv-control Group=agentenv-control +EnvironmentFile=/etc/agentenv/auth.env ExecStart=/usr/local/bin/agentenv-gateway -config /etc/agentenv/control-plane.json Restart=on-failure RestartSec=5 @@ -219,7 +241,8 @@ curl http://10.0.0.22:8000/health curl http://127.0.0.1:8080/health # Wait for node heartbeats, then inspect the cluster through the Gateway -curl http://127.0.0.1:8080/nodes +export AENV_API_KEY="$(sudo sed -n 's/^AENV_API_KEY=//p' /etc/agentenv/auth.env)" +curl -H "X-API-Key: ${AENV_API_KEY}" http://127.0.0.1:8080/nodes ``` The node list should contain `node-a` and `node-b`. Point clients at the @@ -228,7 +251,7 @@ Gateway, not directly at a runtime node: ```bash aenv auth # AENV server URL: http://10.0.0.10:8080 -# API key: dummy +# API key: ``` Sandbox create, list, lifecycle, and data-plane requests can then be routed diff --git a/docs/src/getting-started/aenv-cli.md b/docs/src/getting-started/aenv-cli.md index 7d1227ac3..79500c7e3 100644 --- a/docs/src/getting-started/aenv-cli.md +++ b/docs/src/getting-started/aenv-cli.md @@ -27,7 +27,7 @@ Save the server URL and API key. Credentials are stored at `~/.config/aenv/crede ```bash aenv auth # AENV server URL [http://localhost:8000]: The address of the AgentENV server -# API key: dummy (Any non-empty string works for local development.) +# API key: ``` --- diff --git a/docs/src/getting-started/quickstart.md b/docs/src/getting-started/quickstart.md index 0db8b6582..9ebd17573 100644 --- a/docs/src/getting-started/quickstart.md +++ b/docs/src/getting-started/quickstart.md @@ -50,6 +50,7 @@ default. Transient namespace and daemon-socket state lives under `/run/aenv`. curl -fsSL https://raw.githubusercontent.com/kvcache-ai/AgentENV/main/scripts/docker-setup.sh | sudo bash docker pull ghcr.io/kvcache-ai/aenv-server:latest docker run --rm -it \ + --name aenv-server \ --device /dev/kvm --privileged -v /dev:/dev \ -p 8000:8000 \ ghcr.io/kvcache-ai/aenv-server:latest @@ -61,6 +62,7 @@ To customize the server configuration, download and edit the configuration file, curl -fsSL https://raw.githubusercontent.com/kvcache-ai/AgentENV/main/config/default.toml -o config.toml vim config.toml docker run --rm -it \ + --name aenv-server \ --device /dev/kvm --privileged -v /dev:/dev \ -v "$PWD/config.toml:/workspace/config/default.toml:ro" \ -p 8000:8000 \ @@ -87,14 +89,23 @@ curl -fsSL https://raw.githubusercontent.com/kvcache-ai/AgentENV/main/scripts/in ### 3. Authenticate +The server generates the key on its first normal startup. Native installations +reuse the managed key; a normal Docker container keeps it in its writable +container layer: + +```bash +# Native +sudo cat /var/lib/aenv/secrets/api-key +# Docker +docker exec aenv-server cat /workspace/env/secrets/api-key +``` + ```bash aenv auth # AENV server URL [http://localhost:8000]: http://127.0.0.1:8000 -# API key: dummy +# API key: ``` -For local development, any non-empty string works as the API key. - ### 4. Pull a template and run a sandbox ```bash diff --git a/docs/src/integration/e2b.md b/docs/src/integration/e2b.md index 6a6db3258..653f30371 100644 --- a/docs/src/integration/e2b.md +++ b/docs/src/integration/e2b.md @@ -12,10 +12,16 @@ Set environment variables to point at your AgentENV server. See [Environment Var # Single-node example export E2B_API_URL=http://127.0.0.1:8000 export E2B_SANDBOX_URL=${E2B_API_URL} -export E2B_API_KEY=e2b_000000 -export E2B_ACCESS_TOKEN=dummy +export E2B_API_KEY=${AENV_API_KEY} ``` +No `E2B_ACCESS_TOKEN` is needed. AgentENV returns `trafficAccessToken` for +application proxy traffic and (for secure sandboxes) `envdAccessToken` for envd +control traffic. These credentials have different headers and trust boundaries: +use `e2b-traffic-access-token` for application routes and `X-Access-Token` only +for envd. This is transport data, not the deprecated user-supplied +`E2B_ACCESS_TOKEN`. + ### TypeScript SDK #### Setup diff --git a/docs/src/internals/architecture.md b/docs/src/internals/architecture.md index 1cf4fa214..3f4399f5f 100644 --- a/docs/src/internals/architecture.md +++ b/docs/src/internals/architecture.md @@ -220,10 +220,11 @@ Discovery modes: **Deployment**: ```bash +export AENV_API_KEY="e2b_$(openssl rand -hex 32)" # shared by local runtime and gateway processes # local dev (single node) make start-server && make -C services run-scheduler && make -C services run-gateway -# docker compose (multi-node) +# docker compose (multi-node; shared auth volume is provisioned automatically) make deploy-up # gateway + scheduler + 2 backend nodes make deploy-down # teardown diff --git a/docs/src/internals/services.md b/docs/src/internals/services.md index 98dffb025..5343cf08e 100644 --- a/docs/src/internals/services.md +++ b/docs/src/internals/services.md @@ -25,7 +25,8 @@ make proto # regenerate protobuf # Start scheduler (default: 127.0.0.1:9090) make -C services run-scheduler -# Start gateway +# Start gateway (use the same key on runtime nodes) +export AENV_API_KEY="e2b_$(openssl rand -hex 32)" make -C services run-gateway ``` @@ -41,6 +42,7 @@ The scheduler supports two node discovery modes: ### Docker Compose ```bash +# Run scripts/docker-setup.sh first for host prerequisites. make deploy-up # gateway + scheduler + 2 backend nodes make deploy-ps # status make deploy-logs # logs diff --git a/docs/src/security/secure-sandboxes.md b/docs/src/security/secure-sandboxes.md index 9b763a4ae..b04948df3 100644 --- a/docs/src/security/secure-sandboxes.md +++ b/docs/src/security/secure-sandboxes.md @@ -2,8 +2,10 @@ Secure sandboxes use an envd access token for control-plane communication. This protects envd operations such as command execution and file access. -> [!WARNING] -> This feature does not add authentication to application ports exposed by the sandbox. +> [!NOTE] +> Secure mode protects envd control-plane operations. Application traffic uses +> the sandbox-scoped `trafficAccessToken` described in +> [Authentication](../configuration/authentication.md). Set `secure: true` when creating a sandbox through API or E2B-compatible SDKs to enable secure mode. Or use the CLI: @@ -11,7 +13,7 @@ Set `secure: true` when creating a sandbox through API or E2B-compatible SDKs to aenv start --secure ``` -The API and SDKs return the sandbox's `envdAccessToken` where appropriate and attach it to envd requests automatically. Forked sandboxes get independent tokens. Secure mode is preserved across pause, restart, and resume; legacy sandboxes remain non-secure unless created with `secure: true`. +The API and SDKs return the sandbox's `envdAccessToken` where appropriate and attach it to envd requests automatically. The application proxy credential is independent and is sent as `e2b-traffic-access-token`. Forked sandboxes get independent credentials. Secure mode is preserved across pause, restart, and resume; legacy sandboxes remain non-secure unless created with `secure: true`. ## Access-Token Seed @@ -36,11 +38,11 @@ The runtime DaemonSet reads the optional `agentenv-runtime-secrets` Secret. To c ```bash kubectl apply -f deploy/k8s/base/namespace.yaml -AENV_ACCESS_TOKEN_HASH_SEED="$(openssl rand -hex 32)" +AENV_SANDBOX_ACCESS_TOKEN_HASH_SEED="$(openssl rand -hex 32)" kubectl -n agentenv-system create secret generic agentenv-runtime-secrets \ - --from-literal="sandbox-access-token-hash-seed=${AENV_ACCESS_TOKEN_HASH_SEED}" \ + --from-literal="sandbox-access-token-hash-seed=${AENV_SANDBOX_ACCESS_TOKEN_HASH_SEED}" \ --dry-run=client -o yaml | kubectl apply -f - -unset AENV_ACCESS_TOKEN_HASH_SEED +unset AENV_SANDBOX_ACCESS_TOKEN_HASH_SEED ``` Run this once for a new cluster and preserve the existing Secret during upgrades. An external secret manager may be used instead, provided it creates the same Secret name and key: diff --git a/scripts/install.sh b/scripts/install.sh index 1cc31e2d8..9ccff8e0a 100644 --- a/scripts/install.sh +++ b/scripts/install.sh @@ -256,6 +256,7 @@ sudo chmod 0640 "$CONFIG_PATH" if [[ -d /run/systemd/system ]]; then ENV_FILE_STATUS="exists" if [[ ! -f "$ENV_FILE" ]]; then + sudo install -o root -g "$SERVICE_GROUP" -m 0640 /dev/null "$ENV_FILE" sudo tee "$ENV_FILE" > /dev/null <> "$tmp_env" fi - sudo install -m 0644 "$tmp_env" "$ENV_FILE" + sudo install -o root -g "$SERVICE_GROUP" -m 0640 "$tmp_env" "$ENV_FILE" rm -f "$current_env" "$tmp_env" ENV_FILE_STATUS="updated" fi @@ -370,6 +371,7 @@ echo " CLI : ${INSTALL_DIR}/aenv" echo " Server : ${INSTALL_DIR}/server" echo " Data : ${DATA_DIR}" echo " Config : ${CONFIG_PATH}" +echo " API key: generated on first server start in ${DATA_DIR}/secrets/api-key" echo " Mode : ${VIRTUALIZATION_MODE}" if [[ -d /run/systemd/system ]]; then if [[ "$ENV_FILE_STATUS" == "written" ]]; then diff --git a/scripts/tests/e2e/lib/helpers.sh b/scripts/tests/e2e/lib/helpers.sh index b84b9aad1..f5803b707 100644 --- a/scripts/tests/e2e/lib/helpers.sh +++ b/scripts/tests/e2e/lib/helpers.sh @@ -5,8 +5,7 @@ if [[ -z "${E2E_HELPERS_SH_LOADED:-}" ]]; then E2E_HELPERS_SH_LOADED=1 : "${AENV_URL:?AENV_URL must be set}" - : "${AENV_API_KEY:=e2e-test-key}" - : "${AENV_ADMIN_TOKEN:=e2e-admin-token}" + : "${AENV_API_KEY:=e2b_0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef}" : "${AENV_TEMPLATE_ID:=ubuntu}" : "${AENV_PROXY_URL:=${AENV_URL}/proxy}" : "${E2E_MODE:=single-node}" @@ -124,7 +123,7 @@ if [[ -z "${E2E_HELPERS_SH_LOADED:-}" ]]; then api_admin_get() { local path="$1" _curl_do -s \ - -H "X-Admin-Token: ${AENV_ADMIN_TOKEN}" \ + -H "X-API-Key: ${AENV_API_KEY}" \ "${AENV_URL}${path}" } @@ -132,7 +131,7 @@ if [[ -z "${E2E_HELPERS_SH_LOADED:-}" ]]; then local base_url="$1" local path="$2" _curl_do -s \ - -H "X-Admin-Token: ${AENV_ADMIN_TOKEN}" \ + -H "X-API-Key: ${AENV_API_KEY}" \ "${base_url}${path}" } @@ -140,7 +139,7 @@ if [[ -z "${E2E_HELPERS_SH_LOADED:-}" ]]; then local path="$1" [[ -z "$_E2E_HEADERS" ]] && _E2E_HEADERS=$(mktemp) curl -s -o "$_E2E_BODY" -D "$_E2E_HEADERS" -w '%{http_code}' \ - -H "X-Admin-Token: ${AENV_ADMIN_TOKEN}" \ + -H "X-API-Key: ${AENV_API_KEY}" \ "${AENV_URL}${path}" > "$_E2E_STATUS" 2>/dev/null || true HTTP_STATUS=$(<"$_E2E_STATUS") HTTP_BODY=$(<"$_E2E_BODY") @@ -152,7 +151,7 @@ if [[ -z "${E2E_HELPERS_SH_LOADED:-}" ]]; then local path="$2" [[ -z "$_E2E_HEADERS" ]] && _E2E_HEADERS=$(mktemp) curl -s -o "$_E2E_BODY" -D "$_E2E_HEADERS" -w '%{http_code}' \ - -H "X-Admin-Token: ${AENV_ADMIN_TOKEN}" \ + -H "X-API-Key: ${AENV_API_KEY}" \ "${base_url}${path}" > "$_E2E_STATUS" 2>/dev/null || true HTTP_STATUS=$(<"$_E2E_STATUS") HTTP_BODY=$(<"$_E2E_BODY") diff --git a/scripts/tests/e2e/lib/runtime.sh b/scripts/tests/e2e/lib/runtime.sh index 87886efeb..bdcf71596 100644 --- a/scripts/tests/e2e/lib/runtime.sh +++ b/scripts/tests/e2e/lib/runtime.sh @@ -11,8 +11,8 @@ if [[ -z "${E2E_RUNTIME_SH_LOADED:-}" ]]; then source "${E2E_RUNTIME_DIR}/server.sh" : "${E2E_MODE:=single-node}" - : "${AENV_API_KEY:=e2e-test-key}" - : "${AENV_ADMIN_TOKEN:=e2e-admin-token}" + : "${AENV_API_KEY:=e2b_0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef}" + export AENV_API_KEY : "${E2E_COMPOSE_FILE:=deploy/docker-compose.yml}" : "${E2E_COMPOSE_OVERRIDE_FILE:=scripts/tests/e2e/docker-compose.e2e.yml}" : "${E2E_COMPOSE_START_TIMEOUT:=120}" @@ -237,7 +237,7 @@ if [[ -z "${E2E_RUNTIME_SH_LOADED:-}" ]]; then log "Waiting for scheduler to observe ${expected_count} ready node(s) via ${AENV_URL}/nodes (timeout ${timeout}s) ..." for ((i = 1; i <= timeout; i++)); do response=$(curl -s \ - -H "X-Admin-Token: ${AENV_ADMIN_TOKEN}" \ + -H "X-API-Key: ${AENV_API_KEY}" \ -w $'\n%{http_code}' \ "${AENV_URL}/nodes" 2>/dev/null || true) status="${response##*$'\n'}" @@ -398,6 +398,12 @@ if [[ -z "${E2E_RUNTIME_SH_LOADED:-}" ]]; then _wait_for_health_url "agentenv-b" "${AENV_NODE_B_URL}" "${timeout}" || die "agentenv-b failed to become ready within ${timeout}s" + AENV_API_KEY="$(_compose_cmd exec -T agentenv-a cat /workspace/env/secrets/api-key)" || + die "Failed to read the Compose deployment API key" + [[ "${AENV_API_KEY}" =~ ^[A-Za-z0-9._~-]{32,}$ ]] || + die "Compose deployment returned an invalid API key" + export AENV_API_KEY + expected_nodes="$(_runtime_node_count)" [[ "${expected_nodes}" -gt 0 ]] || expected_nodes=1 _wait_for_scheduler_ready_nodes "${timeout}" "${expected_nodes}" || diff --git a/scripts/tests/e2e/lib/server.sh b/scripts/tests/e2e/lib/server.sh index c550b6613..1def30252 100644 --- a/scripts/tests/e2e/lib/server.sh +++ b/scripts/tests/e2e/lib/server.sh @@ -6,7 +6,7 @@ if [[ -z "${E2E_SERVER_SH_LOADED:-}" ]]; then : "${AENV_PORT:=18080}" : "${AENV_URL:=http://127.0.0.1:${AENV_PORT}}" - : "${AENV_API_KEY:=e2e-test-key}" + : "${AENV_API_KEY:=e2b_0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef}" : "${SERVER_START_TIMEOUT:=30}" _SERVER_PID="" @@ -17,6 +17,7 @@ if [[ -z "${E2E_SERVER_SH_LOADED:-}" ]]; then local env_vars=( "API_ADDR=127.0.0.1:${AENV_PORT}" + "AENV_API_KEY=${AENV_API_KEY}" "RUST_LOG=agentenv=info,envd=info" ) [[ -n "$config" ]] && env_vars+=("AENV_CONFIG_PATH=${config}") diff --git a/scripts/tests/e2e/suites/08_auth.sh b/scripts/tests/e2e/suites/08_auth.sh index 5f0d30dad..478375176 100755 --- a/scripts/tests/e2e/suites/08_auth.sh +++ b/scripts/tests/e2e/suites/08_auth.sh @@ -12,6 +12,25 @@ log "Suite: Authentication" api_get_no_auth "/sandboxes" assert_status "$HTTP_STATUS" "401" "no auth header returns 401" +# -- Alternative and malformed credentials are rejected -- +_curl_do -s -H "X-API-Key: wrong-key" "${AENV_URL}/sandboxes" +assert_status "$HTTP_STATUS" "401" "wrong API key returns 401" + +_curl_do -s -H "Authorization: Bearer ${AENV_API_KEY}" "${AENV_URL}/sandboxes" +assert_status "$HTTP_STATUS" "401" "Authorization does not authenticate AgentENV" + +_curl_do -s -H "X-Admin-Token: ${AENV_API_KEY}" "${AENV_URL}/sandboxes" +assert_status "$HTTP_STATUS" "401" "legacy admin token does not authenticate AgentENV" + +_curl_do -s -H "X-Team-ID: ${AENV_API_KEY}" "${AENV_URL}/sandboxes" +assert_status "$HTTP_STATUS" "401" "legacy team key does not authenticate AgentENV" + +_curl_do -s \ + -H "X-API-Key: ${AENV_API_KEY}" \ + -H "X-API-Key: ${AENV_API_KEY}" \ + "${AENV_URL}/sandboxes" +assert_status "$HTTP_STATUS" "401" "duplicate API key headers return 401" + # -- Request with valid API key succeeds -- api_get "/sandboxes" assert_not_eq "$HTTP_STATUS" "401" "valid API key does not return 401" diff --git a/scripts/tests/e2e/suites/09_e2b_compat.sh b/scripts/tests/e2e/suites/09_e2b_compat.sh index a5bb8ff68..611b09ad6 100755 --- a/scripts/tests/e2e/suites/09_e2b_compat.sh +++ b/scripts/tests/e2e/suites/09_e2b_compat.sh @@ -10,8 +10,7 @@ log "Suite: E2B Compatibility" export E2B_API_URL="${AENV_URL}" export E2B_SANDBOX_URL="${AENV_PROXY_URL}" -export E2B_API_KEY="e2b_000000" -export E2B_ACCESS_TOKEN="${AENV_API_KEY}" +export E2B_API_KEY="${AENV_API_KEY}" export E2B_COMPAT_USER_IMAGE="${E2B_COMPAT_USER_IMAGE:-${E2E_TEMPLATE_USER_IMAGE:-ghcr.io/linuxserver/baseimage-ubuntu:noble}}" cli_available=0 diff --git a/scripts/tests/e2e/suites/14_code_interpreter.sh b/scripts/tests/e2e/suites/14_code_interpreter.sh index e95b2a8b2..2b84c70a8 100755 --- a/scripts/tests/e2e/suites/14_code_interpreter.sh +++ b/scripts/tests/e2e/suites/14_code_interpreter.sh @@ -18,7 +18,7 @@ exit 0 export E2B_API_URL="${AENV_URL}" export E2B_SANDBOX_URL="${AENV_PROXY_URL}" -export E2B_API_KEY="e2b_000000" +export E2B_API_KEY="${AENV_API_KEY}" if ! python3 -c 'import e2b_code_interpreter' >/dev/null 2>&1; then warn "e2b_code_interpreter Python package not installed; skipping" diff --git a/services/README.md b/services/README.md index ee2880ca5..ff6743d65 100644 --- a/services/README.md +++ b/services/README.md @@ -69,14 +69,20 @@ Start scheduler: make run-scheduler ``` -Start gateway: +Start gateway with the same API key configured on every AgentENV runtime node: ```bash +export AENV_API_KEY="e2b_$(openssl rand -hex 32)" make run-gateway ``` The default local config uses `127.0.0.1:9090` for the scheduler. +The gateway and runtime nodes require the same API key. The gateway reads an +explicit `AENV_API_KEY` or `/run/secrets/api-key`; it does not generate one. +Application proxy requests may additionally use the sandbox response's +`trafficAccessToken` in the `e2b-traffic-access-token` header. + ## Scheduler configuration Scheduler discovery modes: @@ -176,6 +182,7 @@ LOG_FORMAT=json make run-gateway From **repository root**, start gateway + scheduler + two backend nodes: ```bash +# Run scripts/docker-setup.sh first for host prerequisites. make deploy-up ``` diff --git a/services/gateway/cmd/main.go b/services/gateway/cmd/main.go index b21738bfa..11a8255b9 100644 --- a/services/gateway/cmd/main.go +++ b/services/gateway/cmd/main.go @@ -4,10 +4,12 @@ import ( "context" "errors" "flag" + "fmt" "log" "net/http" "os" "os/signal" + "strings" "syscall" "time" @@ -22,6 +24,11 @@ import ( "google.golang.org/grpc/credentials/insecure" ) +const ( + apiKeyEnv = "AENV_API_KEY" + defaultAPIKeyPath = "/run/secrets/api-key" +) + func newSchedulerConn(addr string) (*grpc.ClientConn, error) { return grpc.NewClient( addr, @@ -29,6 +36,42 @@ func newSchedulerConn(addr string) (*grpc.ClientConn, error) { ) } +func loadAPIKey() (string, error) { + return loadAPIKeyFrom(os.LookupEnv, defaultAPIKeyPath) +} + +func loadAPIKeyFrom(lookupEnv func(string) (string, bool), secretPath string) (string, error) { + if value, present := lookupEnv(apiKeyEnv); present { + return validateAPIKey(value, apiKeyEnv) + } + + contents, err := os.ReadFile(secretPath) + if err != nil { + if os.IsNotExist(err) { + return "", fmt.Errorf("%s must be set or %s must exist", apiKeyEnv, secretPath) + } + return "", fmt.Errorf("read API key secret %s: %w", secretPath, err) + } + return validateAPIKey(string(contents), secretPath) +} + +func validateAPIKey(value, source string) (string, error) { + value = strings.TrimSpace(value) + if len(value) < 32 { + return "", fmt.Errorf("API key from %s must contain at least 32 URL-safe characters", source) + } + for _, char := range []byte(value) { + if (char >= 'a' && char <= 'z') || + (char >= 'A' && char <= 'Z') || + (char >= '0' && char <= '9') || + char == '.' || char == '_' || char == '~' || char == '-' { + continue + } + return "", fmt.Errorf("API key from %s must contain at least 32 URL-safe characters", source) + } + return value, nil +} + func main() { configPath := flag.String("config", "", "path to JSON config file") flag.Parse() @@ -37,6 +80,10 @@ func main() { if err != nil { log.Fatalf("load config failed: %v", err) } + apiKey, err := loadAPIKey() + if err != nil { + log.Fatalf("load API key failed: %v", err) + } logger, err := logging.New(cfg.LogLevel, cfg.LogFormat) if err != nil { @@ -65,6 +112,7 @@ func main() { s, err := gateway.NewServer(logger, schedulerClient, gateway.ServerOptions{ RequestTimeout: cfg.Gateway.RequestTimeout, MaxResponseSize: cfg.Gateway.ForwardResponseSize, + APIKey: apiKey, DebugMode: cfg.Gateway.DebugMode, SandboxProxyDomains: cfg.Gateway.SandboxProxyDomains, QueryOnlySchedulerClient: queryOnlySchedulerClient, diff --git a/services/gateway/cmd/main_test.go b/services/gateway/cmd/main_test.go new file mode 100644 index 000000000..9064986b3 --- /dev/null +++ b/services/gateway/cmd/main_test.go @@ -0,0 +1,77 @@ +package main + +import ( + "os" + "path/filepath" + "strings" + "testing" +) + +const testAPIKey = "e2b_0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef" + +func TestValidateAPIKey(t *testing.T) { + t.Parallel() + + got, err := validateAPIKey(" "+testAPIKey+"\n", "test") + if err != nil { + t.Fatalf("validateAPIKey() error = %v", err) + } + if got != testAPIKey { + t.Fatalf("validateAPIKey() = %q, want %q", got, testAPIKey) + } + + for _, invalid := range []string{"", "too-short", strings.Repeat("a", 31), strings.Repeat("a", 31) + "!"} { + if _, err := validateAPIKey(invalid, "test"); err == nil { + t.Errorf("validateAPIKey(%q) unexpectedly succeeded", invalid) + } + } +} + +func TestLoadAPIKeyFromEnvironment(t *testing.T) { + got, err := loadAPIKeyFrom( + func(name string) (string, bool) { return testAPIKey, name == apiKeyEnv }, + filepath.Join(t.TempDir(), "missing"), + ) + if err != nil { + t.Fatalf("loadAPIKey() error = %v", err) + } + if got != testAPIKey { + t.Fatalf("loadAPIKey() = %q, want %q", got, testAPIKey) + } +} + +func TestLoadAPIKeyRejectsExplicitEmptyEnvironment(t *testing.T) { + if _, err := loadAPIKeyFrom( + func(name string) (string, bool) { return "", name == apiKeyEnv }, + filepath.Join(t.TempDir(), "missing"), + ); err == nil { + t.Fatal("loadAPIKey() unexpectedly accepted an empty environment value") + } +} + +func TestLoadAPIKeyFromFile(t *testing.T) { + t.Parallel() + + dir := t.TempDir() + path := filepath.Join(dir, "api-key") + if err := os.WriteFile(path, []byte(testAPIKey+"\n"), 0o444); err != nil { + t.Fatal(err) + } + got, err := loadAPIKeyFrom(func(string) (string, bool) { return "", false }, path) + if err != nil { + t.Fatalf("loadAPIKeyFrom() error = %v", err) + } + if got != testAPIKey { + t.Fatalf("loadAPIKeyFrom() = %q, want %q", got, testAPIKey) + } +} + +func TestLoadAPIKeyRejectsMissingFile(t *testing.T) { + t.Parallel() + + dir := t.TempDir() + missing := filepath.Join(dir, "missing") + if _, err := loadAPIKeyFrom(func(string) (string, bool) { return "", false }, missing); err == nil { + t.Fatal("loadAPIKeyFrom() unexpectedly accepted a missing secret") + } +} diff --git a/services/gateway/internal/server.go b/services/gateway/internal/server.go index 2f6d08292..9f67619b0 100644 --- a/services/gateway/internal/server.go +++ b/services/gateway/internal/server.go @@ -3,6 +3,9 @@ package gateway import ( "bytes" "context" + "crypto/hmac" + "crypto/sha256" + "encoding/base64" "encoding/json" "errors" "io" @@ -22,6 +25,12 @@ import ( ) const ( + headerAPIKey = "X-API-Key" + headerTrafficToken = "e2b-traffic-access-token" + headerEnvdAccessToken = "X-Access-Token" + trafficTokenPrefix = "aenv_trf_" + trafficTokenContext = "agentenv-sandbox-traffic-v1\x00" + envdControlPlanePort = 49983 headerSandboxID = "x-agentenv-sandbox-id" headerE2BSandboxID = "e2b-sandbox-id" headerTargetPort = "x-agentenv-target-port" @@ -41,6 +50,7 @@ const ( ) type ServerOptions struct { + APIKey string RequestTimeout time.Duration MaxResponseSize int64 DebugMode bool @@ -53,6 +63,7 @@ type Server struct { scheduler schedulerv1.SchedulerClient queryOnlyScheduler schedulerv1.SchedulerClient httpClient *http.Client + apiKey []byte requestTimeout time.Duration maxRespSize int64 // debugMode, when true, enables debug-only behaviors such as exposing @@ -63,6 +74,11 @@ type Server struct { } func NewServer(logger *zap.Logger, schedulerClient schedulerv1.SchedulerClient, options ServerOptions) (*Server, error) { + apiKey := strings.TrimSpace(options.APIKey) + if apiKey == "" { + return nil, errors.New("API key is required") + } + sandboxProxyDomains, err := normalizeProxyDomains(options.SandboxProxyDomains) if err != nil { return nil, err @@ -80,6 +96,7 @@ func NewServer(logger *zap.Logger, schedulerClient schedulerv1.SchedulerClient, httpClient: &http.Client{}, requestTimeout: options.RequestTimeout, maxRespSize: options.MaxResponseSize, + apiKey: []byte(apiKey), debugMode: options.DebugMode, sandboxProxyDomains: sandboxProxyDomains, }, nil @@ -122,7 +139,7 @@ func (s *Server) Handler() http.Handler { } s.handleProxy(w, r) }) - return s.instrumentGatewayHTTP(core) + return s.instrumentGatewayHTTP(s.authenticate(core)) } func (s *Server) writeJSON(w http.ResponseWriter, status int, value any) { @@ -812,3 +829,89 @@ func extractSandboxIDsFromResponse(body []byte) []string { } return unique } + +func singleHeaderMatches(headers http.Header, name string, expected []byte) bool { + values := headers.Values(name) + return len(values) == 1 && bytes.Equal([]byte(values[0]), expected) +} + +func trafficAccessToken(apiKey []byte, sandboxID string) string { + mac := hmac.New(sha256.New, apiKey) + _, _ = mac.Write([]byte(trafficTokenContext)) + _, _ = mac.Write([]byte(sandboxID)) + return trafficTokenPrefix + base64.RawURLEncoding.EncodeToString(mac.Sum(nil)) +} + +func (s *Server) isSandboxDataPlaneRequest(r *http.Request) bool { + if strings.TrimRight(r.URL.Path, "/") == "/proxy" || strings.HasPrefix(r.URL.Path, "/proxy/") { + return true + } + + hostRoute, err := parseHostRoute(r.Host, s.sandboxProxyDomains) + if hostRoute != nil || err != nil { + return true + } + + return !isSandboxControlPlaneRequest(r) && hasProxyRoutingHeaders(r.Header) +} + +func (s *Server) sandboxIDForDataPlaneAuth(r *http.Request) (string, bool) { + hostRoute, err := parseHostRoute(r.Host, s.sandboxProxyDomains) + if err != nil { + return "", false + } + if hostRoute != nil { + return hostRoute.sandboxID, true + } + return sandboxIDFromHeaders(r.Header) +} + +func (s *Server) isEnvdDataPlaneRequest(r *http.Request) bool { + hostRoute, err := parseHostRoute(r.Host, s.sandboxProxyDomains) + if err != nil { + return false + } + if hostRoute != nil { + return hostRoute.targetPort == envdControlPlanePort + } + targetPort, ok := targetPortFromHeaders(r.Header) + if !ok { + return false + } + port, err := strconv.Atoi(targetPort) + return err == nil && port == envdControlPlanePort +} + +func hasSingleNonEmptyHeader(headers http.Header, name string) bool { + values := headers.Values(name) + return len(values) == 1 && strings.TrimSpace(values[0]) != "" +} + +func (s *Server) authenticate(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + dataPlane := s.isSandboxDataPlaneRequest(r) + if r.URL.Path == "/health" && !dataPlane { + next.ServeHTTP(w, r) + return + } + + authorized := singleHeaderMatches(r.Header, headerAPIKey, s.apiKey) + if !authorized && dataPlane { + if sandboxID, ok := s.sandboxIDForDataPlaneAuth(r); ok { + expected := trafficAccessToken(s.apiKey, sandboxID) + authorized = singleHeaderMatches(r.Header, headerTrafficToken, []byte(expected)) + } + } + if !authorized && dataPlane && s.isEnvdDataPlaneRequest(r) { + // The runtime node owns the envd token seed and performs the definitive + // sandbox-scoped validation before forwarding the request to envd. + authorized = hasSingleNonEmptyHeader(r.Header, headerEnvdAccessToken) + } + if !authorized { + w.WriteHeader(http.StatusUnauthorized) + return + } + + next.ServeHTTP(w, r) + }) +} diff --git a/services/gateway/internal/server_test.go b/services/gateway/internal/server_test.go index 098811503..0a86d860b 100644 --- a/services/gateway/internal/server_test.go +++ b/services/gateway/internal/server_test.go @@ -148,6 +148,8 @@ func (s stubSchedulerClient) UnregisterNode(ctx context.Context, req *schedulerv return s.unregisterNodeFunc(ctx, req, opts...) } +const testAPIKey = "test-api-key" + type testServerOption func(*ServerOptions) func newTestServer(t *testing.T, schedulerClient schedulerv1.SchedulerClient, timeout time.Duration, maxRespSize int64, opts ...testServerOption) *Server { @@ -156,6 +158,7 @@ func newTestServer(t *testing.T, schedulerClient schedulerv1.SchedulerClient, ti options := ServerOptions{ RequestTimeout: timeout, MaxResponseSize: maxRespSize, + APIKey: testAPIKey, } for _, opt := range opts { opt(&options) @@ -168,6 +171,143 @@ func newTestServer(t *testing.T, schedulerClient schedulerv1.SchedulerClient, ti return server } +func authenticatedTestHandler(server *Server) http.Handler { + handler := server.Handler() + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + r.Header.Set(headerAPIKey, testAPIKey) + handler.ServeHTTP(w, r) + }) +} + +func TestNewServerRejectsEmptyAPIKey(t *testing.T) { + _, err := NewServer(zap.NewNop(), stubSchedulerClient{}, ServerOptions{ + RequestTimeout: time.Second, + MaxResponseSize: 1024, + }) + if err == nil { + t.Fatal("NewServer accepted an empty API key") + } +} + +func TestGatewayRequiresExactAPIKey(t *testing.T) { + server := newTestServer(t, stubSchedulerClient{}, time.Second, 1024) + handler := server.Handler() + tests := []struct { + name string + addHeaders func(http.Header) + wantStatus int + }{ + { + name: "missing", + addHeaders: func(http.Header) {}, + wantStatus: http.StatusUnauthorized, + }, + { + name: "wrong", + addHeaders: func(headers http.Header) { + headers.Set(headerAPIKey, "wrong-key") + }, + wantStatus: http.StatusUnauthorized, + }, + { + name: "authorization is application data", + addHeaders: func(headers http.Header) { + headers.Set("Authorization", "Bearer "+testAPIKey) + }, + wantStatus: http.StatusUnauthorized, + }, + { + name: "valid", + addHeaders: func(headers http.Header) { + headers.Set(headerAPIKey, testAPIKey) + }, + wantStatus: http.StatusNotFound, + }, + { + name: "duplicate", + addHeaders: func(headers http.Header) { + headers.Add(headerAPIKey, testAPIKey) + headers.Add(headerAPIKey, testAPIKey) + }, + wantStatus: http.StatusUnauthorized, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + req := httptest.NewRequest(http.MethodGet, "/metrics", nil) + tt.addHeaders(req.Header) + recorder := httptest.NewRecorder() + + handler.ServeHTTP(recorder, req) + + if recorder.Code != tt.wantStatus { + t.Fatalf("status = %d, want %d", recorder.Code, tt.wantStatus) + } + }) + } +} + +func TestGatewayAcceptsSandboxScopedTrafficToken(t *testing.T) { + const sandboxID = "0191f4d0-7b2a-7c11-9c2d-0123456789ab" + lookupCalls := 0 + server := newTestServer(t, stubSchedulerClient{ + lookupNodeFunc: func(context.Context, *schedulerv1.LookupNodeRequest, ...grpc.CallOption) (*schedulerv1.LookupNodeResponse, error) { + lookupCalls++ + return nil, fmt.Errorf("lookup reached") + }, + }, time.Second, 1024) + handler := server.Handler() + + req := httptest.NewRequest(http.MethodGet, "/proxy", nil) + req.Header.Set(headerE2BSandboxID, sandboxID) + req.Header.Set(headerE2BTargetPort, "49983") + req.Header.Set(headerTrafficToken, trafficAccessToken([]byte(testAPIKey), sandboxID)) + recorder := httptest.NewRecorder() + handler.ServeHTTP(recorder, req) + if recorder.Code == http.StatusUnauthorized || lookupCalls != 1 { + t.Fatalf("valid scoped token: status=%d lookup calls=%d", recorder.Code, lookupCalls) + } + + req = httptest.NewRequest(http.MethodGet, "/proxy", nil) + req.Header.Set(headerE2BSandboxID, sandboxID) + req.Header.Set(headerE2BTargetPort, "49983") + req.Header.Set(headerTrafficToken, trafficAccessToken([]byte(testAPIKey), "another-sandbox")) + recorder = httptest.NewRecorder() + handler.ServeHTTP(recorder, req) + if recorder.Code != http.StatusUnauthorized || lookupCalls != 1 { + t.Fatalf("wrong scoped token: status=%d lookup calls=%d", recorder.Code, lookupCalls) + } + + req = httptest.NewRequest(http.MethodGet, "/proxy", nil) + req.Header.Set(headerE2BSandboxID, sandboxID) + req.Header.Set(headerE2BTargetPort, "8080") + req.Header.Set("X-Access-Token", trafficAccessToken([]byte(testAPIKey), sandboxID)) + recorder = httptest.NewRecorder() + handler.ServeHTTP(recorder, req) + if recorder.Code != http.StatusUnauthorized || lookupCalls != 1 { + t.Fatalf("envd token authorized application proxy: status=%d lookup calls=%d", recorder.Code, lookupCalls) + } + + req = httptest.NewRequest(http.MethodPost, "/sandboxes/"+sandboxID+"/pause", nil) + req.Header.Set(headerE2BSandboxID, sandboxID) + req.Header.Set(headerTrafficToken, trafficAccessToken([]byte(testAPIKey), sandboxID)) + recorder = httptest.NewRecorder() + handler.ServeHTTP(recorder, req) + + if recorder.Code != http.StatusUnauthorized || lookupCalls != 1 { + t.Fatalf("scoped token reached control plane: status=%d lookup calls=%d", recorder.Code, lookupCalls) + } +} + +func TestTrafficAccessTokenVector(t *testing.T) { + const sandboxID = "0191f4d0-7b2a-7c11-9c2d-0123456789ab" + const want = "aenv_trf_PwHqhTxLa_mzUCNIGx03uiTHxZ3k995pKDOS50PaGWo" + if got := trafficAccessToken([]byte("test-key"), sandboxID); got != want { + t.Fatalf("trafficAccessToken() = %q, want %q", got, want) + } +} + func withSandboxProxyDomains(domains ...string) testServerOption { return func(options *ServerOptions) { options.SandboxProxyDomains = domains @@ -398,7 +538,7 @@ func TestHandleProxyReturnsAggregatedNodesFromScheduler(t *testing.T) { request := httptest.NewRequest(http.MethodGet, "http://gateway.test/nodes?clusterID=cluster-1", nil) response := httptest.NewRecorder() - server.Handler().ServeHTTP(response, request) + authenticatedTestHandler(server).ServeHTTP(response, request) if response.Code != http.StatusOK { t.Fatalf("expected status 200, got %d", response.Code) @@ -453,7 +593,7 @@ func TestHandleProxyDirectForwardsNodeDetail(t *testing.T) { request := httptest.NewRequest(http.MethodGet, "http://gateway.test/nodes/node-a?clusterID=cluster-1", nil) response := httptest.NewRecorder() - server.Handler().ServeHTTP(response, request) + authenticatedTestHandler(server).ServeHTTP(response, request) if response.Code != http.StatusOK { t.Fatalf("expected status 200, got %d", response.Code) @@ -486,7 +626,7 @@ func TestLookupNodeUsesQueryOnlySchedulerClient(t *testing.T) { } server := newTestServer(t, mainScheduler, time.Second, 1024, withQueryOnlyScheduler(queryScheduler)) - gatewayServer := httptest.NewServer(server.Handler()) + gatewayServer := httptest.NewServer(authenticatedTestHandler(server)) defer gatewayServer.Close() req, err := http.NewRequest(http.MethodGet, gatewayServer.URL+"/health", nil) @@ -555,7 +695,7 @@ func TestSandboxControlPlaneRequestWithE2BHeadersUsesPathRoute(t *testing.T) { }, }, time.Second, 1024) - gatewayServer := httptest.NewServer(server.Handler()) + gatewayServer := httptest.NewServer(authenticatedTestHandler(server)) defer gatewayServer.Close() req, err := http.NewRequest(http.MethodPost, gatewayServer.URL+"/sandboxes/sbx-path/connect", strings.NewReader(`{"timeout":60}`)) @@ -833,7 +973,7 @@ func TestHandleProxyAggregatesSandboxListAcrossNodes(t *testing.T) { }, }, time.Second, 1024) - gatewayServer := httptest.NewServer(server.Handler()) + gatewayServer := httptest.NewServer(authenticatedTestHandler(server)) defer gatewayServer.Close() req, err := http.NewRequest(http.MethodGet, gatewayServer.URL+"/sandboxes?metadata=team%3Dalpha", nil) @@ -909,7 +1049,7 @@ func TestHandleProxyAggregatesV2SandboxesWithGlobalPagination(t *testing.T) { }, }, time.Second, 1024) - gatewayServer := httptest.NewServer(server.Handler()) + gatewayServer := httptest.NewServer(authenticatedTestHandler(server)) defer gatewayServer.Close() req, err := http.NewRequest(http.MethodGet, gatewayServer.URL+"/v2/sandboxes?metadata=team%3Dalpha&state=running%2Cpaused&limit=2", nil) @@ -1012,7 +1152,7 @@ func TestHandleProxyAggregatesSandboxListDedupsDuplicateSandboxIDs(t *testing.T) }, }, time.Second, 1024) - gatewayServer := httptest.NewServer(server.Handler()) + gatewayServer := httptest.NewServer(authenticatedTestHandler(server)) defer gatewayServer.Close() resp, err := http.Get(gatewayServer.URL + "/v2/sandboxes?limit=10") @@ -1055,7 +1195,7 @@ func TestHandleProxyClusterListFailsWhenNodeFails(t *testing.T) { }, }, time.Second, 1024) - gatewayServer := httptest.NewServer(server.Handler()) + gatewayServer := httptest.NewServer(authenticatedTestHandler(server)) defer gatewayServer.Close() resp, err := http.Get(gatewayServer.URL + "/sandboxes") @@ -1085,7 +1225,7 @@ func TestHandleProxyClusterListPropagatesUnauthorized(t *testing.T) { }, }, time.Second, 1024) - gatewayServer := httptest.NewServer(server.Handler()) + gatewayServer := httptest.NewServer(authenticatedTestHandler(server)) defer gatewayServer.Close() resp, err := http.Get(gatewayServer.URL + "/sandboxes") @@ -1379,7 +1519,7 @@ func TestMetricsEndpointReturnsNotFoundWithoutProxyRouting(t *testing.T) { func TestHealthEndpointReturnsGatewayHealthWithoutProxyHeaders(t *testing.T) { server := newTestServer(t, stubSchedulerClient{}, time.Second, 1024) - gatewayServer := httptest.NewServer(server.Handler()) + gatewayServer := httptest.NewServer(authenticatedTestHandler(server)) defer gatewayServer.Close() resp, err := http.Get(gatewayServer.URL + "/health") @@ -1430,7 +1570,7 @@ func TestHealthAndMetricsEndpointsWithSandboxHeadersProxyToSandbox(t *testing.T) }, }, time.Second, 1024) - gatewayServer := httptest.NewServer(server.Handler()) + gatewayServer := httptest.NewServer(authenticatedTestHandler(server)) defer gatewayServer.Close() for _, path := range []string{"/health", "/metrics"} { @@ -1468,7 +1608,7 @@ func TestHealthAndMetricsEndpointsWithSandboxHeadersProxyToSandbox(t *testing.T) func TestHealthAndMetricsEndpointsWithProxyHeadersMissingSandboxIDReturnBadRequest(t *testing.T) { server := newTestServer(t, stubSchedulerClient{}, time.Second, 1024) - gatewayServer := httptest.NewServer(server.Handler()) + gatewayServer := httptest.NewServer(authenticatedTestHandler(server)) defer gatewayServer.Close() for _, path := range []string{"/health", "/metrics"} { @@ -1523,7 +1663,7 @@ func TestHealthAndMetricsEndpointsWithHostRoutingProxyToSandbox(t *testing.T) { }, nil }, }, time.Second, 1024, withSandboxProxyDomains("sandbox-proxy.example.invalid")) - gatewayServer := httptest.NewServer(server.Handler()) + gatewayServer := httptest.NewServer(authenticatedTestHandler(server)) defer gatewayServer.Close() for _, path := range []string{"/health", "/metrics"} { @@ -1602,7 +1742,7 @@ func TestHandleProxyHostBasedRoutingForwardsToSandboxProxy(t *testing.T) { }, }, time.Second, 1024, withSandboxProxyDomains("sandbox-proxy.example.invalid")) - gatewayServer := httptest.NewServer(server.Handler()) + gatewayServer := httptest.NewServer(authenticatedTestHandler(server)) defer gatewayServer.Close() req, err := http.NewRequest(http.MethodGet, gatewayServer.URL+"/readyz?x=1", nil) @@ -1650,7 +1790,7 @@ func TestHandleProxyHostBasedRoutingForwardsToSandboxProxy(t *testing.T) { func TestHandleProxyHostBasedRoutingRejectsInvalidHost(t *testing.T) { server := newTestServer(t, stubSchedulerClient{}, time.Second, 1024, withSandboxProxyDomains("sandbox-proxy.example.invalid")) - gatewayServer := httptest.NewServer(server.Handler()) + gatewayServer := httptest.NewServer(authenticatedTestHandler(server)) defer gatewayServer.Close() req, err := http.NewRequest(http.MethodGet, gatewayServer.URL+"/readyz", nil) @@ -1736,7 +1876,7 @@ func TestHandleProxyHTTPForwardingAndRecordAssignment(t *testing.T) { }, }, time.Second, 1024, withDebugMode(true)) - gatewayServer := httptest.NewServer(server.Handler()) + gatewayServer := httptest.NewServer(authenticatedTestHandler(server)) defer gatewayServer.Close() req, err := http.NewRequest(http.MethodPost, gatewayServer.URL+"/sandboxes", strings.NewReader(`{"template":"base"}`)) @@ -1860,7 +2000,7 @@ func TestHandleProxyColdSandboxCreateRecordsAssignment(t *testing.T) { }, }, time.Second, 1024, withDebugMode(true)) - gatewayServer := httptest.NewServer(server.Handler()) + gatewayServer := httptest.NewServer(authenticatedTestHandler(server)) defer gatewayServer.Close() req, err := http.NewRequest(http.MethodPost, gatewayServer.URL+"/sandboxes-cold", strings.NewReader(`{"image":"ubuntu:24.04"}`)) @@ -1992,7 +2132,7 @@ func TestHandleProxyWebSocketForwarding(t *testing.T) { }, }, time.Second, 1024) - gatewayServer := httptest.NewServer(server.Handler()) + gatewayServer := httptest.NewServer(authenticatedTestHandler(server)) defer gatewayServer.Close() gatewayURL, err := url.Parse(gatewayServer.URL) @@ -2108,7 +2248,7 @@ func TestHandleProxyPreservesEncodedPathSegments(t *testing.T) { }, }, time.Second, 1024) - gatewayServer := httptest.NewServer(server.Handler()) + gatewayServer := httptest.NewServer(authenticatedTestHandler(server)) defer gatewayServer.Close() tests := []struct { diff --git a/src/api/impls/auth.rs b/src/api/impls/auth.rs index c63625835..6d5257243 100644 --- a/src/api/impls/auth.rs +++ b/src/api/impls/auth.rs @@ -1,41 +1,120 @@ use async_trait::async_trait; -use axum::http::header::HeaderMap; +use axum::{ + body::Body, + extract::{Request, State}, + http::{header::HeaderMap, StatusCode}, + middleware::Next, + response::{IntoResponse, Response}, +}; +use base64::{engine::general_purpose::URL_SAFE_NO_PAD, Engine}; +use hmac::{Hmac, Mac}; +use sha2::Sha256; use agentenv_http_server::apis; use super::{ApiImpl, Claims}; +use crate::api::proxy; -fn non_empty_header(headers: &HeaderMap, name: &str) -> bool { - headers - .get(name) - .and_then(|value| value.to_str().ok()) - .is_some_and(|value| !value.is_empty()) +pub(crate) const API_KEY_HEADER: &str = "x-api-key"; +pub(crate) const TRAFFIC_ACCESS_TOKEN_HEADER: &str = "e2b-traffic-access-token"; +pub(crate) const ENVD_ACCESS_TOKEN_HEADER: &str = "x-access-token"; +const TRAFFIC_TOKEN_PREFIX: &str = "aenv_trf_"; +const TRAFFIC_TOKEN_CONTEXT: &[u8] = b"agentenv-sandbox-traffic-v1\0"; + +fn single_header_matches(headers: &HeaderMap, name: &str, expected: &str) -> bool { + let mut values = headers.get_all(name).iter(); + let Some(value) = values.next() else { + return false; + }; + if values.next().is_some() { + return false; + } + + value.as_bytes() == expected.as_bytes() +} + +fn derive_traffic_access_token(api_key: &[u8], sandbox_id: &str) -> String { + let mut mac = + Hmac::::new_from_slice(api_key).expect("HMAC accepts API keys of any length"); + mac.update(TRAFFIC_TOKEN_CONTEXT); + mac.update(sandbox_id.as_bytes()); + format!( + "{TRAFFIC_TOKEN_PREFIX}{}", + URL_SAFE_NO_PAD.encode(mac.finalize().into_bytes()) + ) +} + +impl ApiImpl { + pub(crate) fn has_valid_api_key(&self, headers: &HeaderMap) -> bool { + single_header_matches(headers, API_KEY_HEADER, &self.api_key) + } + + pub(crate) fn traffic_access_token(&self, sandbox_id: &str) -> String { + derive_traffic_access_token(self.api_key.as_bytes(), sandbox_id) + } + + fn has_valid_traffic_access_token(&self, headers: &HeaderMap, sandbox_id: &str) -> bool { + let expected = self.traffic_access_token(sandbox_id); + single_header_matches(headers, TRAFFIC_ACCESS_TOKEN_HEADER, &expected) + } +} + +pub(crate) async fn require_auth( + State(api_impl): State, + request: Request, + next: Next, +) -> Response +where + I: AsRef + Clone + Send + Sync + 'static, +{ + let proxy_request = + proxy::is_sandbox_proxy_request(&request, api_impl.as_ref().sandbox_proxy_domains()); + if request.uri().path() == "/health" && !proxy_request { + return next.run(request).await; + } + + let mut authorized = api_impl.as_ref().has_valid_api_key(request.headers()); + if !authorized && proxy_request { + authorized = + proxy::sandbox_id_for_proxy_auth(&request, api_impl.as_ref().sandbox_proxy_domains()) + .is_some_and(|sandbox_id| { + api_impl + .as_ref() + .has_valid_traffic_access_token(request.headers(), &sandbox_id) + }); + } + if !authorized && proxy_request { + if let Some((sandbox_id, target_port, candidate)) = proxy::envd_access_token_for_proxy_auth( + &request, + api_impl.as_ref().sandbox_proxy_domains(), + ) { + authorized = proxy::has_valid_envd_access_token( + api_impl.as_ref(), + sandbox_id, + target_port, + candidate, + ) + .await; + } + } + + if !authorized { + return StatusCode::UNAUTHORIZED.into_response(); + } + + next.run(request).await } #[async_trait] impl apis::ApiKeyAuthHeader for ApiImpl { type Claims = Claims; - // TODO: Validate configured authentication credentials instead of only - // checking that they are present. async fn extract_claims_from_header( &self, headers: &HeaderMap, - key: &str, + _key: &str, ) -> Option { - let admin_token = non_empty_header(headers, "X-Admin-Token"); - if key == "X-Admin-Token" { - return admin_token.then_some(Claims); - } - - if non_empty_header(headers, "X-API-Key") - || non_empty_header(headers, "X-Team-ID") - || admin_token - { - Some(Claims) - } else { - None - } + self.has_valid_api_key(headers).then_some(Claims) } } @@ -45,17 +124,53 @@ impl apis::ApiAuthBasic for ApiImpl { async fn extract_claims_from_auth_header( &self, - kind: apis::BasicAuthKind, + _kind: apis::BasicAuthKind, headers: &HeaderMap, - key: &str, + _key: &str, ) -> Option { - let expected_scheme = match kind { - apis::BasicAuthKind::Basic => "Basic", - apis::BasicAuthKind::Bearer => "Bearer", - _ => return None, - }; - let value = headers.get(key)?.to_str().ok()?; - let (scheme, credentials) = value.split_once(' ')?; - (scheme.eq_ignore_ascii_case(expected_scheme) && !credentials.is_empty()).then_some(Claims) + // The outer middleware is authoritative. This adapter keeps the + // E2B-compatible generated router from rejecting its API-key request. + self.has_valid_api_key(headers).then_some(Claims) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn header_match_requires_one_exact_value() { + let mut headers = HeaderMap::new(); + assert!(!single_header_matches( + &headers, + API_KEY_HEADER, + "correct-key" + )); + headers.insert(API_KEY_HEADER, "correct-key".parse().unwrap()); + assert!(single_header_matches( + &headers, + API_KEY_HEADER, + "correct-key" + )); + assert!(!single_header_matches( + &headers, + API_KEY_HEADER, + "wrong-key" + )); + + headers.append(API_KEY_HEADER, "correct-key".parse().unwrap()); + assert!(!single_header_matches( + &headers, + API_KEY_HEADER, + "correct-key" + )); + } + + #[test] + fn traffic_access_token_matches_gateway_contract() { + assert_eq!( + derive_traffic_access_token(b"test-key", "0191f4d0-7b2a-7c11-9c2d-0123456789ab"), + "aenv_trf_PwHqhTxLa_mzUCNIGx03uiTHxZ3k995pKDOS50PaGWo" + ); } } diff --git a/src/api/impls/mod.rs b/src/api/impls/mod.rs index 1e1d50bc3..fb8f2bb71 100644 --- a/src/api/impls/mod.rs +++ b/src/api/impls/mod.rs @@ -1,6 +1,6 @@ mod admin; mod attached_drives; -mod auth; +pub(crate) mod auth; mod pagination; mod sandbox; mod snapshots; @@ -33,6 +33,7 @@ pub struct ApiImpl { observability: Option>, proxy_client: ProxyClient, sandbox_proxy_domains: Vec, + api_key: String, } impl ApiImpl { @@ -43,6 +44,7 @@ impl ApiImpl { image_resolver: Arc, observability: Option>, sandbox_proxy_domains: Vec, + api_key: String, ) -> Self { Self { orchestrator, @@ -52,6 +54,7 @@ impl ApiImpl { observability, proxy_client: build_proxy_client(), sandbox_proxy_domains, + api_key, } } diff --git a/src/api/impls/sandbox.rs b/src/api/impls/sandbox.rs index 5382dfb60..85b91f84b 100644 --- a/src/api/impls/sandbox.rs +++ b/src/api/impls/sandbox.rs @@ -220,6 +220,9 @@ impl ApiImpl { .map(|token| token.expose().to_owned()); let mut sandbox = models::Sandbox::from(metadata); sandbox.envd_access_token = envd_access_token; + sandbox.traffic_access_token = Some(Nullable::Present( + self.traffic_access_token(&sandbox.sandbox_id), + )); sandbox.domain = self .sandbox_proxy_domains() .first() diff --git a/src/api/proxy.rs b/src/api/proxy.rs index 326c847f3..1cfcb015e 100644 --- a/src/api/proxy.rs +++ b/src/api/proxy.rs @@ -7,7 +7,7 @@ use axum::{ rejection::WebSocketUpgradeRejection, CloseFrame, Message as WebSocketMessage, WebSocket, WebSocketUpgrade, }, - FromRequestParts, Request, State, + FromRequestParts, MatchedPath, Request, State, }, http::{header, HeaderMap, HeaderName, HeaderValue, Method, Response, StatusCode, Uri}, middleware::Next, @@ -35,10 +35,16 @@ use tokio_tungstenite::{ use tracing::{debug, info, trace, warn}; use crate::{ - api::ApiImpl, + api::{ + impls::auth::{API_KEY_HEADER, ENVD_ACCESS_TOKEN_HEADER, TRAFFIC_ACCESS_TOKEN_HEADER}, + ApiImpl, + }, cfg::ConfigManager, observability::prometheus::HttpRouteSource, - orchestrator::{NewTimeout, OrchestratorError, ProxyLookupResult, ProxyTarget, SandboxState}, + orchestrator::{ + NewTimeout, OrchestratorError, ProxyLookupResult, ProxyTarget, SandboxMetadata, + SandboxState, + }, types::SandboxId, }; @@ -50,6 +56,8 @@ struct ResolvedProxyRequest { sandbox_id: SandboxId, upstream_uri: Uri, original_host: Option, + target_port: u16, + envd_port: u16, } #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -83,8 +91,6 @@ const E2B_SANDBOX_ID_HEADER: &str = "e2b-sandbox-id"; const TARGET_PORT_HEADER: &str = "x-agentenv-target-port"; /// E2B-compatible alias for the target port header. const E2B_TARGET_PORT_HEADER: &str = "e2b-sandbox-port"; -const ENVD_ACCESS_TOKEN_HEADER: &str = "x-access-token"; - #[cfg(test)] const PROXY_CONNECT_TIMEOUT: Duration = Duration::from_millis(100); #[cfg(not(test))] @@ -142,6 +148,91 @@ where .with_state(api_impl) } +fn proxy_route_for_auth(request: &Request, domains: &[String]) -> Option { + match parse_host_proxy_route(request_host(request), domains) { + Ok(Some(route)) => return Some(route), + Err(_) => return None, + Ok(None) => {} + } + + Some(HostProxyRoute { + sandbox_id: parse_sandbox_id_header(request.headers()).ok()?, + target_port: parse_target_port_header(request.headers()).ok()?, + }) +} + +pub(crate) fn sandbox_id_for_proxy_auth(request: &Request, domains: &[String]) -> Option { + Some( + proxy_route_for_auth(request, domains)? + .sandbox_id + .to_string(), + ) +} + +pub(crate) fn envd_access_token_for_proxy_auth( + request: &Request, + domains: &[String], +) -> Option<(SandboxId, u16, String)> { + let route = proxy_route_for_auth(request, domains)?; + let candidate = { + let mut candidates = request.headers().get_all(ENVD_ACCESS_TOKEN_HEADER).iter(); + let candidate = candidates.next()?; + if candidates.next().is_some() { + return None; + } + let candidate = candidate.to_str().ok()?; + candidate.to_owned() + }; + + Some((route.sandbox_id, route.target_port, candidate)) +} + +pub(crate) async fn has_valid_envd_access_token( + api_impl: &ApiImpl, + sandbox_id: SandboxId, + target_port: u16, + candidate: String, +) -> bool { + let Ok(Some(metadata)) = api_impl.orchestrator().get_sandbox(&sandbox_id).await else { + return false; + }; + if !metadata.secure || target_port != effective_envd_port(&metadata) { + return false; + } + + api_impl + .orchestrator() + .validate_envd_access_token(sandbox_id, &candidate) +} + +pub(crate) fn is_sandbox_proxy_request(request: &Request, domains: &[String]) -> bool { + let path = request.uri().path(); + if path == PROXY_ROUTE || path.starts_with("/proxy/") { + return true; + } + + match parse_host_proxy_route(request_host(request), domains) { + Ok(Some(_)) | Err(_) => true, + Ok(None) => { + request.extensions().get::().is_none() + && has_routing_header(request.headers()) + } + } +} + +fn request_host(request: &Request) -> Option<&str> { + request + .headers() + .get(header::HOST) + .and_then(|host| host.to_str().ok()) + .or_else(|| { + request + .uri() + .authority() + .map(|authority| authority.as_str()) + }) +} + pub(crate) async fn sandbox_proxy_classifier( State(api_impl): State, request: Request, @@ -167,7 +258,9 @@ where }); let host_route = match parse_host_proxy_route(host, api_impl.as_ref().sandbox_proxy_domains()) { Ok(Some(route)) => route, - Ok(None) => return next.run(request).await, + Ok(None) => { + return next.run(request).await; + } Err(err) => { return with_route_source(proxy_error_response(&err), HttpRouteSource::ProxyHost); } @@ -358,6 +451,14 @@ fn has_routing_header(headers: &HeaderMap) -> bool { headers.get(SANDBOX_ID_HEADER).is_some() || headers.get(E2B_SANDBOX_ID_HEADER).is_some() } +fn effective_envd_port(metadata: &SandboxMetadata) -> u16 { + metadata + .paused_state + .as_ref() + .and_then(|state| state.control_plane_port()) + .unwrap_or_else(|| ConfigManager::global_config().tools.control_plane_port) +} + /// Proxies a standard HTTP request to the resolved upstream URI and returns the response. async fn proxy_http_request( api_impl: &ApiImpl, @@ -369,9 +470,11 @@ async fn proxy_http_request( sandbox_id, upstream_uri, original_host, + target_port, + envd_port, } = resolved; - sanitize_request_headers(&mut parts.headers); + sanitize_request_headers(&mut parts.headers, target_port, envd_port); inject_forwarded_headers( &mut parts.headers, original_host.as_ref(), @@ -552,9 +655,11 @@ async fn proxy_websocket_request( sandbox_id, upstream_uri, original_host, + target_port, + envd_port, } = resolved; - sanitize_websocket_request_headers(&mut parts.headers); + sanitize_websocket_request_headers(&mut parts.headers, target_port, envd_port); inject_forwarded_headers( &mut parts.headers, original_host.as_ref(), @@ -735,6 +840,14 @@ async fn resolve_proxy_request( } }; + let metadata = api_impl + .orchestrator() + .get_sandbox(&sandbox_id) + .await + .map_err(|_| StatusCode::INTERNAL_SERVER_ERROR.into_response())? + .ok_or_else(|| proxy_error_response(&ProxyRequestError::SandboxNotFound(sandbox_id)))?; + let envd_port = effective_envd_port(&metadata); + let upstream_uri = if is_websocket_request { build_upstream_uri_with_scheme("ws", &target, target_port, proxy_path, parts.uri.query()) } else { @@ -746,6 +859,8 @@ async fn resolve_proxy_request( sandbox_id, upstream_uri, original_host: parts.headers.get(header::HOST).cloned(), + target_port, + envd_port, }) } @@ -755,15 +870,15 @@ async fn authorize_secure_envd_auto_resume( target_port: u16, headers: &HeaderMap, ) -> Result<(), Response> { - if target_port != ConfigManager::global_config().tools.control_plane_port { - return Ok(()); - } let metadata = api_impl .orchestrator() .get_sandbox(&sandbox_id) .await .map_err(|_| StatusCode::INTERNAL_SERVER_ERROR.into_response())? .ok_or_else(|| proxy_error_response(&ProxyRequestError::SandboxNotFound(sandbox_id)))?; + if target_port != effective_envd_port(&metadata) { + return Ok(()); + } if !metadata.secure { return Ok(()); } @@ -963,19 +1078,24 @@ fn build_upstream_uri_with_scheme( .map_err(|_| StatusCode::BAD_REQUEST) } -fn sanitize_request_headers(headers: &mut HeaderMap) { +fn sanitize_request_headers(headers: &mut HeaderMap, target_port: u16, envd_port: u16) { // These headers are only for the control-plane hop between the client and // AgentENV. Upstream sandbox services should not see them. headers.remove(SANDBOX_ID_HEADER); headers.remove(E2B_SANDBOX_ID_HEADER); headers.remove(TARGET_PORT_HEADER); headers.remove(E2B_TARGET_PORT_HEADER); + headers.remove(API_KEY_HEADER); + headers.remove(TRAFFIC_ACCESS_TOKEN_HEADER); headers.remove(header::HOST); + if target_port != envd_port { + headers.remove(ENVD_ACCESS_TOKEN_HEADER); + } remove_hop_by_hop_headers(headers); } -fn sanitize_websocket_request_headers(headers: &mut HeaderMap) { - sanitize_request_headers(headers); +fn sanitize_websocket_request_headers(headers: &mut HeaderMap, target_port: u16, envd_port: u16) { + sanitize_request_headers(headers, target_port, envd_port); headers.remove(header::SEC_WEBSOCKET_ACCEPT); headers.remove(header::SEC_WEBSOCKET_EXTENSIONS); headers.remove(header::SEC_WEBSOCKET_KEY); @@ -1389,9 +1509,14 @@ mod tests { "e2b_sandbox_header_seen": headers.get(E2B_SANDBOX_ID_HEADER).is_some(), "target_port_header_seen": headers.get(TARGET_PORT_HEADER).is_some(), "e2b_target_port_header_seen": headers.get(E2B_TARGET_PORT_HEADER).is_some(), - "envd_access_token": headers + "api_key_header_seen": headers.get(API_KEY_HEADER).is_some(), + "traffic_token_header_seen": headers.get(TRAFFIC_ACCESS_TOKEN_HEADER).is_some(), + "access_token": headers .get(ENVD_ACCESS_TOKEN_HEADER) .and_then(|value| value.to_str().ok()), + "authorization": headers + .get(header::AUTHORIZATION) + .and_then(|value| value.to_str().ok()), "forwarded_host": headers .get("x-forwarded-host") .and_then(|value| value.to_str().ok()), @@ -1622,6 +1747,7 @@ mod tests { image_resolver, None, domains, + "test-key".to_string(), )) } @@ -1650,11 +1776,27 @@ mod tests { .await } + async fn proxy_app_with_access_token_for_sandbox( + sandbox_id: &SandboxId, + ) -> (axum::Router, String) { + let api = build_api().await; + let access_token = api.traffic_access_token(&sandbox_id.to_string()); + api.orchestrator() + .set_proxy_target_for_test( + *sandbox_id, + ProxyTarget::new(Ipv4Addr::LOCALHOST), + crate::orchestrator::SandboxState::Running, + ) + .await; + (server::new(api), access_token) + } + async fn proxy_app_for_sandbox_with_domains( sandbox_id: &SandboxId, domains: Vec, - ) -> axum::Router { + ) -> (axum::Router, String) { let api = build_api_with_sandbox_proxy_domains(domains).await; + let access_token = api.traffic_access_token(&sandbox_id.to_string()); api.orchestrator() .set_proxy_target_for_test( *sandbox_id, @@ -1662,7 +1804,7 @@ mod tests { crate::orchestrator::SandboxState::Running, ) .await; - server::new(api) + (server::new(api), access_token) } async fn proxy_app_for_running_sandbox_without_route(sandbox_id: &SandboxId) -> axum::Router { @@ -1732,7 +1874,11 @@ mod tests { HeaderValue::from_static("keep"), ); - sanitize_request_headers(&mut headers); + sanitize_request_headers( + &mut headers, + 8080, + ConfigManager::global_config().tools.control_plane_port, + ); assert!(headers.get(SANDBOX_ID_HEADER).is_none()); assert!(headers.get(E2B_SANDBOX_ID_HEADER).is_none()); @@ -1786,6 +1932,120 @@ mod tests { assert!(!is_send_request_failure_text(&"client error (Connect)")); } + #[tokio::test] + async fn server_requires_exact_api_key_and_leaves_health_public() { + let app = server::new(build_api().await); + + for request in [ + Request::builder() + .uri("/nonexistent/path") + .body(Body::empty()) + .unwrap(), + Request::builder() + .uri("/nonexistent/path") + .header(header::AUTHORIZATION, "Bearer test-key") + .body(Body::empty()) + .unwrap(), + Request::builder() + .uri("/nonexistent/path") + .header(API_KEY_HEADER, "wrong-key") + .body(Body::empty()) + .unwrap(), + Request::builder() + .uri("/nonexistent/path") + .header(API_KEY_HEADER, "test-key") + .header(API_KEY_HEADER, "test-key") + .body(Body::empty()) + .unwrap(), + ] { + let response = app.clone().oneshot(request).await.unwrap(); + assert_eq!(response.status(), StatusCode::UNAUTHORIZED); + } + + let response = app + .clone() + .oneshot( + Request::builder() + .uri("/nonexistent/path") + .header(API_KEY_HEADER, "test-key") + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::NOT_FOUND); + + let response = app + .clone() + .oneshot( + Request::builder() + .uri("/health") + .header(header::HOST, "localhost") + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::NO_CONTENT); + + let response = app + .oneshot( + Request::builder() + .uri("/health") + .header(header::HOST, "localhost") + .header(SANDBOX_ID_HEADER, SandboxId::new().to_string()) + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::NO_CONTENT); + } + + #[tokio::test] + async fn traffic_token_cannot_authenticate_control_plane() { + let api = build_api().await; + let sandbox_id = SandboxId::new(); + let traffic_token = api.traffic_access_token(&sandbox_id.to_string()); + let app = server::new(api); + let response = app + .oneshot( + Request::builder() + .method(Method::POST) + .uri(format!("/sandboxes/{sandbox_id}/pause")) + .header(TRAFFIC_ACCESS_TOKEN_HEADER, traffic_token) + .header(SANDBOX_ID_HEADER, sandbox_id.to_string()) + .header(TARGET_PORT_HEADER, "80") + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + + assert_eq!(response.status(), StatusCode::UNAUTHORIZED); + } + + #[tokio::test] + async fn envd_token_cannot_authenticate_application_proxy() { + let api = build_api().await; + let sandbox_id = SandboxId::new(); + let traffic_token = api.traffic_access_token(&sandbox_id.to_string()); + let response = server::new(api) + .oneshot( + Request::builder() + .uri("/proxy/hello") + .header(ENVD_ACCESS_TOKEN_HEADER, traffic_token) + .header(SANDBOX_ID_HEADER, sandbox_id.to_string()) + .header(TARGET_PORT_HEADER, "80") + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + + assert_eq!(response.status(), StatusCode::UNAUTHORIZED); + } + #[tokio::test] async fn proxy_requires_routing_headers() { let app = server::new(build_api().await); @@ -2046,9 +2306,11 @@ mod tests { .uri("/proxy/echo/test?foo=bar".to_string()) .header("host", "client.example") .header("x-api-key", "test-key") + .header(header::AUTHORIZATION, "Bearer application-token") .header(SANDBOX_ID_HEADER, sandbox_id.to_string()) .header(TARGET_PORT_HEADER, upstream_addr.port().to_string()) .header(ENVD_ACCESS_TOKEN_HEADER, "envd-token") + .header(TRAFFIC_ACCESS_TOKEN_HEADER, "traffic-token") .body(Body::empty()) .unwrap(), ) @@ -2065,8 +2327,11 @@ mod tests { assert_eq!(payload["e2b_sandbox_header_seen"], false); assert_eq!(payload["target_port_header_seen"], false); assert_eq!(payload["e2b_target_port_header_seen"], false); - assert_eq!(payload["envd_access_token"], "envd-token"); + assert!(payload["access_token"].is_null()); + assert_eq!(payload["traffic_token_header_seen"], false); assert_eq!(payload["forwarded_host"], "client.example"); + assert_eq!(payload["api_key_header_seen"], false); + assert_eq!(payload["authorization"], "Bearer application-token"); } #[tokio::test] @@ -2288,18 +2553,40 @@ mod tests { async fn sandbox_proxy_host_routes_control_paths_and_skips_explicit_proxy() { let upstream_addr = start_upstream_server().await; let sandbox_id = SandboxId::new(); - let app = proxy_app_for_sandbox_with_domains( + let (app, access_token) = proxy_app_for_sandbox_with_domains( &sandbox_id, vec!["sandbox.example.invalid".to_string()], ) .await; + let response = app + .clone() + .oneshot( + Request::builder() + .method(Method::GET) + .uri("/health") + .header( + "host", + format!( + "{}-{}.sandbox.example.invalid", + upstream_addr.port(), + sandbox_id + ), + ) + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::UNAUTHORIZED); + let response = app .clone() .oneshot( Request::builder() .method(Method::GET) .uri("/health?foo=bar") + .header(TRAFFIC_ACCESS_TOKEN_HEADER, access_token) .header( "host", format!( @@ -2332,6 +2619,7 @@ mod tests { upstream_addr.port(), sandbox_id )) + .header("x-api-key", "test-key") .body(Body::empty()) .unwrap(), ) @@ -2343,7 +2631,7 @@ mod tests { let payload: Value = serde_json::from_slice(&body).unwrap(); assert_eq!(payload["path"], "/authority"); - let app = proxy_app_for_sandbox_with_domains( + let (app, _) = proxy_app_for_sandbox_with_domains( &sandbox_id, vec!["sandbox.example.invalid".to_string()], ) @@ -2354,6 +2642,7 @@ mod tests { Request::builder() .method(Method::GET) .uri("/proxy/health") + .header("x-api-key", "test-key") .header( "host", format!( @@ -2375,6 +2664,7 @@ mod tests { Request::builder() .method(Method::GET) .uri("/proxy/health") + .header("x-api-key", "test-key") .header( "host", format!( @@ -2401,14 +2691,14 @@ mod tests { async fn proxy_accepts_e2b_compatible_headers() { let upstream_addr = start_upstream_server().await; let sandbox_id = SandboxId::new(); - let app = proxy_app_for_sandbox(&sandbox_id).await; + let (app, access_token) = proxy_app_with_access_token_for_sandbox(&sandbox_id).await; let response = app .oneshot( Request::builder() .method(Method::GET) .uri("/proxy/e2b/health") - .header("x-api-key", "test-key") + .header(TRAFFIC_ACCESS_TOKEN_HEADER, access_token) .header(E2B_SANDBOX_ID_HEADER, sandbox_id.to_string()) .header(E2B_TARGET_PORT_HEADER, upstream_addr.port().to_string()) .body(Body::empty()) @@ -2426,6 +2716,10 @@ mod tests { assert_eq!(payload["e2b_sandbox_header_seen"], false); assert_eq!(payload["target_port_header_seen"], false); assert_eq!(payload["e2b_target_port_header_seen"], false); + assert!( + payload["access_token"].is_null(), + "envd credential leaked to application port" + ); } #[tokio::test] diff --git a/src/api/server.rs b/src/api/server.rs index 2a7388984..0aed4b7cf 100644 --- a/src/api/server.rs +++ b/src/api/server.rs @@ -1,6 +1,6 @@ use axum::{middleware, routing::get, Router}; -use super::{proxy, ApiImpl}; +use super::{impls::auth, proxy, ApiImpl}; use crate::observability::prometheus; use agentenv_http_server::apis; use agentenv_observability::metrics_handler; @@ -28,8 +28,12 @@ where .merge(proxy::router(api_impl.clone())) .route("/metrics", get(metrics_handler)) .layer(middleware::from_fn_with_state( - api_impl, + api_impl.clone(), proxy::sandbox_proxy_classifier::, )) + .layer(middleware::from_fn_with_state( + api_impl, + auth::require_auth::, + )) .layer(middleware::from_fn(prometheus::http_metrics_middleware)) } diff --git a/src/api_key.rs b/src/api_key.rs new file mode 100644 index 000000000..ec59037e5 --- /dev/null +++ b/src/api_key.rs @@ -0,0 +1,195 @@ +use std::ffi::OsStr; +use std::fs::{self, File}; +use std::io::{self, Write}; +use std::path::Path; + +use anyhow::{bail, Context, Result}; +use rand::{rngs::SysRng, TryRng}; +use tracing::info; + +use crate::cfg::AppConfig; + +const API_KEY_ENV: &str = "AENV_API_KEY"; +const EXTERNAL_API_KEY_PATH: &str = "/run/secrets/api-key"; +const MANAGED_API_KEY_RELATIVE_PATH: &str = "secrets/api-key"; +const GENERATED_API_KEY_PREFIX: &str = "e2b_"; + +pub fn resolve(config: &AppConfig) -> Result { + resolve_from( + std::env::var_os(API_KEY_ENV).as_deref(), + Path::new(EXTERNAL_API_KEY_PATH), + &config.home_path, + ) +} + +fn resolve_from( + explicit: Option<&OsStr>, + external_path: &Path, + home_path: &Path, +) -> Result { + if let Some(explicit) = explicit { + return validate( + explicit + .to_str() + .context("AENV_API_KEY must contain valid UTF-8")?, + ) + .context("invalid AENV_API_KEY"); + } + + match read(external_path) { + Ok(key) => { + info!(path = %external_path.display(), "loaded API key from external secret"); + return Ok(key); + } + Err(error) if error.kind() == io::ErrorKind::NotFound => {} + Err(error) => return Err(error).context("load external API key"), + } + + let managed_path = home_path.join(MANAGED_API_KEY_RELATIVE_PATH); + match read(&managed_path) { + Ok(key) => return Ok(key), + Err(error) if error.kind() == io::ErrorKind::NotFound => {} + Err(error) => return Err(error).context("load managed API key"), + } + + create(&managed_path) +} + +fn read(path: &Path) -> Result { + let value = fs::read_to_string(path)?; + validate(&value).map_err(io::Error::other) +} + +fn validate(value: &str) -> Result { + let value = value.trim(); + if value.len() < 32 + || !value + .bytes() + .all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'.' | b'_' | b'~' | b'-')) + { + bail!("API key must contain at least 32 URL-safe characters"); + } + Ok(value.to_owned()) +} + +fn create(path: &Path) -> Result { + let parent = path + .parent() + .context("managed API key path has no parent")?; + fs::create_dir_all(parent) + .with_context(|| format!("create managed secret directory {}", parent.display()))?; + set_permissions(parent, 0o700)?; + + let mut random = [0_u8; 32]; + SysRng + .try_fill_bytes(&mut random) + .context("generate managed API key")?; + let key = format!("{GENERATED_API_KEY_PREFIX}{}", hex::encode(random)); + + let mut temporary = tempfile::NamedTempFile::new_in(parent) + .with_context(|| format!("create temporary API key in {}", parent.display()))?; + set_permissions(temporary.path(), 0o600)?; + writeln!(temporary, "{key}")?; + temporary.as_file().sync_all()?; + + match temporary.persist_noclobber(path) { + Ok(_) => { + File::open(parent)?.sync_all()?; + info!(path = %path.display(), "generated managed API key"); + Ok(key) + } + Err(error) if error.error.kind() == io::ErrorKind::AlreadyExists => { + read(path).context("load concurrently generated API key") + } + Err(error) => { + Err(error.error).with_context(|| format!("persist managed API key {}", path.display())) + } + } +} + +#[cfg(unix)] +fn set_permissions(path: &Path, mode: u32) -> Result<()> { + use std::os::unix::fs::PermissionsExt; + + fs::set_permissions(path, fs::Permissions::from_mode(mode)) + .with_context(|| format!("set permissions on {}", path.display())) +} + +#[cfg(not(unix))] +fn set_permissions(_path: &Path, _mode: u32) -> Result<()> { + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + use std::sync::{Arc, Barrier}; + + use tempfile::TempDir; + + const TEST_KEY: &str = "e2b_0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"; + + #[test] + fn configured_sources_take_precedence() -> Result<()> { + let temp = TempDir::new()?; + let external_path = temp.path().join("external"); + fs::write(&external_path, format!("{TEST_KEY}\n"))?; + + assert_eq!( + resolve_from(Some(OsStr::new(TEST_KEY)), &external_path, temp.path())?, + TEST_KEY + ); + assert_eq!(resolve_from(None, &external_path, temp.path())?, TEST_KEY); + assert!(!temp.path().join(MANAGED_API_KEY_RELATIVE_PATH).exists()); + Ok(()) + } + + #[test] + fn managed_key_is_private_and_stable() -> Result<()> { + let temp = TempDir::new()?; + let missing_external = temp.path().join("missing"); + let first = resolve_from(None, &missing_external, temp.path())?; + + assert_eq!(resolve_from(None, &missing_external, temp.path())?, first); + assert!(first.starts_with(GENERATED_API_KEY_PREFIX)); + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + + let path = temp.path().join(MANAGED_API_KEY_RELATIVE_PATH); + assert_eq!( + fs::metadata(path.parent().unwrap())?.permissions().mode() & 0o777, + 0o700 + ); + assert_eq!(fs::metadata(path)?.permissions().mode() & 0o777, 0o600); + } + Ok(()) + } + + #[test] + fn concurrent_creation_converges() -> Result<()> { + const THREADS: usize = 8; + let temp = TempDir::new()?; + let home_path = Arc::new(temp.path().to_owned()); + let external_path = Arc::new(temp.path().join("missing")); + let barrier = Arc::new(Barrier::new(THREADS)); + let handles = (0..THREADS) + .map(|_| { + let home_path = Arc::clone(&home_path); + let external_path = Arc::clone(&external_path); + let barrier = Arc::clone(&barrier); + std::thread::spawn(move || { + barrier.wait(); + resolve_from(None, &external_path, &home_path) + }) + }) + .collect::>(); + + let keys = handles + .into_iter() + .map(|handle| handle.join().expect("API key creation thread panicked")) + .collect::>>()?; + assert!(keys.iter().all(|key| key == &keys[0])); + Ok(()) + } +} diff --git a/src/bin/server.rs b/src/bin/server.rs index 479d038c2..f755bad90 100644 --- a/src/bin/server.rs +++ b/src/bin/server.rs @@ -76,6 +76,8 @@ async fn main() -> anyhow::Result<()> { return Ok(()); } + let api_key = agentenv::api_key::resolve(config)?; + agentenv::privileges::require_runtime_capabilities()?; agentenv::privileges::clear_ambient_capabilities()?; @@ -147,6 +149,7 @@ async fn main() -> anyhow::Result<()> { image_resolver, observability, config.sandbox_proxy.domains.clone(), + api_key, )); let app = server::new(api_impl); let shutdown_orchestrator = Arc::clone(&orchestrator); diff --git a/src/lib.rs b/src/lib.rs index 5968c26e1..14d02f7e8 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,4 +1,5 @@ pub mod api; +pub mod api_key; pub mod cfg; mod digest; pub mod identity; diff --git a/src/sandbox/backend.rs b/src/sandbox/backend.rs index 267029178..575e3a62a 100644 --- a/src/sandbox/backend.rs +++ b/src/sandbox/backend.rs @@ -35,6 +35,10 @@ pub trait PausedSandboxState: Any + fmt::Debug + Send + Sync + 'static { /// The orchestrator only carries this value to the image-liveness layer; it /// does not interpret the backend-specific artifact identities inside it. fn runtime_artifacts(&self) -> RuntimeArtifactSet; + /// Effective envd control-plane port persisted with the paused runtime, when available. + fn control_plane_port(&self) -> Option { + None + } } impl dyn PausedSandboxState { diff --git a/src/sandbox/firecracker/sandbox.rs b/src/sandbox/firecracker/sandbox.rs index a3a7c9035..a4b943cdd 100644 --- a/src/sandbox/firecracker/sandbox.rs +++ b/src/sandbox/firecracker/sandbox.rs @@ -229,6 +229,10 @@ impl FirecrackerPausedState { } impl PausedSandboxState for FirecrackerPausedState { + fn control_plane_port(&self) -> Option { + Some(self.snapshot_config.common.control_plane_port) + } + fn encode(&self) -> Result { serde_json::to_value(&self.snapshot_config).context("serialize Firecracker paused state") } From b28eba163cfdfb6af3b41dbc811ec3ec2ba74ecd Mon Sep 17 00:00:00 2001 From: Yingdi Shan <5491399+yingdi-shan@users.noreply.github.com> Date: Sat, 15 Aug 2026 12:29:56 +0000 Subject: [PATCH 2/7] docs: document API key setup in README --- README.md | 23 ++++++++++++++++++----- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 4c649223e..0c1627460 100644 --- a/README.md +++ b/README.md @@ -44,9 +44,9 @@ If your server does not support standard KVM, see the [PVM deployment guide](htt ## ⚡ Quick Start (Single Node) > [!WARNING] -> **AgentENV currently does not support authorization.** Do not expose the AgentENV -> API to the public network. Run it only on a trusted network or behind an -> authorization proxy with appropriate network controls. +> AgentENV authenticates API requests but does not encrypt traffic. Do not send +> the API key over an untrusted plaintext network. Run AgentENV on a trusted +> network or terminate HTTPS at a reverse proxy or load balancer. **1. Install and start the server** @@ -66,7 +66,7 @@ Set up the server: ```bash curl -fsSL https://raw.githubusercontent.com/kvcache-ai/AgentENV/main/scripts/docker-setup.sh | sudo bash docker pull ghcr.io/kvcache-ai/aenv-server:latest -docker run -d --privileged -v /dev:/dev -p 8000:8000 ghcr.io/kvcache-ai/aenv-server:latest +docker run -d --name aenv-server --privileged -v /dev:/dev -p 8000:8000 ghcr.io/kvcache-ai/aenv-server:latest ``` The server is accessible at `http://127.0.0.1:8000` by default. @@ -83,10 +83,23 @@ curl -fsSL https://raw.githubusercontent.com/kvcache-ai/AgentENV/main/scripts/in **3. Authenticate** +The server generates an API key on its first startup. Retrieve it for the +installation method used in step 1: + +```bash +# Native install +sudo cat /var/lib/aenv/secrets/api-key + +# Docker +docker exec aenv-server cat /workspace/env/secrets/api-key +``` + +Then run `aenv auth` and paste that key: + ```bash aenv auth # AENV server URL [http://localhost:8000]: http://127.0.0.1:8000 -# API key: dummy +# API key: ``` **4. Pull a template and run a sandbox** From bc0ad47a6e6be641e05350c60fba01b30489255a Mon Sep 17 00:00:00 2001 From: Yingdi Shan <5491399+yingdi-shan@users.noreply.github.com> Date: Sat, 15 Aug 2026 13:40:00 +0000 Subject: [PATCH 3/7] fix: derive sandbox access tokens from shared seed --- config/default.toml | 2 +- deploy/docker-compose.yml | 2 +- deploy/k8s/base/agentenv-daemonset.yaml | 5 +- deploy/k8s/base/gateway-deployment.yaml | 5 + deploy/k8s/base/kustomization.yaml | 1 + .../k8s/overlays/local-dev/kustomization.yaml | 8 -- deploy/k8s/run.sh | 67 +++++++--- docs/src/configuration/authentication.md | 27 ++-- docs/src/configuration/env-vars.md | 2 +- docs/src/configuration/reference.md | 4 +- docs/src/deployment/docker-compose.md | 17 +-- docs/src/deployment/kubernetes.md | 17 ++- docs/src/deployment/static-multi-node.md | 17 +-- .../persistence-artifact-inventory.md | 4 +- docs/src/security/secure-sandboxes.md | 30 ++--- services/README.md | 6 +- services/gateway/cmd/main.go | 46 ++++++- services/gateway/cmd/main_test.go | 27 ++++ services/gateway/internal/server.go | 23 ++-- services/gateway/internal/server_test.go | 35 +++-- src/api/impls/auth.rs | 49 +++---- src/api/impls/sandbox.rs | 5 +- src/api/proxy.rs | 19 ++- src/orchestrator/service.rs | 12 +- src/sandbox/access.rs | 123 +++++++++++++----- 25 files changed, 349 insertions(+), 204 deletions(-) diff --git a/config/default.toml b/config/default.toml index 00f94bf72..2ec626f9a 100644 --- a/config/default.toml +++ b/config/default.toml @@ -188,7 +188,7 @@ init_timeout_secs = 60 poll_ms = 3 [sandbox] -# Optional secret used to derive per-sandbox envd access tokens. When unset, +# Optional secret used to derive per-sandbox envd and traffic access tokens. When unset, # AgentENV creates a node-local seed under $AENV_HOME/secrets. Configure the # same explicit value on every node when cross-node sandbox recovery is required. # access_token_hash_seed = "replace-with-a-secret" diff --git a/deploy/docker-compose.yml b/deploy/docker-compose.yml index 4a6bfb2da..8f60d7bef 100644 --- a/deploy/docker-compose.yml +++ b/deploy/docker-compose.yml @@ -16,7 +16,7 @@ x-agentenv-base: &agentenv-base - /dev:/dev - ${CONFIG_PATH:-../config/default.toml}:/workspace/config/default.toml:ro # Runtime assets are baked into the image by `server --setup-only`; compose - # persists committed snapshots and the deployment API key across restarts. + # persists committed snapshots and deployment secrets across restarts. - agentenv-snapshot-store:/workspace/env/snapshot-store - agentenv-auth:/workspace/env/secrets devices: diff --git a/deploy/k8s/base/agentenv-daemonset.yaml b/deploy/k8s/base/agentenv-daemonset.yaml index 470b2a9cb..e95fefb96 100644 --- a/deploy/k8s/base/agentenv-daemonset.yaml +++ b/deploy/k8s/base/agentenv-daemonset.yaml @@ -31,9 +31,8 @@ spec: - name: AENV_SANDBOX_ACCESS_TOKEN_HASH_SEED valueFrom: secretKeyRef: - name: agentenv-runtime-secrets - key: sandbox-access-token-hash-seed - optional: true + name: agentenv-auth + key: AENV_SANDBOX_ACCESS_TOKEN_HASH_SEED - name: AENV_VIRTUALIZATION_MODE value: "kvm" - name: API_ADDR diff --git a/deploy/k8s/base/gateway-deployment.yaml b/deploy/k8s/base/gateway-deployment.yaml index 08536fc0e..7f7f8644f 100644 --- a/deploy/k8s/base/gateway-deployment.yaml +++ b/deploy/k8s/base/gateway-deployment.yaml @@ -32,6 +32,11 @@ spec: secretKeyRef: name: agentenv-auth key: AENV_API_KEY + - name: AENV_SANDBOX_ACCESS_TOKEN_HASH_SEED + valueFrom: + secretKeyRef: + name: agentenv-auth + key: AENV_SANDBOX_ACCESS_TOKEN_HASH_SEED - name: GATEWAY_SANDBOX_PROXY_DOMAINS valueFrom: configMapKeyRef: diff --git a/deploy/k8s/base/kustomization.yaml b/deploy/k8s/base/kustomization.yaml index 556f0d700..157d8d626 100644 --- a/deploy/k8s/base/kustomization.yaml +++ b/deploy/k8s/base/kustomization.yaml @@ -34,6 +34,7 @@ secretGenerator: - name: agentenv-auth literals: - AENV_API_KEY= + - AENV_SANDBOX_ACCESS_TOKEN_HASH_SEED= images: - name: agentenv-gateway diff --git a/deploy/k8s/overlays/local-dev/kustomization.yaml b/deploy/k8s/overlays/local-dev/kustomization.yaml index 2a71de5e1..ab182ed57 100644 --- a/deploy/k8s/overlays/local-dev/kustomization.yaml +++ b/deploy/k8s/overlays/local-dev/kustomization.yaml @@ -6,14 +6,6 @@ namespace: agentenv-system resources: - ../../base -secretGenerator: - - name: agentenv-runtime-secrets - literals: - - sandbox-access-token-hash-seed=agentenv-local-dev-access-token-hash-seed - -generatorOptions: - disableNameSuffixHash: true - patches: - target: kind: DaemonSet diff --git a/deploy/k8s/run.sh b/deploy/k8s/run.sh index 4d39680ba..49f6aec92 100644 --- a/deploy/k8s/run.sh +++ b/deploy/k8s/run.sh @@ -29,29 +29,59 @@ sed_in_place() { cp -R "${SCRIPT_DIR}" "${TEMP_DIR}/k8s" cp "${REPO_ROOT}/config/default.toml" "${TEMP_DIR}/k8s/base/config/agentenv.toml" + +namespace_name="" +if [[ "${MODE}" == "apply" ]]; then + if ! namespace_name="$("${KUBECTL_BIN}" get namespace "${NAMESPACE}" --ignore-not-found -o name)"; then + echo "failed to check namespace ${NAMESPACE}" >&2 + exit 1 + fi +fi + +read_existing_secret() { + local secret="$1" + local key="$2" + local encoded_value="" + + if [[ -z "${namespace_name}" ]]; then + return 0 + fi + if ! encoded_value="$("${KUBECTL_BIN}" -n "${NAMESPACE}" get secret "${secret}" \ + --ignore-not-found -o "go-template={{index .data \"${key}\"}}")"; then + echo "failed to read ${key} from Secret ${NAMESPACE}/${secret}" >&2 + return 1 + fi + if [[ -n "${encoded_value}" ]]; then + printf '%s' "${encoded_value}" | base64 -d + fi +} + if [[ "${MODE}" != "delete" ]]; then API_KEY_VALUE="" if [[ "${AENV_API_KEY+x}" == "x" ]]; then API_KEY_VALUE="${AENV_API_KEY}" - elif [[ "${MODE}" == "apply" ]]; then - encoded_key="" - if ! namespace_name="$("${KUBECTL_BIN}" get namespace "${NAMESPACE}" --ignore-not-found -o name)"; then - echo "failed to check namespace ${NAMESPACE}" >&2 + elif ! API_KEY_VALUE="$(read_existing_secret agentenv-auth AENV_API_KEY)"; then + exit 1 + fi + + ACCESS_TOKEN_SEED_VALUE="" + if [[ "${AENV_SANDBOX_ACCESS_TOKEN_HASH_SEED+x}" == "x" ]]; then + ACCESS_TOKEN_SEED_VALUE="${AENV_SANDBOX_ACCESS_TOKEN_HASH_SEED}" + elif ! ACCESS_TOKEN_SEED_VALUE="$(read_existing_secret agentenv-auth AENV_SANDBOX_ACCESS_TOKEN_HASH_SEED)"; then + exit 1 + fi + if [[ -z "${ACCESS_TOKEN_SEED_VALUE}" ]]; then + if ! ACCESS_TOKEN_SEED_VALUE="$(read_existing_secret agentenv-runtime-secrets sandbox-access-token-hash-seed)"; then exit 1 fi - if [[ -n "${namespace_name}" ]]; then - if ! encoded_key="$("${KUBECTL_BIN}" -n "${NAMESPACE}" get secret agentenv-auth \ - --ignore-not-found -o jsonpath='{.data.AENV_API_KEY}')"; then - echo "failed to read existing Secret ${NAMESPACE}/agentenv-auth" >&2 - exit 1 - fi - fi - if [[ -n "${encoded_key}" ]]; then - if ! API_KEY_VALUE="$(printf '%s' "${encoded_key}" | base64 -d)"; then - echo "failed to decode the existing agentenv-auth Secret" >&2 - exit 1 - fi - fi + fi + + if [[ -z "${ACCESS_TOKEN_SEED_VALUE}" ]]; then + ACCESS_TOKEN_SEED_VALUE="$(od -An -N32 -tx1 /dev/urandom | tr -d '[:space:]')" + fi + if [[ ! "${ACCESS_TOKEN_SEED_VALUE}" =~ ^[A-Za-z0-9._~-]{32,}$ ]]; then + echo "AENV_SANDBOX_ACCESS_TOKEN_HASH_SEED must contain at least 32 URL-safe characters" >&2 + exit 1 fi if [[ -z "${API_KEY_VALUE}" ]]; then @@ -65,6 +95,9 @@ if [[ "${MODE}" != "delete" ]]; then sed_in_place \ "s#- AENV_API_KEY=.*#- AENV_API_KEY=${API_KEY_VALUE}#" \ "${TEMP_DIR}/k8s/base/kustomization.yaml" + sed_in_place \ + "s#- AENV_SANDBOX_ACCESS_TOKEN_HASH_SEED=.*#- AENV_SANDBOX_ACCESS_TOKEN_HASH_SEED=${ACCESS_TOKEN_SEED_VALUE}#" \ + "${TEMP_DIR}/k8s/base/kustomization.yaml" fi if [[ "${SANDBOX_PROXY_DOMAINS+x}" == "x" ]]; then diff --git a/docs/src/configuration/authentication.md b/docs/src/configuration/authentication.md index b57dac1e9..c04773f79 100644 --- a/docs/src/configuration/authentication.md +++ b/docs/src/configuration/authentication.md @@ -23,6 +23,9 @@ For secure sandboxes, `envdAccessToken` is a separate credential for envd control traffic and must be sent as `X-Access-Token` only when targeting the envd control-plane port. It is absent for insecure sandboxes. +Both sandbox credentials are derived from the sandbox ID and one independent +`AENV_SANDBOX_ACCESS_TOKEN_HASH_SEED`. They are not derived from the API key. + ## Key Resolution On normal startup, a runtime node uses the first available source: @@ -36,8 +39,10 @@ generates a 256-bit key and atomically stores it in the managed path with `0600` permissions. It reuses that key on later starts. Dependency and host setup modes do not create a key. -The gateway uses `AENV_API_KEY` or `/run/secrets/api-key`; it never generates a -key because every gateway and runtime node in a cluster must share one. +The gateway uses `AENV_API_KEY` or `/run/secrets/api-key`. It also reads the +sandbox seed from `AENV_SANDBOX_ACCESS_TOKEN_HASH_SEED` or +`/run/secrets/sandbox-access-token-hash-seed`. The gateway never generates +either value because it must share them with every runtime node. ## Installation Methods @@ -64,7 +69,8 @@ container replacements. The checked-in Compose deployment mounts one named volume read-write on both runtime nodes and read-only at `/run/secrets` on the gateway. Concurrent node -startup is safe: atomic creation makes both nodes converge on the same key. +startup is safe: atomic creation makes both nodes converge on the same key and +sandbox seed. Read it with: ```bash @@ -75,8 +81,8 @@ docker compose -f deploy/docker-compose.yml exec -T agentenv-a \ `docker compose down` preserves the key. `docker compose down -v` removes the auth volume, so the next startup generates a new key. -`make k8s-apply` creates `Secret/agentenv-auth` on the first apply and reuses -the existing key on later applies. Read it with: +`make k8s-apply` creates `Secret/agentenv-auth` with an API key and sandbox +seed on the first apply, then reuses both values. Read the API key with: ```bash kubectl -n agentenv-system get secret agentenv-auth \ @@ -113,9 +119,8 @@ network, use a VPN, or terminate HTTPS at a reverse proxy or load balancer. ## Rotation -Set a new `AENV_API_KEY` on the gateway and every runtime node, or replace the -shared secret file, then restart them. Existing clients must switch to the new -value. Previously issued -`trafficAccessToken` values stop working when the key changes. -`envdAccessToken` values are unaffected and rotate only when the optional envd -seed changes. +Changing `AENV_API_KEY` invalidates existing client API credentials without +changing sandbox credentials. Changing +`AENV_SANDBOX_ACCESS_TOKEN_HASH_SEED` rotates both `trafficAccessToken` and +`envdAccessToken` values. Apply either change to the gateway and every runtime +node together, then restart them. diff --git a/docs/src/configuration/env-vars.md b/docs/src/configuration/env-vars.md index fc70ccaac..bb458564b 100644 --- a/docs/src/configuration/env-vars.md +++ b/docs/src/configuration/env-vars.md @@ -24,7 +24,7 @@ These variables are consumed by the repository's Docker Compose and Kubernetes h | `AENV_OBSERVABILITY_SCHEDULER_ENDPOINT` | unset | Override scheduler heartbeat reporting endpoint | | `AENV_OBSERVABILITY_REPORT_INTERVAL_SECS` | `5` | Override heartbeat reporting interval in seconds | | `AENV_CUSTOM_EXTENSION_URL` | unset | Override `[custom_extension].url`, the HTTP base URL of the custom extension service | -| `AENV_SANDBOX_ACCESS_TOKEN_HASH_SEED` | auto-generated under `$AENV_HOME/secrets` | Optional override for the secret used to derive secure sandbox envd access tokens. Configure the same value on every node when cross-node recovery of the same sandbox ID is required; otherwise each node uses its own managed seed. | +| `AENV_SANDBOX_ACCESS_TOKEN_HASH_SEED` | `/run/secrets/sandbox-access-token-hash-seed`, then auto-generated under `$AENV_HOME/secrets` | Optional runtime override for the secret used to derive sandbox envd and traffic access tokens. Clustered deployments must configure the same value on the gateway and every runtime node. | | `AENV_SANDBOX_PROXY_DOMAINS` | from config | Comma-separated DNS domains that enable server-side host-based sandbox proxy URLs like `{port}-{sandboxID}.{domain}` and populate the sandbox response `domain` field. Empty or unset keeps `[sandbox_proxy].domains`. | | `AENV_HOME_PATH` | `/var/lib/aenv` | Override the base directory from which AgentENV derives local state, caches, logs, generated configs, and downloaded dependencies. Component-specific path settings remain available as advanced overrides. | | `AENV_RUNTIME_PATH` | `/run/aenv` | Override the transient runtime directory used for network namespace mount points and the default ublk daemon socket. | diff --git a/docs/src/configuration/reference.md b/docs/src/configuration/reference.md index d2ad80496..a81591d62 100644 --- a/docs/src/configuration/reference.md +++ b/docs/src/configuration/reference.md @@ -257,11 +257,11 @@ Sandbox control communication settings. | Key | Type | Default | Description | |-----|------|---------|-------------| -| `access_token_hash_seed` | string | auto-generated | Optional override for the secret used to derive secure sandbox envd access tokens. When unset, normal server startup creates and reuses `$AENV_HOME/secrets/sandbox-access-token-hash-seed`. Configure an explicit shared value when the deployment needs to recover the same sandbox ID on another node. | +| `access_token_hash_seed` | string | auto-generated | Optional override for the secret used to derive sandbox envd and traffic access tokens. When unset, normal server startup creates and reuses `$AENV_HOME/secrets/sandbox-access-token-hash-seed`. Configure an explicit shared value for clustered deployments. | The managed seed is node-local persistent state and must be included in backups of `$AENV_HOME`. AgentENV refuses to generate a replacement when persisted secure sandboxes exist. An explicit environment or TOML value takes precedence over the managed file; changing that effective value invalidates access tokens for existing secure sandboxes. -Configure `AENV_SANDBOX_ACCESS_TOKEN_HASH_SEED` with the same value on every node when cross-node recovery of the same sandbox is required. Nodes use their own managed seed when it is unset. +Configure `AENV_SANDBOX_ACCESS_TOKEN_HASH_SEED` with the same value on the gateway and every runtime node in a clustered deployment. Standalone runtime nodes use their managed seed when it is unset. ## `[orchestrator]` diff --git a/docs/src/deployment/docker-compose.md b/docs/src/deployment/docker-compose.md index 1417c4d91..f9edc7e03 100644 --- a/docs/src/deployment/docker-compose.md +++ b/docs/src/deployment/docker-compose.md @@ -23,11 +23,6 @@ git clone https://github.com/kvcache-ai/AgentENV.git cd AgentENV ``` -## Configure the Access-Token Seed (Optional) - -See [Secure Sandboxes](../security/secure-sandboxes.md) -if the deployment needs future cross-node sandbox recovery. - ## Start the Cluster ```bash @@ -38,10 +33,11 @@ make deploy-up The Gateway is available at `http://127.0.0.1:8000` and forwards requests to the backend nodes. -On first startup, the runtime nodes atomically generate one API key in the +On first startup, the runtime nodes atomically generate one API key and sandbox +access-token seed in the shared `agentenv-auth` volume. The gateway mounts that volume read-only at -`/run/secrets`, so all three services use the same key. Normal -`make deploy-down` calls preserve the volume and key. +`/run/secrets`, so all three services use the same secrets. Normal +`make deploy-down` calls preserve the volume and both values. To enable host-based sandbox data-plane URLs, set the shared sandbox proxy domain variable when starting the stack: @@ -81,8 +77,9 @@ make deploy-logs # Stream logs from all services make deploy-down # Tear down the cluster ``` -Removing Compose volumes with `docker compose down -v` also removes the API -key. The next startup generates a new key and existing clients must be updated. +Removing Compose volumes with `docker compose down -v` also removes both +secrets. The next startup generates new values, so existing clients and sandbox +access tokens are invalidated. To provide an existing key through Docker Compose secrets, add a file-backed secret in an override file and mount it with `target: api-key` on the gateway diff --git a/docs/src/deployment/kubernetes.md b/docs/src/deployment/kubernetes.md index 412a63538..463359f79 100644 --- a/docs/src/deployment/kubernetes.md +++ b/docs/src/deployment/kubernetes.md @@ -61,18 +61,18 @@ make k8s-render make k8s-apply ``` -`make k8s-apply` generates a 256-bit API key on the first deployment and -stores it in `Secret/agentenv-auth`. Later applies reuse that key. Read it -locally when configuring clients: +`make k8s-apply` generates a 256-bit API key and sandbox access-token seed on +the first deployment and stores both in `Secret/agentenv-auth`. Later applies +reuse both values. Read the API key locally when configuring clients: ```bash kubectl -n agentenv-system get secret agentenv-auth \ -o go-template='{{index .data "AENV_API_KEY" | base64decode}}{{"\n"}}' ``` -Set `AENV_API_KEY` when applying to supply your own key instead. A standalone -`make k8s-render` uses a temporary generated value because it does not modify -or read cluster state. +Set `AENV_API_KEY` and `AENV_SANDBOX_ACCESS_TOKEN_HASH_SEED` when applying to +supply your own values. A standalone `make k8s-render` uses temporary generated +values because it does not modify or read cluster state. To enable host-based sandbox data-plane URLs, set the shared sandbox proxy domain variable when rendering or applying manifests: @@ -114,9 +114,8 @@ make k8s-delete A dedicated `local-dev` overlay mounts the repository's `env/` directory directly into the DaemonSet at `/workspace/env`, avoiding runtime asset copies: -This overlay also generates `agentenv-runtime-secrets` with a fixed test-only -seed so local and E2E deployments do not require production secret management. -Do not reuse that value outside local development. +The apply helper provisions the same generated `agentenv-auth` secrets used by +the default overlay. ```bash make k8s-build diff --git a/docs/src/deployment/static-multi-node.md b/docs/src/deployment/static-multi-node.md index a67abc97f..e71f9818f 100644 --- a/docs/src/deployment/static-multi-node.md +++ b/docs/src/deployment/static-multi-node.md @@ -55,11 +55,13 @@ an external metrics collector needs them. ## 1. Install the runtime nodes -Generate one API key, deliver it through your normal secret-management channel, -and use the same value on every runtime node and the Gateway: +Generate one API key and one sandbox access-token seed, deliver them through +your normal secret-management channel, and use the same values on every runtime +node and the Gateway: ```bash export AENV_API_KEY="e2b_$(openssl rand -hex 32)" +export AENV_SANDBOX_ACCESS_TOKEN_HASH_SEED="$(openssl rand -hex 32)" ``` Run the installation on each runtime node: @@ -69,11 +71,9 @@ curl -fsSL https://raw.githubusercontent.com/kvcache-ai/AgentENV/main/scripts/in ``` Edit `/etc/default/aenv` on each machine without removing the paths written by -the installer, and add `AENV_API_KEY=` before starting the -services. A multi-node deployment must not let each node generate an -independent managed key. - -See [Secure Sandboxes](../security/secure-sandboxes.md) if the deployment needs future cross-node sandbox recovery. +the installer, and add both shared values before starting the services. A +multi-node deployment must not let each node generate independent managed +secrets. Node A uses: @@ -117,7 +117,7 @@ sudo useradd --system --no-create-home --shell /usr/sbin/nologin agentenv-contro sudo install -d -o root -g agentenv-control -m 0750 /etc/agentenv ``` -Create `/etc/agentenv/auth.env` with the same key used on the runtime nodes: +Create `/etc/agentenv/auth.env` with the same values used on the runtime nodes: ```bash sudo install -o root -g agentenv-control -m 0640 /dev/null /etc/agentenv/auth.env @@ -126,6 +126,7 @@ sudoedit /etc/agentenv/auth.env ```text AENV_API_KEY= +AENV_SANDBOX_ACCESS_TOKEN_HASH_SEED= ``` If the `agentenv-control` account already exists, the `useradd` command reports diff --git a/docs/src/internals/persistence-artifact-inventory.md b/docs/src/internals/persistence-artifact-inventory.md index eefdb8825..8cf259048 100644 --- a/docs/src/internals/persistence-artifact-inventory.md +++ b/docs/src/internals/persistence-artifact-inventory.md @@ -9,7 +9,7 @@ This document lists AgentENV artifacts that can remain on disk or in object stor | `home_path` | `/var/lib/aenv` | `src/cfg.rs` | Base for paths containing the literal `$AENV_HOME` placeholder. `AENV_HOME_PATH` overrides it before placeholder expansion. | | `runtime_path` | `/run/aenv` | `src/cfg.rs`, `src/sandbox/network/*` | Base for transient namespace mount points and daemon sockets. `AENV_RUNTIME_PATH` overrides it. | | `deps_path` | `$AENV_HOME/deps` | `src/cfg.rs`, `src/setup/*` | Base for downloaded runtime dependencies. `AENV_DEPS_PATH` can place these rebuildable assets outside `home_path`. | -| Managed envd access-token seed | `$AENV_HOME/secrets/sandbox-access-token-hash-seed` | `src/sandbox/access.rs` | Node-local secret used when `[sandbox].access_token_hash_seed` is unset. It must be preserved with persisted secure sandboxes. | +| Managed sandbox access-token seed | `$AENV_HOME/secrets/sandbox-access-token-hash-seed` | `src/sandbox/access.rs` | Node-local secret used to derive envd and traffic tokens when `[sandbox].access_token_hash_seed` is unset. It must be preserved with persisted sandboxes. | | Firecracker sandbox work dirs | `$AENV_HOME/firecracker-work` with `agentenv-fc-` children | `src/sandbox/firecracker/*` | Per-sandbox runtime directories for sockets, symlinks, ublk runtime dirs, local logs, and writable OverlayBD upper layer data (`overlaybd/upper.data`, `overlaybd/upper.index`). An explicit `[firecracker].work_dir` overrides the root. | | `firecracker.serial_dir` | `$AENV_HOME/logs/serial` | `src/sandbox/firecracker/*` | Durable Firecracker stdout/stderr root, grouped by sandbox ID. An explicit `[firecracker].serial_dir` overrides the root. | | `managed_snapshot_root` | `/managed-snapshots` | `src/sandbox/firecracker/*` | In-process live snapshot artifact root used to keep captured snapshots alive until publish or drop. | @@ -37,7 +37,7 @@ Owned by `src/setup/*` and `src/cfg.rs`. | Overlaybd package downloads | `/overlaybd/downloads/*` | Temporary downloaded package archives | Setup staging for overlaybd release packages | Removed after a successful install. | | Generated overlaybd config | `$AENV_HOME/overlaybd/overlaybd-global.json`, `$AENV_HOME/overlaybd/mem-overlaybd-global.json`, `$AENV_HOME/overlaybd/convert-overlaybd-global.json`, `$AENV_HOME/overlaybd/resize-overlaybd-global.json` | Runtime global config, cache path, credentials config | Configures overlaybd runtime, memory snapshot overlaybd access, and the offline C++ tools (`overlaybd-apply`, `overlaybd-resize`), which get dedicated configs with isolated cacheDirs (`convert-blocks`, `resize-blocks`) and download disabled | Rewritten during setup/startup. | | Overlaybd runtime log | `$AENV_HOME/overlaybd/overlaybd.log` | Overlaybd runtime logs | Debugging | Appended by overlaybd runtime; no automatic GC. | -| Managed envd access-token seed | `$AENV_HOME/secrets/sandbox-access-token-hash-seed` | 32 random bytes encoded as lowercase hexadecimal | Derives stable per-sandbox envd access tokens when no explicit seed is configured | Atomically created with mode `0600` during normal startup and reused thereafter. Must not be deleted while secure sandboxes are persisted. | +| Managed sandbox access-token seed | `$AENV_HOME/secrets/sandbox-access-token-hash-seed` | 32 random bytes encoded as lowercase hexadecimal | Derives stable per-sandbox envd and traffic access tokens when no explicit seed is configured | Atomically created with mode `0600` during normal startup and reused thereafter. Must not be deleted while sandboxes are persisted. | ## Firecracker Sandbox diff --git a/docs/src/security/secure-sandboxes.md b/docs/src/security/secure-sandboxes.md index b04948df3..2eadf4b65 100644 --- a/docs/src/security/secure-sandboxes.md +++ b/docs/src/security/secure-sandboxes.md @@ -17,44 +17,34 @@ The API and SDKs return the sandbox's `envdAccessToken` where appropriate and at ## Access-Token Seed -A seed is a random value used to derive the access token for each sandbox. This seed is optional. When it is unset, each runtime node automatically creates and persists a node-local seed under `$AENV_HOME/secrets`. +A seed is a random value used to derive each sandbox's envd and traffic access tokens. This seed is optional for a standalone runtime. When it is unset, the runtime automatically creates and persists a seed under `$AENV_HOME/secrets`. This is sufficient for normal single-node operation and does not require additional setup. -Configure the same explicit seed on every runtime node when the deployment needs to recover the same sandbox ID on another node in the future. Generate it once and store it in the deployment's secret manager: +Configure the same explicit seed on the gateway and every runtime node in a clustered deployment. Generate it once and store it in the deployment's secret manager: ```bash openssl rand -hex 32 ``` -Set the value as `AENV_SANDBOX_ACCESS_TOKEN_HASH_SEED` on every runtime node. +Set the value as `AENV_SANDBOX_ACCESS_TOKEN_HASH_SEED` on the gateway and every runtime node. For TOML configuration, use `[sandbox].access_token_hash_seed` instead. +Container deployments may mount it at +`/run/secrets/sandbox-access-token-hash-seed`. -Preserve the seed across upgrades; changing it rotates access tokens for existing secure sandboxes. +Preserve the seed across upgrades; changing it rotates both sandbox access tokens. ### Kubernetes -The runtime DaemonSet reads the optional `agentenv-runtime-secrets` Secret. To configure a shared seed for all runtime Pods, create it before applying the runtime manifests: +`make k8s-apply` generates and preserves the seed in `Secret/agentenv-auth`, then injects it into the gateway and runtime Pods. Set `AENV_SANDBOX_ACCESS_TOKEN_HASH_SEED` before applying to supply your own value. -```bash -kubectl apply -f deploy/k8s/base/namespace.yaml - -AENV_SANDBOX_ACCESS_TOKEN_HASH_SEED="$(openssl rand -hex 32)" -kubectl -n agentenv-system create secret generic agentenv-runtime-secrets \ - --from-literal="sandbox-access-token-hash-seed=${AENV_SANDBOX_ACCESS_TOKEN_HASH_SEED}" \ - --dry-run=client -o yaml | kubectl apply -f - -unset AENV_SANDBOX_ACCESS_TOKEN_HASH_SEED -``` - -Run this once for a new cluster and preserve the existing Secret during upgrades. An external secret manager may be used instead, provided it creates the same Secret name and key: +An external secret manager may provide the same Secret and key: ```yaml apiVersion: v1 kind: Secret metadata: - name: agentenv-runtime-secrets + name: agentenv-auth namespace: agentenv-system stringData: - sandbox-access-token-hash-seed: + AENV_SANDBOX_ACCESS_TOKEN_HASH_SEED: ``` - -If the Secret is not created, the DaemonSet still starts and each runtime Pod uses its automatically managed node-local seed. diff --git a/services/README.md b/services/README.md index ff6743d65..9bb1d066b 100644 --- a/services/README.md +++ b/services/README.md @@ -73,13 +73,15 @@ Start gateway with the same API key configured on every AgentENV runtime node: ```bash export AENV_API_KEY="e2b_$(openssl rand -hex 32)" +export AENV_SANDBOX_ACCESS_TOKEN_HASH_SEED="$(openssl rand -hex 32)" make run-gateway ``` The default local config uses `127.0.0.1:9090` for the scheduler. -The gateway and runtime nodes require the same API key. The gateway reads an -explicit `AENV_API_KEY` or `/run/secrets/api-key`; it does not generate one. +The gateway and runtime nodes require the same API key and sandbox access-token +seed. The gateway reads explicit environment values or the corresponding files +under `/run/secrets`; it does not generate either secret. Application proxy requests may additionally use the sandbox response's `trafficAccessToken` in the `e2b-traffic-access-token` header. diff --git a/services/gateway/cmd/main.go b/services/gateway/cmd/main.go index 11a8255b9..2471da2c7 100644 --- a/services/gateway/cmd/main.go +++ b/services/gateway/cmd/main.go @@ -25,8 +25,10 @@ import ( ) const ( - apiKeyEnv = "AENV_API_KEY" - defaultAPIKeyPath = "/run/secrets/api-key" + apiKeyEnv = "AENV_API_KEY" + accessTokenSeedEnv = "AENV_SANDBOX_ACCESS_TOKEN_HASH_SEED" + defaultAPIKeyPath = "/run/secrets/api-key" + defaultAccessTokenSeedPath = "/run/secrets/sandbox-access-token-hash-seed" ) func newSchedulerConn(addr string) (*grpc.ClientConn, error) { @@ -41,18 +43,35 @@ func loadAPIKey() (string, error) { } func loadAPIKeyFrom(lookupEnv func(string) (string, bool), secretPath string) (string, error) { - if value, present := lookupEnv(apiKeyEnv); present { - return validateAPIKey(value, apiKeyEnv) + return loadSecretFrom(lookupEnv, apiKeyEnv, secretPath, validateAPIKey) +} + +func loadAccessTokenSeed() (string, error) { + return loadAccessTokenSeedFrom(os.LookupEnv, defaultAccessTokenSeedPath) +} + +func loadAccessTokenSeedFrom(lookupEnv func(string) (string, bool), secretPath string) (string, error) { + return loadSecretFrom(lookupEnv, accessTokenSeedEnv, secretPath, validateAccessTokenSeed) +} + +func loadSecretFrom( + lookupEnv func(string) (string, bool), + envName string, + secretPath string, + validate func(string, string) (string, error), +) (string, error) { + if value, present := lookupEnv(envName); present { + return validate(value, envName) } contents, err := os.ReadFile(secretPath) if err != nil { if os.IsNotExist(err) { - return "", fmt.Errorf("%s must be set or %s must exist", apiKeyEnv, secretPath) + return "", fmt.Errorf("%s must be set or %s must exist", envName, secretPath) } - return "", fmt.Errorf("read API key secret %s: %w", secretPath, err) + return "", fmt.Errorf("read secret %s: %w", secretPath, err) } - return validateAPIKey(string(contents), secretPath) + return validate(string(contents), secretPath) } func validateAPIKey(value, source string) (string, error) { @@ -72,6 +91,14 @@ func validateAPIKey(value, source string) (string, error) { return value, nil } +func validateAccessTokenSeed(value, source string) (string, error) { + value = strings.TrimSpace(value) + if value == "" { + return "", fmt.Errorf("sandbox access-token seed from %s must be non-empty", source) + } + return value, nil +} + func main() { configPath := flag.String("config", "", "path to JSON config file") flag.Parse() @@ -84,6 +111,10 @@ func main() { if err != nil { log.Fatalf("load API key failed: %v", err) } + accessTokenSeed, err := loadAccessTokenSeed() + if err != nil { + log.Fatalf("load sandbox access-token seed failed: %v", err) + } logger, err := logging.New(cfg.LogLevel, cfg.LogFormat) if err != nil { @@ -113,6 +144,7 @@ func main() { RequestTimeout: cfg.Gateway.RequestTimeout, MaxResponseSize: cfg.Gateway.ForwardResponseSize, APIKey: apiKey, + SandboxAccessTokenSeed: accessTokenSeed, DebugMode: cfg.Gateway.DebugMode, SandboxProxyDomains: cfg.Gateway.SandboxProxyDomains, QueryOnlySchedulerClient: queryOnlySchedulerClient, diff --git a/services/gateway/cmd/main_test.go b/services/gateway/cmd/main_test.go index 9064986b3..2fc941651 100644 --- a/services/gateway/cmd/main_test.go +++ b/services/gateway/cmd/main_test.go @@ -8,6 +8,7 @@ import ( ) const testAPIKey = "e2b_0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef" +const testAccessTokenSeed = "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef" func TestValidateAPIKey(t *testing.T) { t.Parallel() @@ -75,3 +76,29 @@ func TestLoadAPIKeyRejectsMissingFile(t *testing.T) { t.Fatal("loadAPIKeyFrom() unexpectedly accepted a missing secret") } } + +func TestLoadAccessTokenSeed(t *testing.T) { + t.Parallel() + + dir := t.TempDir() + path := filepath.Join(dir, "sandbox-access-token-hash-seed") + if err := os.WriteFile(path, []byte(testAccessTokenSeed+"\n"), 0o444); err != nil { + t.Fatal(err) + } + got, err := loadAccessTokenSeedFrom(func(string) (string, bool) { return "", false }, path) + if err != nil { + t.Fatalf("loadAccessTokenSeedFrom() error = %v", err) + } + if got != testAccessTokenSeed { + t.Fatalf("loadAccessTokenSeedFrom() = %q, want %q", got, testAccessTokenSeed) + } +} + +func TestLoadAccessTokenSeedRejectsExplicitEmptyEnvironment(t *testing.T) { + if _, err := loadAccessTokenSeedFrom( + func(name string) (string, bool) { return "", name == accessTokenSeedEnv }, + filepath.Join(t.TempDir(), "missing"), + ); err == nil { + t.Fatal("loadAccessTokenSeedFrom() unexpectedly accepted an empty environment value") + } +} diff --git a/services/gateway/internal/server.go b/services/gateway/internal/server.go index 9f67619b0..24774c6b8 100644 --- a/services/gateway/internal/server.go +++ b/services/gateway/internal/server.go @@ -5,7 +5,7 @@ import ( "context" "crypto/hmac" "crypto/sha256" - "encoding/base64" + "encoding/hex" "encoding/json" "errors" "io" @@ -28,8 +28,7 @@ const ( headerAPIKey = "X-API-Key" headerTrafficToken = "e2b-traffic-access-token" headerEnvdAccessToken = "X-Access-Token" - trafficTokenPrefix = "aenv_trf_" - trafficTokenContext = "agentenv-sandbox-traffic-v1\x00" + trafficTokenPrefix = "sandbox-traffic" envdControlPlanePort = 49983 headerSandboxID = "x-agentenv-sandbox-id" headerE2BSandboxID = "e2b-sandbox-id" @@ -51,6 +50,7 @@ const ( type ServerOptions struct { APIKey string + SandboxAccessTokenSeed string RequestTimeout time.Duration MaxResponseSize int64 DebugMode bool @@ -64,6 +64,7 @@ type Server struct { queryOnlyScheduler schedulerv1.SchedulerClient httpClient *http.Client apiKey []byte + accessTokenSeed []byte requestTimeout time.Duration maxRespSize int64 // debugMode, when true, enables debug-only behaviors such as exposing @@ -78,6 +79,10 @@ func NewServer(logger *zap.Logger, schedulerClient schedulerv1.SchedulerClient, if apiKey == "" { return nil, errors.New("API key is required") } + accessTokenSeed := strings.TrimSpace(options.SandboxAccessTokenSeed) + if accessTokenSeed == "" { + return nil, errors.New("sandbox access-token seed is required") + } sandboxProxyDomains, err := normalizeProxyDomains(options.SandboxProxyDomains) if err != nil { @@ -97,6 +102,7 @@ func NewServer(logger *zap.Logger, schedulerClient schedulerv1.SchedulerClient, requestTimeout: options.RequestTimeout, maxRespSize: options.MaxResponseSize, apiKey: []byte(apiKey), + accessTokenSeed: []byte(accessTokenSeed), debugMode: options.DebugMode, sandboxProxyDomains: sandboxProxyDomains, }, nil @@ -835,11 +841,10 @@ func singleHeaderMatches(headers http.Header, name string, expected []byte) bool return len(values) == 1 && bytes.Equal([]byte(values[0]), expected) } -func trafficAccessToken(apiKey []byte, sandboxID string) string { - mac := hmac.New(sha256.New, apiKey) - _, _ = mac.Write([]byte(trafficTokenContext)) - _, _ = mac.Write([]byte(sandboxID)) - return trafficTokenPrefix + base64.RawURLEncoding.EncodeToString(mac.Sum(nil)) +func trafficAccessToken(seed []byte, sandboxID string) string { + mac := hmac.New(sha256.New, seed) + _, _ = mac.Write([]byte(trafficTokenPrefix + "-" + sandboxID)) + return hex.EncodeToString(mac.Sum(nil)) } func (s *Server) isSandboxDataPlaneRequest(r *http.Request) bool { @@ -898,7 +903,7 @@ func (s *Server) authenticate(next http.Handler) http.Handler { authorized := singleHeaderMatches(r.Header, headerAPIKey, s.apiKey) if !authorized && dataPlane { if sandboxID, ok := s.sandboxIDForDataPlaneAuth(r); ok { - expected := trafficAccessToken(s.apiKey, sandboxID) + expected := trafficAccessToken(s.accessTokenSeed, sandboxID) authorized = singleHeaderMatches(r.Header, headerTrafficToken, []byte(expected)) } } diff --git a/services/gateway/internal/server_test.go b/services/gateway/internal/server_test.go index 0a86d860b..85f49e342 100644 --- a/services/gateway/internal/server_test.go +++ b/services/gateway/internal/server_test.go @@ -148,7 +148,10 @@ func (s stubSchedulerClient) UnregisterNode(ctx context.Context, req *schedulerv return s.unregisterNodeFunc(ctx, req, opts...) } -const testAPIKey = "test-api-key" +const ( + testAPIKey = "test-api-key" + testAccessTokenSeed = "test-access-token-seed" +) type testServerOption func(*ServerOptions) @@ -156,9 +159,10 @@ func newTestServer(t *testing.T, schedulerClient schedulerv1.SchedulerClient, ti t.Helper() options := ServerOptions{ - RequestTimeout: timeout, - MaxResponseSize: maxRespSize, - APIKey: testAPIKey, + RequestTimeout: timeout, + MaxResponseSize: maxRespSize, + APIKey: testAPIKey, + SandboxAccessTokenSeed: testAccessTokenSeed, } for _, opt := range opts { opt(&options) @@ -189,6 +193,17 @@ func TestNewServerRejectsEmptyAPIKey(t *testing.T) { } } +func TestNewServerRejectsEmptySandboxAccessTokenSeed(t *testing.T) { + _, err := NewServer(zap.NewNop(), stubSchedulerClient{}, ServerOptions{ + APIKey: testAPIKey, + RequestTimeout: time.Second, + MaxResponseSize: 1024, + }) + if err == nil { + t.Fatal("NewServer accepted an empty sandbox access-token seed") + } +} + func TestGatewayRequiresExactAPIKey(t *testing.T) { server := newTestServer(t, stubSchedulerClient{}, time.Second, 1024) handler := server.Handler() @@ -262,7 +277,7 @@ func TestGatewayAcceptsSandboxScopedTrafficToken(t *testing.T) { req := httptest.NewRequest(http.MethodGet, "/proxy", nil) req.Header.Set(headerE2BSandboxID, sandboxID) req.Header.Set(headerE2BTargetPort, "49983") - req.Header.Set(headerTrafficToken, trafficAccessToken([]byte(testAPIKey), sandboxID)) + req.Header.Set(headerTrafficToken, trafficAccessToken([]byte(testAccessTokenSeed), sandboxID)) recorder := httptest.NewRecorder() handler.ServeHTTP(recorder, req) if recorder.Code == http.StatusUnauthorized || lookupCalls != 1 { @@ -272,7 +287,7 @@ func TestGatewayAcceptsSandboxScopedTrafficToken(t *testing.T) { req = httptest.NewRequest(http.MethodGet, "/proxy", nil) req.Header.Set(headerE2BSandboxID, sandboxID) req.Header.Set(headerE2BTargetPort, "49983") - req.Header.Set(headerTrafficToken, trafficAccessToken([]byte(testAPIKey), "another-sandbox")) + req.Header.Set(headerTrafficToken, trafficAccessToken([]byte(testAccessTokenSeed), "another-sandbox")) recorder = httptest.NewRecorder() handler.ServeHTTP(recorder, req) if recorder.Code != http.StatusUnauthorized || lookupCalls != 1 { @@ -282,7 +297,7 @@ func TestGatewayAcceptsSandboxScopedTrafficToken(t *testing.T) { req = httptest.NewRequest(http.MethodGet, "/proxy", nil) req.Header.Set(headerE2BSandboxID, sandboxID) req.Header.Set(headerE2BTargetPort, "8080") - req.Header.Set("X-Access-Token", trafficAccessToken([]byte(testAPIKey), sandboxID)) + req.Header.Set("X-Access-Token", trafficAccessToken([]byte(testAccessTokenSeed), sandboxID)) recorder = httptest.NewRecorder() handler.ServeHTTP(recorder, req) if recorder.Code != http.StatusUnauthorized || lookupCalls != 1 { @@ -291,7 +306,7 @@ func TestGatewayAcceptsSandboxScopedTrafficToken(t *testing.T) { req = httptest.NewRequest(http.MethodPost, "/sandboxes/"+sandboxID+"/pause", nil) req.Header.Set(headerE2BSandboxID, sandboxID) - req.Header.Set(headerTrafficToken, trafficAccessToken([]byte(testAPIKey), sandboxID)) + req.Header.Set(headerTrafficToken, trafficAccessToken([]byte(testAccessTokenSeed), sandboxID)) recorder = httptest.NewRecorder() handler.ServeHTTP(recorder, req) @@ -302,8 +317,8 @@ func TestGatewayAcceptsSandboxScopedTrafficToken(t *testing.T) { func TestTrafficAccessTokenVector(t *testing.T) { const sandboxID = "0191f4d0-7b2a-7c11-9c2d-0123456789ab" - const want = "aenv_trf_PwHqhTxLa_mzUCNIGx03uiTHxZ3k995pKDOS50PaGWo" - if got := trafficAccessToken([]byte("test-key"), sandboxID); got != want { + const want = "f5457a589b09265b169392dd49506ec70458f685cf2ba7fc2c5b4763c42a5b17" + if got := trafficAccessToken([]byte("test-seed"), sandboxID); got != want { t.Fatalf("trafficAccessToken() = %q, want %q", got, want) } } diff --git a/src/api/impls/auth.rs b/src/api/impls/auth.rs index 6d5257243..9084f9f10 100644 --- a/src/api/impls/auth.rs +++ b/src/api/impls/auth.rs @@ -1,3 +1,4 @@ +use agentenv_http_server::apis; use async_trait::async_trait; use axum::{ body::Body, @@ -6,20 +7,13 @@ use axum::{ middleware::Next, response::{IntoResponse, Response}, }; -use base64::{engine::general_purpose::URL_SAFE_NO_PAD, Engine}; -use hmac::{Hmac, Mac}; -use sha2::Sha256; - -use agentenv_http_server::apis; use super::{ApiImpl, Claims}; -use crate::api::proxy; +use crate::{api::proxy, types::SandboxId}; pub(crate) const API_KEY_HEADER: &str = "x-api-key"; pub(crate) const TRAFFIC_ACCESS_TOKEN_HEADER: &str = "e2b-traffic-access-token"; pub(crate) const ENVD_ACCESS_TOKEN_HEADER: &str = "x-access-token"; -const TRAFFIC_TOKEN_PREFIX: &str = "aenv_trf_"; -const TRAFFIC_TOKEN_CONTEXT: &[u8] = b"agentenv-sandbox-traffic-v1\0"; fn single_header_matches(headers: &HeaderMap, name: &str, expected: &str) -> bool { let mut values = headers.get_all(name).iter(); @@ -33,29 +27,26 @@ fn single_header_matches(headers: &HeaderMap, name: &str, expected: &str) -> boo value.as_bytes() == expected.as_bytes() } -fn derive_traffic_access_token(api_key: &[u8], sandbox_id: &str) -> String { - let mut mac = - Hmac::::new_from_slice(api_key).expect("HMAC accepts API keys of any length"); - mac.update(TRAFFIC_TOKEN_CONTEXT); - mac.update(sandbox_id.as_bytes()); - format!( - "{TRAFFIC_TOKEN_PREFIX}{}", - URL_SAFE_NO_PAD.encode(mac.finalize().into_bytes()) - ) -} - impl ApiImpl { pub(crate) fn has_valid_api_key(&self, headers: &HeaderMap) -> bool { single_header_matches(headers, API_KEY_HEADER, &self.api_key) } - pub(crate) fn traffic_access_token(&self, sandbox_id: &str) -> String { - derive_traffic_access_token(self.api_key.as_bytes(), sandbox_id) + pub(crate) fn traffic_access_token(&self, sandbox_id: SandboxId) -> String { + self.orchestrator.traffic_access_token(sandbox_id) } - fn has_valid_traffic_access_token(&self, headers: &HeaderMap, sandbox_id: &str) -> bool { - let expected = self.traffic_access_token(sandbox_id); - single_header_matches(headers, TRAFFIC_ACCESS_TOKEN_HEADER, &expected) + fn has_valid_traffic_access_token(&self, headers: &HeaderMap, sandbox_id: SandboxId) -> bool { + let mut values = headers.get_all(TRAFFIC_ACCESS_TOKEN_HEADER).iter(); + let Some(candidate) = values.next().and_then(|value| value.to_str().ok()) else { + return false; + }; + if values.next().is_some() { + return false; + } + + self.orchestrator + .validate_traffic_access_token(sandbox_id, candidate) } } @@ -80,7 +71,7 @@ where .is_some_and(|sandbox_id| { api_impl .as_ref() - .has_valid_traffic_access_token(request.headers(), &sandbox_id) + .has_valid_traffic_access_token(request.headers(), sandbox_id) }); } if !authorized && proxy_request { @@ -165,12 +156,4 @@ mod tests { "correct-key" )); } - - #[test] - fn traffic_access_token_matches_gateway_contract() { - assert_eq!( - derive_traffic_access_token(b"test-key", "0191f4d0-7b2a-7c11-9c2d-0123456789ab"), - "aenv_trf_PwHqhTxLa_mzUCNIGx03uiTHxZ3k995pKDOS50PaGWo" - ); - } } diff --git a/src/api/impls/sandbox.rs b/src/api/impls/sandbox.rs index 85b91f84b..48648dc8a 100644 --- a/src/api/impls/sandbox.rs +++ b/src/api/impls/sandbox.rs @@ -214,15 +214,14 @@ impl From for models::SandboxDetail { impl ApiImpl { fn sandbox_model(&self, metadata: SandboxMetadata) -> models::Sandbox { + let traffic_access_token = self.traffic_access_token(metadata.id); let envd_access_token = self .orchestrator .get_envd_access_token(&metadata) .map(|token| token.expose().to_owned()); let mut sandbox = models::Sandbox::from(metadata); sandbox.envd_access_token = envd_access_token; - sandbox.traffic_access_token = Some(Nullable::Present( - self.traffic_access_token(&sandbox.sandbox_id), - )); + sandbox.traffic_access_token = Some(Nullable::Present(traffic_access_token)); sandbox.domain = self .sandbox_proxy_domains() .first() diff --git a/src/api/proxy.rs b/src/api/proxy.rs index 1cfcb015e..df8d7b19f 100644 --- a/src/api/proxy.rs +++ b/src/api/proxy.rs @@ -161,12 +161,11 @@ fn proxy_route_for_auth(request: &Request, domains: &[String]) -> Option Option { - Some( - proxy_route_for_auth(request, domains)? - .sandbox_id - .to_string(), - ) +pub(crate) fn sandbox_id_for_proxy_auth( + request: &Request, + domains: &[String], +) -> Option { + Some(proxy_route_for_auth(request, domains)?.sandbox_id) } pub(crate) fn envd_access_token_for_proxy_auth( @@ -1780,7 +1779,7 @@ mod tests { sandbox_id: &SandboxId, ) -> (axum::Router, String) { let api = build_api().await; - let access_token = api.traffic_access_token(&sandbox_id.to_string()); + let access_token = api.traffic_access_token(*sandbox_id); api.orchestrator() .set_proxy_target_for_test( *sandbox_id, @@ -1796,7 +1795,7 @@ mod tests { domains: Vec, ) -> (axum::Router, String) { let api = build_api_with_sandbox_proxy_domains(domains).await; - let access_token = api.traffic_access_token(&sandbox_id.to_string()); + let access_token = api.traffic_access_token(*sandbox_id); api.orchestrator() .set_proxy_target_for_test( *sandbox_id, @@ -2006,7 +2005,7 @@ mod tests { async fn traffic_token_cannot_authenticate_control_plane() { let api = build_api().await; let sandbox_id = SandboxId::new(); - let traffic_token = api.traffic_access_token(&sandbox_id.to_string()); + let traffic_token = api.traffic_access_token(sandbox_id); let app = server::new(api); let response = app .oneshot( @@ -2029,7 +2028,7 @@ mod tests { async fn envd_token_cannot_authenticate_application_proxy() { let api = build_api().await; let sandbox_id = SandboxId::new(); - let traffic_token = api.traffic_access_token(&sandbox_id.to_string()); + let traffic_token = api.traffic_access_token(sandbox_id); let response = server::new(api) .oneshot( Request::builder() diff --git a/src/orchestrator/service.rs b/src/orchestrator/service.rs index 079362ede..9f4a0e65c 100644 --- a/src/orchestrator/service.rs +++ b/src/orchestrator/service.rs @@ -162,12 +162,12 @@ where // Restore persisted sandboxes from the previous run, keeping the paused // ones (with their state) for the paused-protection reconcile below. let persisted = persister.load_all(&factory).await?; - let managed_seed_must_exist = persisted.iter().any(|metadata| metadata.secure); + let managed_seed_must_exist = !persisted.is_empty(); let access_tokens = tokio::task::spawn_blocking(move || { SandboxAccessTokenGenerator::load_or_create(app_config, managed_seed_must_exist) }) .await - .context("join envd access-token seed loader")??; + .context("join sandbox access-token seed loader")??; let restored_paused: Vec<(SandboxId, Arc)> = persisted .iter() .filter(|metadata| metadata.state == SandboxState::Paused) @@ -741,6 +741,14 @@ where self.access_tokens.matches(sandbox_id, candidate) } + pub fn traffic_access_token(&self, sandbox_id: SandboxId) -> String { + self.access_tokens.generate_traffic(sandbox_id) + } + + pub fn validate_traffic_access_token(&self, sandbox_id: SandboxId, candidate: &str) -> bool { + self.access_tokens.matches_traffic(sandbox_id, candidate) + } + /// Resolves the current proxyability of a sandbox without touching the sandbox mutex. #[tracing::instrument(skip(self), fields(sandbox_id = %sandbox_id))] pub async fn proxy_lookup_for(&self, sandbox_id: &SandboxId) -> Result { diff --git a/src/sandbox/access.rs b/src/sandbox/access.rs index 3a6204302..f3c88dc6a 100644 --- a/src/sandbox/access.rs +++ b/src/sandbox/access.rs @@ -15,9 +15,11 @@ use crate::types::SandboxId; type HmacSha256 = Hmac; const MANAGED_SEED_RELATIVE_PATH: &str = "secrets/sandbox-access-token-hash-seed"; +const EXTERNAL_SEED_PATH: &str = "/run/secrets/sandbox-access-token-hash-seed"; const MANAGED_SEED_BYTES: usize = 32; const SEED_HEX_LEN: usize = MANAGED_SEED_BYTES * 2; const MANAGED_SEED_FILE_MAX_LEN: usize = SEED_HEX_LEN + 1; +const TRAFFIC_ACCESS_TOKEN_PREFIX: &str = "sandbox-traffic"; #[derive(Clone, PartialEq, Eq)] pub struct EnvdAccessToken(String); @@ -55,6 +57,24 @@ impl SandboxAccessTokenGenerator { return Self::new(seed); } + match fs::read_to_string(EXTERNAL_SEED_PATH) { + Ok(seed) => { + let generator = + Self::new(&seed).context("invalid external sandbox access-token seed")?; + info!( + path = EXTERNAL_SEED_PATH, + "loaded sandbox access-token seed from external secret" + ); + return Ok(generator); + } + Err(error) if error.kind() == io::ErrorKind::NotFound => {} + Err(error) => { + return Err(error).with_context(|| { + format!("read external sandbox access-token seed {EXTERNAL_SEED_PATH}") + }); + } + } + let managed_seed_path = config.home_path.join(MANAGED_SEED_RELATIVE_PATH); let seed = resolve_seed(&managed_seed_path, managed_seed_must_exist)?; @@ -63,7 +83,7 @@ impl SandboxAccessTokenGenerator { { warn!( path = %managed_seed_path.display(), - "using a node-local managed envd access-token seed; configure AENV_SANDBOX_ACCESS_TOKEN_HASH_SEED with the same value on every node before enabling cross-node sandbox recovery" + "using a node-local managed sandbox access-token seed; configure AENV_SANDBOX_ACCESS_TOKEN_HASH_SEED with the same value on every node in a clustered deployment" ); } @@ -71,18 +91,37 @@ impl SandboxAccessTokenGenerator { } pub fn generate(&self, subject: SandboxId) -> EnvdAccessToken { + EnvdAccessToken(self.generate_for(subject.to_string().as_bytes())) + } + + pub fn generate_traffic(&self, subject: SandboxId) -> String { + self.generate_for(format!("{TRAFFIC_ACCESS_TOKEN_PREFIX}-{subject}").as_bytes()) + } + + fn generate_for(&self, subject: &[u8]) -> String { let mut mac = HmacSha256::new_from_slice(&self.seed).expect("HMAC accepts keys of any length"); - mac.update(subject.to_string().as_bytes()); - EnvdAccessToken(hex::encode(mac.finalize().into_bytes())) + mac.update(subject); + hex::encode(mac.finalize().into_bytes()) } pub fn matches(&self, subject: SandboxId, candidate: &str) -> bool { + self.matches_for(subject.to_string().as_bytes(), candidate) + } + + pub fn matches_traffic(&self, subject: SandboxId, candidate: &str) -> bool { + self.matches_for( + format!("{TRAFFIC_ACCESS_TOKEN_PREFIX}-{subject}").as_bytes(), + candidate, + ) + } + + fn matches_for(&self, subject: &[u8], candidate: &str) -> bool { let mut candidate_bytes = [0_u8; 32]; let decoded = hex::decode_to_slice(candidate, &mut candidate_bytes).is_ok(); let mut mac = HmacSha256::new_from_slice(&self.seed).expect("HMAC accepts keys of any length"); - mac.update(subject.to_string().as_bytes()); + mac.update(subject); mac.verify_slice(&candidate_bytes).is_ok() & decoded } } @@ -98,7 +137,7 @@ fn validate_explicit_seed(seed: &str) -> Result<&str> { fn resolve_seed(managed_path: &Path, managed_seed_must_exist: bool) -> Result { let parent = managed_path .parent() - .context("managed envd access-token seed path has no parent")?; + .context("managed sandbox access-token seed path has no parent")?; match validate_managed_seed_directory(parent) { Ok(()) => {} Err(error) if error.kind() == io::ErrorKind::NotFound => {} @@ -115,7 +154,7 @@ fn resolve_seed(managed_path: &Path, managed_seed_must_exist: bool) -> Result { return Err(error).with_context(|| { format!( - "open managed envd access-token seed {}", + "open managed sandbox access-token seed {}", managed_path.display() ) }); @@ -124,7 +163,7 @@ fn resolve_seed(managed_path: &Path, managed_seed_must_exist: bool) -> Result io::Result { } fn validate_managed_seed_file(path: &Path, file: &File) -> Result { - let metadata = file - .metadata() - .with_context(|| format!("inspect managed envd access-token seed {}", path.display()))?; + let metadata = file.metadata().with_context(|| { + format!( + "inspect managed sandbox access-token seed {}", + path.display() + ) + })?; if !metadata.is_file() { bail!( - "managed envd access-token seed {} must be a regular file", + "managed sandbox access-token seed {} must be a regular file", path.display() ); } @@ -164,14 +206,14 @@ fn validate_managed_seed_file(path: &Path, file: &File) -> Result let mode = metadata.permissions().mode() & 0o777; if mode != 0o600 { bail!( - "managed envd access-token seed {} must have permissions 0600, found {mode:04o}", + "managed sandbox access-token seed {} must have permissions 0600, found {mode:04o}", path.display() ); } let expected_uid = nix::unistd::Uid::effective().as_raw(); if metadata.uid() != expected_uid { bail!( - "managed envd access-token seed {} must be owned by uid {expected_uid}, found uid {}", + "managed sandbox access-token seed {} must be owned by uid {expected_uid}, found uid {}", path.display(), metadata.uid() ); @@ -186,7 +228,7 @@ fn read_managed_seed(path: &Path, mut file: File) -> Result { if metadata.len() > MANAGED_SEED_FILE_MAX_LEN as u64 { bail!( - "managed envd access-token seed {} must be at most {MANAGED_SEED_FILE_MAX_LEN} bytes", + "managed sandbox access-token seed {} must be at most {MANAGED_SEED_FILE_MAX_LEN} bytes", path.display() ); } @@ -195,17 +237,17 @@ fn read_managed_seed(path: &Path, mut file: File) -> Result { Read::by_ref(&mut file) .take((MANAGED_SEED_FILE_MAX_LEN + 1) as u64) .read_to_string(&mut contents) - .with_context(|| format!("read managed envd access-token seed {}", path.display()))?; + .with_context(|| format!("read managed sandbox access-token seed {}", path.display()))?; if contents.len() > MANAGED_SEED_FILE_MAX_LEN { bail!( - "managed envd access-token seed {} must be at most {MANAGED_SEED_FILE_MAX_LEN} bytes", + "managed sandbox access-token seed {} must be at most {MANAGED_SEED_FILE_MAX_LEN} bytes", path.display() ); } let seed = contents.strip_suffix('\n').unwrap_or(&contents); if !is_valid_managed_seed(seed) { bail!( - "managed envd access-token seed {} must contain exactly {SEED_HEX_LEN} lowercase hexadecimal characters, optionally followed by a newline", + "managed sandbox access-token seed {} must contain exactly {SEED_HEX_LEN} lowercase hexadecimal characters, optionally followed by a newline", path.display() ); } @@ -216,7 +258,7 @@ fn read_managed_seed(path: &Path, mut file: File) -> Result { fn create_managed_seed(path: &Path) -> Result { let parent = path .parent() - .context("managed envd access-token seed path has no parent")?; + .context("managed sandbox access-token seed path has no parent")?; fs::create_dir_all(parent) .with_context(|| format!("create managed secret directory {}", parent.display()))?; validate_managed_seed_directory_identity(parent).with_context(|| { @@ -232,7 +274,7 @@ fn create_managed_seed(path: &Path) -> Result { let mut random = [0_u8; MANAGED_SEED_BYTES]; SysRng .try_fill_bytes(&mut random) - .context("generate managed envd access-token seed")?; + .context("generate managed sandbox access-token seed")?; let seed = hex::encode(random); let mut temporary = tempfile::NamedTempFile::new_in(parent) @@ -250,17 +292,21 @@ fn create_managed_seed(path: &Path) -> Result { fs::File::open(parent) .and_then(|directory| directory.sync_all()) .with_context(|| format!("sync managed secret directory {}", parent.display()))?; - info!(path = %path.display(), "generated managed envd access-token seed"); + info!(path = %path.display(), "generated managed sandbox access-token seed"); Ok(seed) } Err(error) if error.error.kind() == io::ErrorKind::AlreadyExists => { let file = open_managed_seed(path).with_context(|| { - format!("open managed envd access-token seed {}", path.display()) + format!("open managed sandbox access-token seed {}", path.display()) })?; read_managed_seed(path, file) } - Err(error) => Err(error.error) - .with_context(|| format!("persist managed envd access-token seed {}", path.display())), + Err(error) => Err(error.error).with_context(|| { + format!( + "persist managed sandbox access-token seed {}", + path.display() + ) + }), } } @@ -350,22 +396,31 @@ mod tests { } #[test] - fn generates_lowercase_hex_hmac_sha256() { + fn generates_e2b_compatible_access_tokens() { let generator = SandboxAccessTokenGenerator::new("test-seed").unwrap(); let subject = SandboxId::try_from("01936f8e-72f5-7000-8000-000000000001").unwrap(); - let token = generator.generate(subject); + let envd_token = generator.generate(subject); + let traffic_token = generator.generate_traffic(subject); - assert_eq!(token.expose().len(), 64); assert_eq!( - token.expose(), + envd_token.expose(), "4f00f2a93a87c37161ae01c59b6d4f84506668113441277e9f6272dd4bfae1a7" ); - assert!(token.expose().bytes().all(|byte| byte.is_ascii_hexdigit())); - assert_eq!(token.expose(), token.expose().to_ascii_lowercase()); - assert!(generator.matches(subject, token.expose())); + assert_eq!( + traffic_token, + "586547d7c10facb0f4871297fdbfd9d2b4376f4b02b2e1487646c1c87a293bd8" + ); + assert!(envd_token + .expose() + .bytes() + .all(|byte| byte.is_ascii_hexdigit())); + assert!(traffic_token.bytes().all(|byte| byte.is_ascii_hexdigit())); + assert!(generator.matches(subject, envd_token.expose())); + assert!(generator.matches_traffic(subject, &traffic_token)); assert!(!generator.matches(subject, "not-a-token")); assert!(!generator.matches(subject, &"0".repeat(64))); + assert!(!generator.matches_traffic(subject, envd_token.expose())); } #[test] @@ -511,7 +566,7 @@ mod tests { assert!(error .to_string() - .contains("open managed envd access-token seed")); + .contains("open managed sandbox access-token seed")); Ok(()) } @@ -562,15 +617,13 @@ mod tests { } #[test] - fn missing_managed_seed_is_not_recreated_for_secure_state() -> Result<()> { + fn missing_managed_seed_is_not_recreated_for_persisted_state() -> Result<()> { let temp = TempDir::new()?; let managed_path = temp.path().join(MANAGED_SEED_RELATIVE_PATH); let error = resolve_seed(&managed_path, true).unwrap_err(); - assert!(error - .to_string() - .contains("persisted secure sandboxes exist")); + assert!(error.to_string().contains("persisted sandboxes exist")); assert!(!managed_path.exists()); Ok(()) } From 326ebfb83a817f0abe202c1a205efa06886e2056 Mon Sep 17 00:00:00 2001 From: Yingdi Shan <5491399+yingdi-shan@users.noreply.github.com> Date: Sat, 15 Aug 2026 14:25:03 +0000 Subject: [PATCH 4/7] refactor: simplify shared authentication secrets --- deploy/k8s/base/agentenv-daemonset.yaml | 5 +- deploy/k8s/base/gateway-deployment.yaml | 5 - deploy/k8s/base/kustomization.yaml | 1 - .../k8s/overlays/local-dev/kustomization.yaml | 8 ++ deploy/k8s/run.sh | 35 +----- docs/src/configuration/authentication.md | 20 ++-- docs/src/configuration/env-vars.md | 2 +- docs/src/configuration/reference.md | 4 +- docs/src/deployment/docker-compose.md | 7 +- docs/src/deployment/kubernetes.md | 19 ++-- docs/src/deployment/static-multi-node.md | 9 +- docs/src/security/secure-sandboxes.md | 28 +++-- services/README.md | 6 +- services/gateway/cmd/main.go | 59 +++------- services/gateway/cmd/main_test.go | 27 ----- services/gateway/internal/server.go | 58 +--------- services/gateway/internal/server_test.go | 51 +++------ src/api/impls/auth.rs | 106 +++++------------- src/api/proxy.rs | 56 ++------- src/sandbox/access.rs | 19 ---- 20 files changed, 142 insertions(+), 383 deletions(-) diff --git a/deploy/k8s/base/agentenv-daemonset.yaml b/deploy/k8s/base/agentenv-daemonset.yaml index e95fefb96..470b2a9cb 100644 --- a/deploy/k8s/base/agentenv-daemonset.yaml +++ b/deploy/k8s/base/agentenv-daemonset.yaml @@ -31,8 +31,9 @@ spec: - name: AENV_SANDBOX_ACCESS_TOKEN_HASH_SEED valueFrom: secretKeyRef: - name: agentenv-auth - key: AENV_SANDBOX_ACCESS_TOKEN_HASH_SEED + name: agentenv-runtime-secrets + key: sandbox-access-token-hash-seed + optional: true - name: AENV_VIRTUALIZATION_MODE value: "kvm" - name: API_ADDR diff --git a/deploy/k8s/base/gateway-deployment.yaml b/deploy/k8s/base/gateway-deployment.yaml index 7f7f8644f..08536fc0e 100644 --- a/deploy/k8s/base/gateway-deployment.yaml +++ b/deploy/k8s/base/gateway-deployment.yaml @@ -32,11 +32,6 @@ spec: secretKeyRef: name: agentenv-auth key: AENV_API_KEY - - name: AENV_SANDBOX_ACCESS_TOKEN_HASH_SEED - valueFrom: - secretKeyRef: - name: agentenv-auth - key: AENV_SANDBOX_ACCESS_TOKEN_HASH_SEED - name: GATEWAY_SANDBOX_PROXY_DOMAINS valueFrom: configMapKeyRef: diff --git a/deploy/k8s/base/kustomization.yaml b/deploy/k8s/base/kustomization.yaml index 157d8d626..556f0d700 100644 --- a/deploy/k8s/base/kustomization.yaml +++ b/deploy/k8s/base/kustomization.yaml @@ -34,7 +34,6 @@ secretGenerator: - name: agentenv-auth literals: - AENV_API_KEY= - - AENV_SANDBOX_ACCESS_TOKEN_HASH_SEED= images: - name: agentenv-gateway diff --git a/deploy/k8s/overlays/local-dev/kustomization.yaml b/deploy/k8s/overlays/local-dev/kustomization.yaml index ab182ed57..2a71de5e1 100644 --- a/deploy/k8s/overlays/local-dev/kustomization.yaml +++ b/deploy/k8s/overlays/local-dev/kustomization.yaml @@ -6,6 +6,14 @@ namespace: agentenv-system resources: - ../../base +secretGenerator: + - name: agentenv-runtime-secrets + literals: + - sandbox-access-token-hash-seed=agentenv-local-dev-access-token-hash-seed + +generatorOptions: + disableNameSuffixHash: true + patches: - target: kind: DaemonSet diff --git a/deploy/k8s/run.sh b/deploy/k8s/run.sh index 49f6aec92..e1e2ba892 100644 --- a/deploy/k8s/run.sh +++ b/deploy/k8s/run.sh @@ -38,17 +38,15 @@ if [[ "${MODE}" == "apply" ]]; then fi fi -read_existing_secret() { - local secret="$1" - local key="$2" +read_existing_api_key() { local encoded_value="" if [[ -z "${namespace_name}" ]]; then return 0 fi - if ! encoded_value="$("${KUBECTL_BIN}" -n "${NAMESPACE}" get secret "${secret}" \ - --ignore-not-found -o "go-template={{index .data \"${key}\"}}")"; then - echo "failed to read ${key} from Secret ${NAMESPACE}/${secret}" >&2 + if ! encoded_value="$("${KUBECTL_BIN}" -n "${NAMESPACE}" get secret agentenv-auth \ + --ignore-not-found -o 'go-template={{index .data "AENV_API_KEY"}}')"; then + echo "failed to read AENV_API_KEY from Secret ${NAMESPACE}/agentenv-auth" >&2 return 1 fi if [[ -n "${encoded_value}" ]]; then @@ -60,27 +58,7 @@ if [[ "${MODE}" != "delete" ]]; then API_KEY_VALUE="" if [[ "${AENV_API_KEY+x}" == "x" ]]; then API_KEY_VALUE="${AENV_API_KEY}" - elif ! API_KEY_VALUE="$(read_existing_secret agentenv-auth AENV_API_KEY)"; then - exit 1 - fi - - ACCESS_TOKEN_SEED_VALUE="" - if [[ "${AENV_SANDBOX_ACCESS_TOKEN_HASH_SEED+x}" == "x" ]]; then - ACCESS_TOKEN_SEED_VALUE="${AENV_SANDBOX_ACCESS_TOKEN_HASH_SEED}" - elif ! ACCESS_TOKEN_SEED_VALUE="$(read_existing_secret agentenv-auth AENV_SANDBOX_ACCESS_TOKEN_HASH_SEED)"; then - exit 1 - fi - if [[ -z "${ACCESS_TOKEN_SEED_VALUE}" ]]; then - if ! ACCESS_TOKEN_SEED_VALUE="$(read_existing_secret agentenv-runtime-secrets sandbox-access-token-hash-seed)"; then - exit 1 - fi - fi - - if [[ -z "${ACCESS_TOKEN_SEED_VALUE}" ]]; then - ACCESS_TOKEN_SEED_VALUE="$(od -An -N32 -tx1 /dev/urandom | tr -d '[:space:]')" - fi - if [[ ! "${ACCESS_TOKEN_SEED_VALUE}" =~ ^[A-Za-z0-9._~-]{32,}$ ]]; then - echo "AENV_SANDBOX_ACCESS_TOKEN_HASH_SEED must contain at least 32 URL-safe characters" >&2 + elif ! API_KEY_VALUE="$(read_existing_api_key)"; then exit 1 fi @@ -95,9 +73,6 @@ if [[ "${MODE}" != "delete" ]]; then sed_in_place \ "s#- AENV_API_KEY=.*#- AENV_API_KEY=${API_KEY_VALUE}#" \ "${TEMP_DIR}/k8s/base/kustomization.yaml" - sed_in_place \ - "s#- AENV_SANDBOX_ACCESS_TOKEN_HASH_SEED=.*#- AENV_SANDBOX_ACCESS_TOKEN_HASH_SEED=${ACCESS_TOKEN_SEED_VALUE}#" \ - "${TEMP_DIR}/k8s/base/kustomization.yaml" fi if [[ "${SANDBOX_PROXY_DOMAINS+x}" == "x" ]]; then diff --git a/docs/src/configuration/authentication.md b/docs/src/configuration/authentication.md index c04773f79..98d5aa689 100644 --- a/docs/src/configuration/authentication.md +++ b/docs/src/configuration/authentication.md @@ -39,10 +39,9 @@ generates a 256-bit key and atomically stores it in the managed path with `0600` permissions. It reuses that key on later starts. Dependency and host setup modes do not create a key. -The gateway uses `AENV_API_KEY` or `/run/secrets/api-key`. It also reads the -sandbox seed from `AENV_SANDBOX_ACCESS_TOKEN_HASH_SEED` or -`/run/secrets/sandbox-access-token-hash-seed`. The gateway never generates -either value because it must share them with every runtime node. +The gateway uses `AENV_API_KEY` or `/run/secrets/api-key`; it never generates a +key. Runtime nodes validate sandbox-scoped tokens, so the gateway does not need +the sandbox seed. ## Installation Methods @@ -70,8 +69,7 @@ container replacements. The checked-in Compose deployment mounts one named volume read-write on both runtime nodes and read-only at `/run/secrets` on the gateway. Concurrent node startup is safe: atomic creation makes both nodes converge on the same key and -sandbox seed. -Read it with: +sandbox seed. The gateway reads only the API key from that volume. Read it with: ```bash docker compose -f deploy/docker-compose.yml exec -T agentenv-a \ @@ -81,8 +79,8 @@ docker compose -f deploy/docker-compose.yml exec -T agentenv-a \ `docker compose down` preserves the key. `docker compose down -v` removes the auth volume, so the next startup generates a new key. -`make k8s-apply` creates `Secret/agentenv-auth` with an API key and sandbox -seed on the first apply, then reuses both values. Read the API key with: +`make k8s-apply` creates `Secret/agentenv-auth` with an API key on the first +apply, then reuses it. Read the key with: ```bash kubectl -n agentenv-system get secret agentenv-auth \ @@ -120,7 +118,7 @@ network, use a VPN, or terminate HTTPS at a reverse proxy or load balancer. ## Rotation Changing `AENV_API_KEY` invalidates existing client API credentials without -changing sandbox credentials. Changing +changing sandbox credentials; apply it to the gateway and every runtime node +together. Changing `AENV_SANDBOX_ACCESS_TOKEN_HASH_SEED` rotates both `trafficAccessToken` and -`envdAccessToken` values. Apply either change to the gateway and every runtime -node together, then restart them. +`envdAccessToken` values and must be changed on every runtime node together. diff --git a/docs/src/configuration/env-vars.md b/docs/src/configuration/env-vars.md index bb458564b..a5f8c47cb 100644 --- a/docs/src/configuration/env-vars.md +++ b/docs/src/configuration/env-vars.md @@ -24,7 +24,7 @@ These variables are consumed by the repository's Docker Compose and Kubernetes h | `AENV_OBSERVABILITY_SCHEDULER_ENDPOINT` | unset | Override scheduler heartbeat reporting endpoint | | `AENV_OBSERVABILITY_REPORT_INTERVAL_SECS` | `5` | Override heartbeat reporting interval in seconds | | `AENV_CUSTOM_EXTENSION_URL` | unset | Override `[custom_extension].url`, the HTTP base URL of the custom extension service | -| `AENV_SANDBOX_ACCESS_TOKEN_HASH_SEED` | `/run/secrets/sandbox-access-token-hash-seed`, then auto-generated under `$AENV_HOME/secrets` | Optional runtime override for the secret used to derive sandbox envd and traffic access tokens. Clustered deployments must configure the same value on the gateway and every runtime node. | +| `AENV_SANDBOX_ACCESS_TOKEN_HASH_SEED` | auto-generated under `$AENV_HOME/secrets` | Optional runtime override for the secret used to derive sandbox envd and traffic access tokens. Configure the same value on every runtime node in clustered deployments. | | `AENV_SANDBOX_PROXY_DOMAINS` | from config | Comma-separated DNS domains that enable server-side host-based sandbox proxy URLs like `{port}-{sandboxID}.{domain}` and populate the sandbox response `domain` field. Empty or unset keeps `[sandbox_proxy].domains`. | | `AENV_HOME_PATH` | `/var/lib/aenv` | Override the base directory from which AgentENV derives local state, caches, logs, generated configs, and downloaded dependencies. Component-specific path settings remain available as advanced overrides. | | `AENV_RUNTIME_PATH` | `/run/aenv` | Override the transient runtime directory used for network namespace mount points and the default ublk daemon socket. | diff --git a/docs/src/configuration/reference.md b/docs/src/configuration/reference.md index a81591d62..6f6465763 100644 --- a/docs/src/configuration/reference.md +++ b/docs/src/configuration/reference.md @@ -259,9 +259,9 @@ Sandbox control communication settings. |-----|------|---------|-------------| | `access_token_hash_seed` | string | auto-generated | Optional override for the secret used to derive sandbox envd and traffic access tokens. When unset, normal server startup creates and reuses `$AENV_HOME/secrets/sandbox-access-token-hash-seed`. Configure an explicit shared value for clustered deployments. | -The managed seed is node-local persistent state and must be included in backups of `$AENV_HOME`. AgentENV refuses to generate a replacement when persisted secure sandboxes exist. An explicit environment or TOML value takes precedence over the managed file; changing that effective value invalidates access tokens for existing secure sandboxes. +The managed seed is node-local persistent state and must be included in backups of `$AENV_HOME`. AgentENV refuses to generate a replacement when persisted sandboxes exist. An explicit environment or TOML value takes precedence over the managed file; changing that effective value invalidates existing sandbox access tokens. -Configure `AENV_SANDBOX_ACCESS_TOKEN_HASH_SEED` with the same value on the gateway and every runtime node in a clustered deployment. Standalone runtime nodes use their managed seed when it is unset. +Configure `AENV_SANDBOX_ACCESS_TOKEN_HASH_SEED` with the same value on every runtime node in a clustered deployment. Standalone runtime nodes use their managed seed when it is unset. ## `[orchestrator]` diff --git a/docs/src/deployment/docker-compose.md b/docs/src/deployment/docker-compose.md index f9edc7e03..4e10bbd97 100644 --- a/docs/src/deployment/docker-compose.md +++ b/docs/src/deployment/docker-compose.md @@ -34,10 +34,9 @@ The Gateway is available at `http://127.0.0.1:8000` and forwards requests to the backend nodes. On first startup, the runtime nodes atomically generate one API key and sandbox -access-token seed in the -shared `agentenv-auth` volume. The gateway mounts that volume read-only at -`/run/secrets`, so all three services use the same secrets. Normal -`make deploy-down` calls preserve the volume and both values. +access-token seed in the shared `agentenv-auth` volume. The gateway mounts that +volume read-only and reads the API key; sandbox tokens are validated by the +runtime nodes. Normal `make deploy-down` calls preserve both values. To enable host-based sandbox data-plane URLs, set the shared sandbox proxy domain variable when starting the stack: diff --git a/docs/src/deployment/kubernetes.md b/docs/src/deployment/kubernetes.md index 463359f79..f3133a516 100644 --- a/docs/src/deployment/kubernetes.md +++ b/docs/src/deployment/kubernetes.md @@ -61,18 +61,20 @@ make k8s-render make k8s-apply ``` -`make k8s-apply` generates a 256-bit API key and sandbox access-token seed on -the first deployment and stores both in `Secret/agentenv-auth`. Later applies -reuse both values. Read the API key locally when configuring clients: +`make k8s-apply` generates a 256-bit API key on the first deployment and stores +it in `Secret/agentenv-auth`. Later applies reuse it. Read the key locally when +configuring clients: ```bash kubectl -n agentenv-system get secret agentenv-auth \ -o go-template='{{index .data "AENV_API_KEY" | base64decode}}{{"\n"}}' ``` -Set `AENV_API_KEY` and `AENV_SANDBOX_ACCESS_TOKEN_HASH_SEED` when applying to -supply your own values. A standalone `make k8s-render` uses temporary generated -values because it does not modify or read cluster state. +Set `AENV_API_KEY` when applying to supply your own value. A standalone +`make k8s-render` uses a temporary generated value because it does not modify or +read cluster state. The optional runtime seed keeps its existing +`agentenv-runtime-secrets` contract described in +[Secure Sandboxes](../security/secure-sandboxes.md). To enable host-based sandbox data-plane URLs, set the shared sandbox proxy domain variable when rendering or applying manifests: @@ -114,8 +116,9 @@ make k8s-delete A dedicated `local-dev` overlay mounts the repository's `env/` directory directly into the DaemonSet at `/workspace/env`, avoiding runtime asset copies: -The apply helper provisions the same generated `agentenv-auth` secrets used by -the default overlay. +The apply helper provisions the same generated API key used by the default +overlay. The local development overlay retains its fixed test-only runtime seed; +do not reuse that seed outside local development. ```bash make k8s-build diff --git a/docs/src/deployment/static-multi-node.md b/docs/src/deployment/static-multi-node.md index e71f9818f..354cd2f30 100644 --- a/docs/src/deployment/static-multi-node.md +++ b/docs/src/deployment/static-multi-node.md @@ -55,9 +55,9 @@ an external metrics collector needs them. ## 1. Install the runtime nodes -Generate one API key and one sandbox access-token seed, deliver them through -your normal secret-management channel, and use the same values on every runtime -node and the Gateway: +Generate one API key and one sandbox access-token seed through your normal +secret-management channel. Use the API key on the gateway and every runtime +node; use the seed only on runtime nodes: ```bash export AENV_API_KEY="e2b_$(openssl rand -hex 32)" @@ -117,7 +117,7 @@ sudo useradd --system --no-create-home --shell /usr/sbin/nologin agentenv-contro sudo install -d -o root -g agentenv-control -m 0750 /etc/agentenv ``` -Create `/etc/agentenv/auth.env` with the same values used on the runtime nodes: +Create `/etc/agentenv/auth.env` with the API key used on the runtime nodes: ```bash sudo install -o root -g agentenv-control -m 0640 /dev/null /etc/agentenv/auth.env @@ -126,7 +126,6 @@ sudoedit /etc/agentenv/auth.env ```text AENV_API_KEY= -AENV_SANDBOX_ACCESS_TOKEN_HASH_SEED= ``` If the `agentenv-control` account already exists, the `useradd` command reports diff --git a/docs/src/security/secure-sandboxes.md b/docs/src/security/secure-sandboxes.md index 2eadf4b65..32e387501 100644 --- a/docs/src/security/secure-sandboxes.md +++ b/docs/src/security/secure-sandboxes.md @@ -20,31 +20,43 @@ The API and SDKs return the sandbox's `envdAccessToken` where appropriate and at A seed is a random value used to derive each sandbox's envd and traffic access tokens. This seed is optional for a standalone runtime. When it is unset, the runtime automatically creates and persists a seed under `$AENV_HOME/secrets`. This is sufficient for normal single-node operation and does not require additional setup. -Configure the same explicit seed on the gateway and every runtime node in a clustered deployment. Generate it once and store it in the deployment's secret manager: +Configure the same explicit seed on every runtime node in a clustered deployment. Generate it once and store it in the deployment's secret manager: ```bash openssl rand -hex 32 ``` -Set the value as `AENV_SANDBOX_ACCESS_TOKEN_HASH_SEED` on the gateway and every runtime node. +Set the value as `AENV_SANDBOX_ACCESS_TOKEN_HASH_SEED` on every runtime node. For TOML configuration, use `[sandbox].access_token_hash_seed` instead. -Container deployments may mount it at -`/run/secrets/sandbox-access-token-hash-seed`. Preserve the seed across upgrades; changing it rotates both sandbox access tokens. ### Kubernetes -`make k8s-apply` generates and preserves the seed in `Secret/agentenv-auth`, then injects it into the gateway and runtime Pods. Set `AENV_SANDBOX_ACCESS_TOKEN_HASH_SEED` before applying to supply your own value. +The runtime DaemonSet retains the existing optional `agentenv-runtime-secrets` +contract. Create one shared seed before applying the runtime manifests: -An external secret manager may provide the same Secret and key: +```bash +kubectl apply -f deploy/k8s/base/namespace.yaml + +AENV_ACCESS_TOKEN_HASH_SEED="$(openssl rand -hex 32)" +kubectl -n agentenv-system create secret generic agentenv-runtime-secrets \ + --from-literal="sandbox-access-token-hash-seed=${AENV_ACCESS_TOKEN_HASH_SEED}" \ + --dry-run=client -o yaml | kubectl apply -f - +unset AENV_ACCESS_TOKEN_HASH_SEED +``` + +Preserve this Secret during upgrades. An external secret manager may provide +the same name and key: ```yaml apiVersion: v1 kind: Secret metadata: - name: agentenv-auth + name: agentenv-runtime-secrets namespace: agentenv-system stringData: - AENV_SANDBOX_ACCESS_TOKEN_HASH_SEED: + sandbox-access-token-hash-seed: ``` + +If the Secret is absent, each runtime Pod uses its managed node-local seed. diff --git a/services/README.md b/services/README.md index 9bb1d066b..a5fe1869d 100644 --- a/services/README.md +++ b/services/README.md @@ -73,15 +73,13 @@ Start gateway with the same API key configured on every AgentENV runtime node: ```bash export AENV_API_KEY="e2b_$(openssl rand -hex 32)" -export AENV_SANDBOX_ACCESS_TOKEN_HASH_SEED="$(openssl rand -hex 32)" make run-gateway ``` The default local config uses `127.0.0.1:9090` for the scheduler. -The gateway and runtime nodes require the same API key and sandbox access-token -seed. The gateway reads explicit environment values or the corresponding files -under `/run/secrets`; it does not generate either secret. +The gateway and runtime nodes require the same API key. The gateway reads +`AENV_API_KEY` or `/run/secrets/api-key`; it does not generate a key. Application proxy requests may additionally use the sandbox response's `trafficAccessToken` in the `e2b-traffic-access-token` header. diff --git a/services/gateway/cmd/main.go b/services/gateway/cmd/main.go index 2471da2c7..389920ffc 100644 --- a/services/gateway/cmd/main.go +++ b/services/gateway/cmd/main.go @@ -25,10 +25,8 @@ import ( ) const ( - apiKeyEnv = "AENV_API_KEY" - accessTokenSeedEnv = "AENV_SANDBOX_ACCESS_TOKEN_HASH_SEED" - defaultAPIKeyPath = "/run/secrets/api-key" - defaultAccessTokenSeedPath = "/run/secrets/sandbox-access-token-hash-seed" + apiKeyEnv = "AENV_API_KEY" + defaultAPIKeyPath = "/run/secrets/api-key" ) func newSchedulerConn(addr string) (*grpc.ClientConn, error) { @@ -43,35 +41,20 @@ func loadAPIKey() (string, error) { } func loadAPIKeyFrom(lookupEnv func(string) (string, bool), secretPath string) (string, error) { - return loadSecretFrom(lookupEnv, apiKeyEnv, secretPath, validateAPIKey) -} - -func loadAccessTokenSeed() (string, error) { - return loadAccessTokenSeedFrom(os.LookupEnv, defaultAccessTokenSeedPath) -} - -func loadAccessTokenSeedFrom(lookupEnv func(string) (string, bool), secretPath string) (string, error) { - return loadSecretFrom(lookupEnv, accessTokenSeedEnv, secretPath, validateAccessTokenSeed) -} - -func loadSecretFrom( - lookupEnv func(string) (string, bool), - envName string, - secretPath string, - validate func(string, string) (string, error), -) (string, error) { - if value, present := lookupEnv(envName); present { - return validate(value, envName) - } - - contents, err := os.ReadFile(secretPath) - if err != nil { - if os.IsNotExist(err) { - return "", fmt.Errorf("%s must be set or %s must exist", envName, secretPath) + value, source := "", apiKeyEnv + if explicit, present := lookupEnv(apiKeyEnv); present { + value = explicit + } else { + contents, err := os.ReadFile(secretPath) + if err != nil { + if os.IsNotExist(err) { + return "", fmt.Errorf("%s must be set or %s must exist", apiKeyEnv, secretPath) + } + return "", fmt.Errorf("read secret %s: %w", secretPath, err) } - return "", fmt.Errorf("read secret %s: %w", secretPath, err) + value, source = string(contents), secretPath } - return validate(string(contents), secretPath) + return validateAPIKey(value, source) } func validateAPIKey(value, source string) (string, error) { @@ -91,14 +74,6 @@ func validateAPIKey(value, source string) (string, error) { return value, nil } -func validateAccessTokenSeed(value, source string) (string, error) { - value = strings.TrimSpace(value) - if value == "" { - return "", fmt.Errorf("sandbox access-token seed from %s must be non-empty", source) - } - return value, nil -} - func main() { configPath := flag.String("config", "", "path to JSON config file") flag.Parse() @@ -111,11 +86,6 @@ func main() { if err != nil { log.Fatalf("load API key failed: %v", err) } - accessTokenSeed, err := loadAccessTokenSeed() - if err != nil { - log.Fatalf("load sandbox access-token seed failed: %v", err) - } - logger, err := logging.New(cfg.LogLevel, cfg.LogFormat) if err != nil { log.Fatalf("init logger failed: %v", err) @@ -144,7 +114,6 @@ func main() { RequestTimeout: cfg.Gateway.RequestTimeout, MaxResponseSize: cfg.Gateway.ForwardResponseSize, APIKey: apiKey, - SandboxAccessTokenSeed: accessTokenSeed, DebugMode: cfg.Gateway.DebugMode, SandboxProxyDomains: cfg.Gateway.SandboxProxyDomains, QueryOnlySchedulerClient: queryOnlySchedulerClient, diff --git a/services/gateway/cmd/main_test.go b/services/gateway/cmd/main_test.go index 2fc941651..9064986b3 100644 --- a/services/gateway/cmd/main_test.go +++ b/services/gateway/cmd/main_test.go @@ -8,7 +8,6 @@ import ( ) const testAPIKey = "e2b_0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef" -const testAccessTokenSeed = "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef" func TestValidateAPIKey(t *testing.T) { t.Parallel() @@ -76,29 +75,3 @@ func TestLoadAPIKeyRejectsMissingFile(t *testing.T) { t.Fatal("loadAPIKeyFrom() unexpectedly accepted a missing secret") } } - -func TestLoadAccessTokenSeed(t *testing.T) { - t.Parallel() - - dir := t.TempDir() - path := filepath.Join(dir, "sandbox-access-token-hash-seed") - if err := os.WriteFile(path, []byte(testAccessTokenSeed+"\n"), 0o444); err != nil { - t.Fatal(err) - } - got, err := loadAccessTokenSeedFrom(func(string) (string, bool) { return "", false }, path) - if err != nil { - t.Fatalf("loadAccessTokenSeedFrom() error = %v", err) - } - if got != testAccessTokenSeed { - t.Fatalf("loadAccessTokenSeedFrom() = %q, want %q", got, testAccessTokenSeed) - } -} - -func TestLoadAccessTokenSeedRejectsExplicitEmptyEnvironment(t *testing.T) { - if _, err := loadAccessTokenSeedFrom( - func(name string) (string, bool) { return "", name == accessTokenSeedEnv }, - filepath.Join(t.TempDir(), "missing"), - ); err == nil { - t.Fatal("loadAccessTokenSeedFrom() unexpectedly accepted an empty environment value") - } -} diff --git a/services/gateway/internal/server.go b/services/gateway/internal/server.go index 24774c6b8..796d8f69a 100644 --- a/services/gateway/internal/server.go +++ b/services/gateway/internal/server.go @@ -3,9 +3,6 @@ package gateway import ( "bytes" "context" - "crypto/hmac" - "crypto/sha256" - "encoding/hex" "encoding/json" "errors" "io" @@ -28,8 +25,6 @@ const ( headerAPIKey = "X-API-Key" headerTrafficToken = "e2b-traffic-access-token" headerEnvdAccessToken = "X-Access-Token" - trafficTokenPrefix = "sandbox-traffic" - envdControlPlanePort = 49983 headerSandboxID = "x-agentenv-sandbox-id" headerE2BSandboxID = "e2b-sandbox-id" headerTargetPort = "x-agentenv-target-port" @@ -50,7 +45,6 @@ const ( type ServerOptions struct { APIKey string - SandboxAccessTokenSeed string RequestTimeout time.Duration MaxResponseSize int64 DebugMode bool @@ -64,7 +58,6 @@ type Server struct { queryOnlyScheduler schedulerv1.SchedulerClient httpClient *http.Client apiKey []byte - accessTokenSeed []byte requestTimeout time.Duration maxRespSize int64 // debugMode, when true, enables debug-only behaviors such as exposing @@ -79,11 +72,6 @@ func NewServer(logger *zap.Logger, schedulerClient schedulerv1.SchedulerClient, if apiKey == "" { return nil, errors.New("API key is required") } - accessTokenSeed := strings.TrimSpace(options.SandboxAccessTokenSeed) - if accessTokenSeed == "" { - return nil, errors.New("sandbox access-token seed is required") - } - sandboxProxyDomains, err := normalizeProxyDomains(options.SandboxProxyDomains) if err != nil { return nil, err @@ -102,7 +90,6 @@ func NewServer(logger *zap.Logger, schedulerClient schedulerv1.SchedulerClient, requestTimeout: options.RequestTimeout, maxRespSize: options.MaxResponseSize, apiKey: []byte(apiKey), - accessTokenSeed: []byte(accessTokenSeed), debugMode: options.DebugMode, sandboxProxyDomains: sandboxProxyDomains, }, nil @@ -841,12 +828,6 @@ func singleHeaderMatches(headers http.Header, name string, expected []byte) bool return len(values) == 1 && bytes.Equal([]byte(values[0]), expected) } -func trafficAccessToken(seed []byte, sandboxID string) string { - mac := hmac.New(sha256.New, seed) - _, _ = mac.Write([]byte(trafficTokenPrefix + "-" + sandboxID)) - return hex.EncodeToString(mac.Sum(nil)) -} - func (s *Server) isSandboxDataPlaneRequest(r *http.Request) bool { if strings.TrimRight(r.URL.Path, "/") == "/proxy" || strings.HasPrefix(r.URL.Path, "/proxy/") { return true @@ -860,33 +841,6 @@ func (s *Server) isSandboxDataPlaneRequest(r *http.Request) bool { return !isSandboxControlPlaneRequest(r) && hasProxyRoutingHeaders(r.Header) } -func (s *Server) sandboxIDForDataPlaneAuth(r *http.Request) (string, bool) { - hostRoute, err := parseHostRoute(r.Host, s.sandboxProxyDomains) - if err != nil { - return "", false - } - if hostRoute != nil { - return hostRoute.sandboxID, true - } - return sandboxIDFromHeaders(r.Header) -} - -func (s *Server) isEnvdDataPlaneRequest(r *http.Request) bool { - hostRoute, err := parseHostRoute(r.Host, s.sandboxProxyDomains) - if err != nil { - return false - } - if hostRoute != nil { - return hostRoute.targetPort == envdControlPlanePort - } - targetPort, ok := targetPortFromHeaders(r.Header) - if !ok { - return false - } - port, err := strconv.Atoi(targetPort) - return err == nil && port == envdControlPlanePort -} - func hasSingleNonEmptyHeader(headers http.Header, name string) bool { values := headers.Values(name) return len(values) == 1 && strings.TrimSpace(values[0]) != "" @@ -902,15 +856,9 @@ func (s *Server) authenticate(next http.Handler) http.Handler { authorized := singleHeaderMatches(r.Header, headerAPIKey, s.apiKey) if !authorized && dataPlane { - if sandboxID, ok := s.sandboxIDForDataPlaneAuth(r); ok { - expected := trafficAccessToken(s.accessTokenSeed, sandboxID) - authorized = singleHeaderMatches(r.Header, headerTrafficToken, []byte(expected)) - } - } - if !authorized && dataPlane && s.isEnvdDataPlaneRequest(r) { - // The runtime node owns the envd token seed and performs the definitive - // sandbox-scoped validation before forwarding the request to envd. - authorized = hasSingleNonEmptyHeader(r.Header, headerEnvdAccessToken) + // Runtime nodes perform the definitive sandbox-scoped token validation. + authorized = hasSingleNonEmptyHeader(r.Header, headerTrafficToken) || + hasSingleNonEmptyHeader(r.Header, headerEnvdAccessToken) } if !authorized { w.WriteHeader(http.StatusUnauthorized) diff --git a/services/gateway/internal/server_test.go b/services/gateway/internal/server_test.go index 85f49e342..7818628a7 100644 --- a/services/gateway/internal/server_test.go +++ b/services/gateway/internal/server_test.go @@ -148,10 +148,7 @@ func (s stubSchedulerClient) UnregisterNode(ctx context.Context, req *schedulerv return s.unregisterNodeFunc(ctx, req, opts...) } -const ( - testAPIKey = "test-api-key" - testAccessTokenSeed = "test-access-token-seed" -) +const testAPIKey = "test-api-key" type testServerOption func(*ServerOptions) @@ -159,10 +156,9 @@ func newTestServer(t *testing.T, schedulerClient schedulerv1.SchedulerClient, ti t.Helper() options := ServerOptions{ - RequestTimeout: timeout, - MaxResponseSize: maxRespSize, - APIKey: testAPIKey, - SandboxAccessTokenSeed: testAccessTokenSeed, + RequestTimeout: timeout, + MaxResponseSize: maxRespSize, + APIKey: testAPIKey, } for _, opt := range opts { opt(&options) @@ -193,17 +189,6 @@ func TestNewServerRejectsEmptyAPIKey(t *testing.T) { } } -func TestNewServerRejectsEmptySandboxAccessTokenSeed(t *testing.T) { - _, err := NewServer(zap.NewNop(), stubSchedulerClient{}, ServerOptions{ - APIKey: testAPIKey, - RequestTimeout: time.Second, - MaxResponseSize: 1024, - }) - if err == nil { - t.Fatal("NewServer accepted an empty sandbox access-token seed") - } -} - func TestGatewayRequiresExactAPIKey(t *testing.T) { server := newTestServer(t, stubSchedulerClient{}, time.Second, 1024) handler := server.Handler() @@ -263,7 +248,7 @@ func TestGatewayRequiresExactAPIKey(t *testing.T) { } } -func TestGatewayAcceptsSandboxScopedTrafficToken(t *testing.T) { +func TestGatewayForwardsSandboxTokensOnlyOnDataPlane(t *testing.T) { const sandboxID = "0191f4d0-7b2a-7c11-9c2d-0123456789ab" lookupCalls := 0 server := newTestServer(t, stubSchedulerClient{ @@ -277,7 +262,7 @@ func TestGatewayAcceptsSandboxScopedTrafficToken(t *testing.T) { req := httptest.NewRequest(http.MethodGet, "/proxy", nil) req.Header.Set(headerE2BSandboxID, sandboxID) req.Header.Set(headerE2BTargetPort, "49983") - req.Header.Set(headerTrafficToken, trafficAccessToken([]byte(testAccessTokenSeed), sandboxID)) + req.Header.Set(headerTrafficToken, "runtime-validates-this-token") recorder := httptest.NewRecorder() handler.ServeHTTP(recorder, req) if recorder.Code == http.StatusUnauthorized || lookupCalls != 1 { @@ -287,42 +272,34 @@ func TestGatewayAcceptsSandboxScopedTrafficToken(t *testing.T) { req = httptest.NewRequest(http.MethodGet, "/proxy", nil) req.Header.Set(headerE2BSandboxID, sandboxID) req.Header.Set(headerE2BTargetPort, "49983") - req.Header.Set(headerTrafficToken, trafficAccessToken([]byte(testAccessTokenSeed), "another-sandbox")) + req.Header.Set(headerTrafficToken, "wrong-token") recorder = httptest.NewRecorder() handler.ServeHTTP(recorder, req) - if recorder.Code != http.StatusUnauthorized || lookupCalls != 1 { - t.Fatalf("wrong scoped token: status=%d lookup calls=%d", recorder.Code, lookupCalls) + if recorder.Code == http.StatusUnauthorized || lookupCalls != 2 { + t.Fatalf("runtime-scoped token: status=%d lookup calls=%d", recorder.Code, lookupCalls) } req = httptest.NewRequest(http.MethodGet, "/proxy", nil) req.Header.Set(headerE2BSandboxID, sandboxID) req.Header.Set(headerE2BTargetPort, "8080") - req.Header.Set("X-Access-Token", trafficAccessToken([]byte(testAccessTokenSeed), sandboxID)) + req.Header.Set("X-Access-Token", "runtime-validates-this-token") recorder = httptest.NewRecorder() handler.ServeHTTP(recorder, req) - if recorder.Code != http.StatusUnauthorized || lookupCalls != 1 { - t.Fatalf("envd token authorized application proxy: status=%d lookup calls=%d", recorder.Code, lookupCalls) + if recorder.Code == http.StatusUnauthorized || lookupCalls != 3 { + t.Fatalf("runtime-scoped envd token: status=%d lookup calls=%d", recorder.Code, lookupCalls) } req = httptest.NewRequest(http.MethodPost, "/sandboxes/"+sandboxID+"/pause", nil) req.Header.Set(headerE2BSandboxID, sandboxID) - req.Header.Set(headerTrafficToken, trafficAccessToken([]byte(testAccessTokenSeed), sandboxID)) + req.Header.Set(headerTrafficToken, "runtime-validates-this-token") recorder = httptest.NewRecorder() handler.ServeHTTP(recorder, req) - if recorder.Code != http.StatusUnauthorized || lookupCalls != 1 { + if recorder.Code != http.StatusUnauthorized || lookupCalls != 3 { t.Fatalf("scoped token reached control plane: status=%d lookup calls=%d", recorder.Code, lookupCalls) } } -func TestTrafficAccessTokenVector(t *testing.T) { - const sandboxID = "0191f4d0-7b2a-7c11-9c2d-0123456789ab" - const want = "f5457a589b09265b169392dd49506ec70458f685cf2ba7fc2c5b4763c42a5b17" - if got := trafficAccessToken([]byte("test-seed"), sandboxID); got != want { - t.Fatalf("trafficAccessToken() = %q, want %q", got, want) - } -} - func withSandboxProxyDomains(domains ...string) testServerOption { return func(options *ServerOptions) { options.SandboxProxyDomains = domains diff --git a/src/api/impls/auth.rs b/src/api/impls/auth.rs index 9084f9f10..6b197c616 100644 --- a/src/api/impls/auth.rs +++ b/src/api/impls/auth.rs @@ -3,7 +3,7 @@ use async_trait::async_trait; use axum::{ body::Body, extract::{Request, State}, - http::{header::HeaderMap, StatusCode}, + http::{header::HeaderMap, HeaderValue, StatusCode}, middleware::Next, response::{IntoResponse, Response}, }; @@ -15,21 +15,16 @@ pub(crate) const API_KEY_HEADER: &str = "x-api-key"; pub(crate) const TRAFFIC_ACCESS_TOKEN_HEADER: &str = "e2b-traffic-access-token"; pub(crate) const ENVD_ACCESS_TOKEN_HEADER: &str = "x-access-token"; -fn single_header_matches(headers: &HeaderMap, name: &str, expected: &str) -> bool { +fn single_header<'a>(headers: &'a HeaderMap, name: &str) -> Option<&'a HeaderValue> { let mut values = headers.get_all(name).iter(); - let Some(value) = values.next() else { - return false; - }; - if values.next().is_some() { - return false; - } - - value.as_bytes() == expected.as_bytes() + let value = values.next()?; + values.next().is_none().then_some(value) } impl ApiImpl { pub(crate) fn has_valid_api_key(&self, headers: &HeaderMap) -> bool { - single_header_matches(headers, API_KEY_HEADER, &self.api_key) + single_header(headers, API_KEY_HEADER) + .is_some_and(|value| value.as_bytes() == self.api_key.as_bytes()) } pub(crate) fn traffic_access_token(&self, sandbox_id: SandboxId) -> String { @@ -37,16 +32,12 @@ impl ApiImpl { } fn has_valid_traffic_access_token(&self, headers: &HeaderMap, sandbox_id: SandboxId) -> bool { - let mut values = headers.get_all(TRAFFIC_ACCESS_TOKEN_HEADER).iter(); - let Some(candidate) = values.next().and_then(|value| value.to_str().ok()) else { - return false; - }; - if values.next().is_some() { - return false; - } - - self.orchestrator - .validate_traffic_access_token(sandbox_id, candidate) + single_header(headers, TRAFFIC_ACCESS_TOKEN_HEADER) + .and_then(|value| value.to_str().ok()) + .is_some_and(|candidate| { + self.orchestrator + .validate_traffic_access_token(sandbox_id, candidate) + }) } } @@ -64,28 +55,26 @@ where return next.run(request).await; } - let mut authorized = api_impl.as_ref().has_valid_api_key(request.headers()); - if !authorized && proxy_request { - authorized = - proxy::sandbox_id_for_proxy_auth(&request, api_impl.as_ref().sandbox_proxy_domains()) - .is_some_and(|sandbox_id| { - api_impl - .as_ref() - .has_valid_traffic_access_token(request.headers(), sandbox_id) - }); - } + let api_impl = api_impl.as_ref(); + let mut authorized = api_impl.has_valid_api_key(request.headers()); if !authorized && proxy_request { - if let Some((sandbox_id, target_port, candidate)) = proxy::envd_access_token_for_proxy_auth( - &request, - api_impl.as_ref().sandbox_proxy_domains(), - ) { - authorized = proxy::has_valid_envd_access_token( - api_impl.as_ref(), - sandbox_id, - target_port, - candidate, - ) - .await; + if let Some((sandbox_id, target_port)) = + proxy::route_for_auth(&request, api_impl.sandbox_proxy_domains()) + { + authorized = api_impl.has_valid_traffic_access_token(request.headers(), sandbox_id); + let envd_candidate = single_header(request.headers(), ENVD_ACCESS_TOKEN_HEADER) + .and_then(|value| value.to_str().ok()); + if !authorized { + if let Some(candidate) = envd_candidate { + authorized = proxy::has_valid_envd_access_token( + api_impl, + sandbox_id, + target_port, + candidate, + ) + .await; + } + } } } @@ -124,36 +113,3 @@ impl apis::ApiAuthBasic for ApiImpl { self.has_valid_api_key(headers).then_some(Claims) } } - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn header_match_requires_one_exact_value() { - let mut headers = HeaderMap::new(); - assert!(!single_header_matches( - &headers, - API_KEY_HEADER, - "correct-key" - )); - headers.insert(API_KEY_HEADER, "correct-key".parse().unwrap()); - assert!(single_header_matches( - &headers, - API_KEY_HEADER, - "correct-key" - )); - assert!(!single_header_matches( - &headers, - API_KEY_HEADER, - "wrong-key" - )); - - headers.append(API_KEY_HEADER, "correct-key".parse().unwrap()); - assert!(!single_header_matches( - &headers, - API_KEY_HEADER, - "correct-key" - )); - } -} diff --git a/src/api/proxy.rs b/src/api/proxy.rs index df8d7b19f..76a8fdac4 100644 --- a/src/api/proxy.rs +++ b/src/api/proxy.rs @@ -148,49 +148,24 @@ where .with_state(api_impl) } -fn proxy_route_for_auth(request: &Request, domains: &[String]) -> Option { +pub(crate) fn route_for_auth(request: &Request, domains: &[String]) -> Option<(SandboxId, u16)> { match parse_host_proxy_route(request_host(request), domains) { - Ok(Some(route)) => return Some(route), + Ok(Some(route)) => return Some((route.sandbox_id, route.target_port)), Err(_) => return None, Ok(None) => {} } - Some(HostProxyRoute { - sandbox_id: parse_sandbox_id_header(request.headers()).ok()?, - target_port: parse_target_port_header(request.headers()).ok()?, - }) -} - -pub(crate) fn sandbox_id_for_proxy_auth( - request: &Request, - domains: &[String], -) -> Option { - Some(proxy_route_for_auth(request, domains)?.sandbox_id) -} - -pub(crate) fn envd_access_token_for_proxy_auth( - request: &Request, - domains: &[String], -) -> Option<(SandboxId, u16, String)> { - let route = proxy_route_for_auth(request, domains)?; - let candidate = { - let mut candidates = request.headers().get_all(ENVD_ACCESS_TOKEN_HEADER).iter(); - let candidate = candidates.next()?; - if candidates.next().is_some() { - return None; - } - let candidate = candidate.to_str().ok()?; - candidate.to_owned() - }; - - Some((route.sandbox_id, route.target_port, candidate)) + Some(( + parse_sandbox_id_header(request.headers()).ok()?, + parse_target_port_header(request.headers()).ok()?, + )) } pub(crate) async fn has_valid_envd_access_token( api_impl: &ApiImpl, sandbox_id: SandboxId, target_port: u16, - candidate: String, + candidate: &str, ) -> bool { let Ok(Some(metadata)) = api_impl.orchestrator().get_sandbox(&sandbox_id).await else { return false; @@ -201,7 +176,7 @@ pub(crate) async fn has_valid_envd_access_token( api_impl .orchestrator() - .validate_envd_access_token(sandbox_id, &candidate) + .validate_envd_access_token(sandbox_id, candidate) } pub(crate) fn is_sandbox_proxy_request(request: &Request, domains: &[String]) -> bool { @@ -245,17 +220,10 @@ where return next.run(request).await; } - let host = request - .headers() - .get(header::HOST) - .and_then(|host| host.to_str().ok()) - .or_else(|| { - request - .uri() - .authority() - .map(|authority| authority.as_str()) - }); - let host_route = match parse_host_proxy_route(host, api_impl.as_ref().sandbox_proxy_domains()) { + let host_route = match parse_host_proxy_route( + request_host(&request), + api_impl.as_ref().sandbox_proxy_domains(), + ) { Ok(Some(route)) => route, Ok(None) => { return next.run(request).await; diff --git a/src/sandbox/access.rs b/src/sandbox/access.rs index f3c88dc6a..abf13788e 100644 --- a/src/sandbox/access.rs +++ b/src/sandbox/access.rs @@ -15,7 +15,6 @@ use crate::types::SandboxId; type HmacSha256 = Hmac; const MANAGED_SEED_RELATIVE_PATH: &str = "secrets/sandbox-access-token-hash-seed"; -const EXTERNAL_SEED_PATH: &str = "/run/secrets/sandbox-access-token-hash-seed"; const MANAGED_SEED_BYTES: usize = 32; const SEED_HEX_LEN: usize = MANAGED_SEED_BYTES * 2; const MANAGED_SEED_FILE_MAX_LEN: usize = SEED_HEX_LEN + 1; @@ -57,24 +56,6 @@ impl SandboxAccessTokenGenerator { return Self::new(seed); } - match fs::read_to_string(EXTERNAL_SEED_PATH) { - Ok(seed) => { - let generator = - Self::new(&seed).context("invalid external sandbox access-token seed")?; - info!( - path = EXTERNAL_SEED_PATH, - "loaded sandbox access-token seed from external secret" - ); - return Ok(generator); - } - Err(error) if error.kind() == io::ErrorKind::NotFound => {} - Err(error) => { - return Err(error).with_context(|| { - format!("read external sandbox access-token seed {EXTERNAL_SEED_PATH}") - }); - } - } - let managed_seed_path = config.home_path.join(MANAGED_SEED_RELATIVE_PATH); let seed = resolve_seed(&managed_seed_path, managed_seed_must_exist)?; From 131e0dc53247b2d8825e96ef057acb6e00422a12 Mon Sep 17 00:00:00 2001 From: Yingdi Shan <5491399+yingdi-shan@users.noreply.github.com> Date: Sat, 15 Aug 2026 16:47:30 +0000 Subject: [PATCH 5/7] fix: harden managed secret loading --- Cargo.lock | 1 + Cargo.toml | 1 + deploy/k8s/run.sh | 52 ++++- docs/src/configuration/authentication.md | 2 +- scripts/tests/e2e/lib/runtime.sh | 2 +- scripts/tests/e2e/suites/09_e2b_compat.sh | 1 + services/gateway/cmd/main.go | 20 +- services/gateway/cmd/main_test.go | 4 +- src/api/impls/auth.rs | 37 ++-- src/api/proxy.rs | 51 ++--- src/api_key.rs | 92 ++++---- src/lib.rs | 1 + src/managed_secret.rs | 233 ++++++++++++++++++++ src/sandbox/access.rs | 251 ++++------------------ 14 files changed, 419 insertions(+), 329 deletions(-) create mode 100644 src/managed_secret.rs diff --git a/Cargo.lock b/Cargo.lock index 94520db67..44e627c28 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -147,6 +147,7 @@ dependencies = [ "sha2 0.10.9", "shell-util", "storage-util", + "subtle", "tar", "tempfile", "thiserror 2.0.18", diff --git a/Cargo.toml b/Cargo.toml index 62083f75a..b596b9aaf 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -107,6 +107,7 @@ io-uring = "0.7.9" tokio-tungstenite = "0.28" sha2 = "0.10" hmac = "0.12" +subtle = "2.6" hex = "0.4" semver = "1" iroh = { version = "=1.0.0-rc.0" } diff --git a/deploy/k8s/run.sh b/deploy/k8s/run.sh index e1e2ba892..93d9cdd63 100644 --- a/deploy/k8s/run.sh +++ b/deploy/k8s/run.sh @@ -11,6 +11,10 @@ shift KUBECTL_BIN="${KUBECTL:-kubectl}" OVERLAY_NAME="${K8S_OVERLAY:-default}" NAMESPACE="${K8S_NAMESPACE:-agentenv-system}" +if [[ ${#NAMESPACE} -gt 63 || ! "${NAMESPACE}" =~ ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ ]]; then + echo "K8S_NAMESPACE must be a valid Kubernetes namespace name" >&2 + exit 1 +fi SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" REPO_ROOT="$(cd "${SCRIPT_DIR}/../.." && pwd)" @@ -27,8 +31,36 @@ sed_in_place() { fi } +render_api_key() { + local file="$1" + local rendered_file + + rendered_file="$(mktemp "${TEMP_DIR}/api-key.XXXXXX")" + if ! { + printf '%s\n' "${API_KEY_VALUE}" + cat "${file}" + } | awk ' + NR == 1 { api_key = $0; next } + /^ - AENV_API_KEY=/ { print " - AENV_API_KEY=" api_key; replaced = 1; next } + { print } + END { if (!replaced) exit 1 } + ' >"${rendered_file}"; then + return 1 + fi + mv "${rendered_file}" "${file}" +} + cp -R "${SCRIPT_DIR}" "${TEMP_DIR}/k8s" cp "${REPO_ROOT}/config/default.toml" "${TEMP_DIR}/k8s/base/config/agentenv.toml" +OVERLAY_PATH="${TEMP_DIR}/k8s/overlays/${OVERLAY_NAME}" +if [[ ! -d "${OVERLAY_PATH}" ]]; then + echo "unknown overlay: ${OVERLAY_NAME}" >&2 + exit 1 +fi +sed_in_place "s#^namespace: agentenv-system#namespace: ${NAMESPACE}#" "${TEMP_DIR}/k8s/base/kustomization.yaml" +sed_in_place "s#^namespace: agentenv-system#namespace: ${NAMESPACE}#" "${OVERLAY_PATH}/kustomization.yaml" +sed_in_place "s# name: agentenv-system# name: ${NAMESPACE}#" "${TEMP_DIR}/k8s/base/namespace.yaml" +sed_in_place "s#\"namespace\": \"agentenv-system\"#\"namespace\": \"${NAMESPACE}\"#" "${TEMP_DIR}/k8s/base/config/scheduler.json" namespace_name="" if [[ "${MODE}" == "apply" ]]; then @@ -55,6 +87,11 @@ read_existing_api_key() { } if [[ "${MODE}" != "delete" ]]; then + restore_xtrace=0 + if [[ $- == *x* ]]; then + restore_xtrace=1 + set +x + fi API_KEY_VALUE="" if [[ "${AENV_API_KEY+x}" == "x" ]]; then API_KEY_VALUE="${AENV_API_KEY}" @@ -65,14 +102,13 @@ if [[ "${MODE}" != "delete" ]]; then if [[ -z "${API_KEY_VALUE}" ]]; then API_KEY_VALUE="e2b_$(od -An -N32 -tx1 /dev/urandom | tr -d '[:space:]')" fi - if [[ ! "${API_KEY_VALUE}" =~ ^[A-Za-z0-9._~-]{32,}$ ]]; then - echo "AENV_API_KEY must contain at least 32 URL-safe characters" >&2 + if [[ ! "${API_KEY_VALUE}" =~ ^[A-Za-z0-9._~-]{32,4096}$ ]]; then + echo "AENV_API_KEY must contain between 32 and 4096 URL-safe characters" >&2 exit 1 fi - sed_in_place \ - "s#- AENV_API_KEY=.*#- AENV_API_KEY=${API_KEY_VALUE}#" \ - "${TEMP_DIR}/k8s/base/kustomization.yaml" + render_api_key "${TEMP_DIR}/k8s/base/kustomization.yaml" + [[ "${restore_xtrace}" == "0" ]] || set -x fi if [[ "${SANDBOX_PROXY_DOMAINS+x}" == "x" ]]; then @@ -82,12 +118,6 @@ if [[ "${SANDBOX_PROXY_DOMAINS+x}" == "x" ]]; then sed_in_place "s#- SANDBOX_PROXY_DOMAINS=.*#- SANDBOX_PROXY_DOMAINS=${ESCAPED_SANDBOX_PROXY_DOMAINS}#" "${TEMP_DIR}/k8s/base/kustomization.yaml" fi -OVERLAY_PATH="${TEMP_DIR}/k8s/overlays/${OVERLAY_NAME}" -if [[ ! -d "${OVERLAY_PATH}" ]]; then - echo "unknown overlay: ${OVERLAY_NAME}" >&2 - exit 1 -fi - if [[ "${OVERLAY_NAME}" == "local-dev" ]]; then REPO_ENV_PATH="${AENV_LOCAL_REPO_ENV_PATH:-${REPO_ROOT}/env}" if [[ ! -d "${REPO_ENV_PATH}" ]]; then diff --git a/docs/src/configuration/authentication.md b/docs/src/configuration/authentication.md index 98d5aa689..4d4349f5a 100644 --- a/docs/src/configuration/authentication.md +++ b/docs/src/configuration/authentication.md @@ -96,7 +96,7 @@ export AENV_API_KEY="e2b_$(openssl rand -hex 32)" make start-server ``` -Custom keys must contain at least 32 URL-safe characters. In a multi-node +Custom keys must contain between 32 and 4096 URL-safe characters. In a multi-node deployment, use exactly the same value for the gateway and every runtime node. The generated keys use `e2b_` followed by hexadecimal characters so they pass the E2B SDK default API-key validation. Use that format for custom keys when diff --git a/scripts/tests/e2e/lib/runtime.sh b/scripts/tests/e2e/lib/runtime.sh index bdcf71596..3d51d31d5 100644 --- a/scripts/tests/e2e/lib/runtime.sh +++ b/scripts/tests/e2e/lib/runtime.sh @@ -400,7 +400,7 @@ if [[ -z "${E2E_RUNTIME_SH_LOADED:-}" ]]; then AENV_API_KEY="$(_compose_cmd exec -T agentenv-a cat /workspace/env/secrets/api-key)" || die "Failed to read the Compose deployment API key" - [[ "${AENV_API_KEY}" =~ ^[A-Za-z0-9._~-]{32,}$ ]] || + [[ "${AENV_API_KEY}" =~ ^[A-Za-z0-9._~-]{32,4096}$ ]] || die "Compose deployment returned an invalid API key" export AENV_API_KEY diff --git a/scripts/tests/e2e/suites/09_e2b_compat.sh b/scripts/tests/e2e/suites/09_e2b_compat.sh index 611b09ad6..b4898edaa 100755 --- a/scripts/tests/e2e/suites/09_e2b_compat.sh +++ b/scripts/tests/e2e/suites/09_e2b_compat.sh @@ -11,6 +11,7 @@ log "Suite: E2B Compatibility" export E2B_API_URL="${AENV_URL}" export E2B_SANDBOX_URL="${AENV_PROXY_URL}" export E2B_API_KEY="${AENV_API_KEY}" +unset E2B_ACCESS_TOKEN export E2B_COMPAT_USER_IMAGE="${E2B_COMPAT_USER_IMAGE:-${E2E_TEMPLATE_USER_IMAGE:-ghcr.io/linuxserver/baseimage-ubuntu:noble}}" cli_available=0 diff --git a/services/gateway/cmd/main.go b/services/gateway/cmd/main.go index 389920ffc..a0cfacbb7 100644 --- a/services/gateway/cmd/main.go +++ b/services/gateway/cmd/main.go @@ -5,6 +5,7 @@ import ( "errors" "flag" "fmt" + "io" "log" "net/http" "os" @@ -27,6 +28,8 @@ import ( const ( apiKeyEnv = "AENV_API_KEY" defaultAPIKeyPath = "/run/secrets/api-key" + maxAPIKeyLen = 4096 + maxAPIKeyFileLen = maxAPIKeyLen + 2 ) func newSchedulerConn(addr string) (*grpc.ClientConn, error) { @@ -45,22 +48,27 @@ func loadAPIKeyFrom(lookupEnv func(string) (string, bool), secretPath string) (s if explicit, present := lookupEnv(apiKeyEnv); present { value = explicit } else { - contents, err := os.ReadFile(secretPath) + file, err := os.Open(secretPath) if err != nil { if os.IsNotExist(err) { return "", fmt.Errorf("%s must be set or %s must exist", apiKeyEnv, secretPath) } return "", fmt.Errorf("read secret %s: %w", secretPath, err) } - value, source = string(contents), secretPath + defer file.Close() + contents, err := io.ReadAll(io.LimitReader(file, maxAPIKeyFileLen+1)) + if err != nil { + return "", fmt.Errorf("read secret %s: %w", secretPath, err) + } + value = strings.TrimSuffix(strings.TrimSuffix(string(contents), "\n"), "\r") + source = secretPath } return validateAPIKey(value, source) } func validateAPIKey(value, source string) (string, error) { - value = strings.TrimSpace(value) - if len(value) < 32 { - return "", fmt.Errorf("API key from %s must contain at least 32 URL-safe characters", source) + if len(value) < 32 || len(value) > maxAPIKeyLen { + return "", fmt.Errorf("API key from %s must contain between 32 and %d URL-safe characters", source, maxAPIKeyLen) } for _, char := range []byte(value) { if (char >= 'a' && char <= 'z') || @@ -69,7 +77,7 @@ func validateAPIKey(value, source string) (string, error) { char == '.' || char == '_' || char == '~' || char == '-' { continue } - return "", fmt.Errorf("API key from %s must contain at least 32 URL-safe characters", source) + return "", fmt.Errorf("API key from %s must contain between 32 and %d URL-safe characters", source, maxAPIKeyLen) } return value, nil } diff --git a/services/gateway/cmd/main_test.go b/services/gateway/cmd/main_test.go index 9064986b3..bc88faebe 100644 --- a/services/gateway/cmd/main_test.go +++ b/services/gateway/cmd/main_test.go @@ -12,7 +12,7 @@ const testAPIKey = "e2b_0123456789abcdef0123456789abcdef0123456789abcdef01234567 func TestValidateAPIKey(t *testing.T) { t.Parallel() - got, err := validateAPIKey(" "+testAPIKey+"\n", "test") + got, err := validateAPIKey(testAPIKey, "test") if err != nil { t.Fatalf("validateAPIKey() error = %v", err) } @@ -20,7 +20,7 @@ func TestValidateAPIKey(t *testing.T) { t.Fatalf("validateAPIKey() = %q, want %q", got, testAPIKey) } - for _, invalid := range []string{"", "too-short", strings.Repeat("a", 31), strings.Repeat("a", 31) + "!"} { + for _, invalid := range []string{"", "too-short", " " + testAPIKey, testAPIKey + "\n", strings.Repeat("a", 31), strings.Repeat("a", maxAPIKeyLen+1), strings.Repeat("a", 31) + "!"} { if _, err := validateAPIKey(invalid, "test"); err == nil { t.Errorf("validateAPIKey(%q) unexpectedly succeeded", invalid) } diff --git a/src/api/impls/auth.rs b/src/api/impls/auth.rs index 6b197c616..435ea1188 100644 --- a/src/api/impls/auth.rs +++ b/src/api/impls/auth.rs @@ -7,6 +7,7 @@ use axum::{ middleware::Next, response::{IntoResponse, Response}, }; +use subtle::ConstantTimeEq; use super::{ApiImpl, Claims}; use crate::{api::proxy, types::SandboxId}; @@ -23,8 +24,11 @@ fn single_header<'a>(headers: &'a HeaderMap, name: &str) -> Option<&'a HeaderVal impl ApiImpl { pub(crate) fn has_valid_api_key(&self, headers: &HeaderMap) -> bool { - single_header(headers, API_KEY_HEADER) - .is_some_and(|value| value.as_bytes() == self.api_key.as_bytes()) + single_header(headers, API_KEY_HEADER).is_some_and(|value| { + let candidate = value.as_bytes(); + let expected = self.api_key.as_bytes(); + candidate.len() == expected.len() && bool::from(candidate.ct_eq(expected)) + }) } pub(crate) fn traffic_access_token(&self, sandbox_id: SandboxId) -> String { @@ -43,7 +47,7 @@ impl ApiImpl { pub(crate) async fn require_auth( State(api_impl): State, - request: Request, + mut request: Request, next: Next, ) -> Response where @@ -57,23 +61,23 @@ where let api_impl = api_impl.as_ref(); let mut authorized = api_impl.has_valid_api_key(request.headers()); - if !authorized && proxy_request { + let mut envd_authorized = false; + if proxy_request { if let Some((sandbox_id, target_port)) = proxy::route_for_auth(&request, api_impl.sandbox_proxy_domains()) { - authorized = api_impl.has_valid_traffic_access_token(request.headers(), sandbox_id); + authorized |= api_impl.has_valid_traffic_access_token(request.headers(), sandbox_id); let envd_candidate = single_header(request.headers(), ENVD_ACCESS_TOKEN_HEADER) .and_then(|value| value.to_str().ok()); - if !authorized { - if let Some(candidate) = envd_candidate { - authorized = proxy::has_valid_envd_access_token( - api_impl, - sandbox_id, - target_port, - candidate, - ) - .await; - } + if let Some(candidate) = envd_candidate { + envd_authorized = proxy::has_valid_envd_access_token( + api_impl, + sandbox_id, + target_port, + candidate, + ) + .await; + authorized |= envd_authorized; } } } @@ -81,6 +85,9 @@ where if !authorized { return StatusCode::UNAUTHORIZED.into_response(); } + if !envd_authorized { + request.headers_mut().remove(ENVD_ACCESS_TOKEN_HEADER); + } next.run(request).await } diff --git a/src/api/proxy.rs b/src/api/proxy.rs index 76a8fdac4..d275ed4d3 100644 --- a/src/api/proxy.rs +++ b/src/api/proxy.rs @@ -56,8 +56,6 @@ struct ResolvedProxyRequest { sandbox_id: SandboxId, upstream_uri: Uri, original_host: Option, - target_port: u16, - envd_port: u16, } #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -149,10 +147,10 @@ where } pub(crate) fn route_for_auth(request: &Request, domains: &[String]) -> Option<(SandboxId, u16)> { - match parse_host_proxy_route(request_host(request), domains) { - Ok(Some(route)) => return Some((route.sandbox_id, route.target_port)), - Err(_) => return None, - Ok(None) => {} + if !has_proxy_prefix(request.uri().path()) { + if let Ok(Some(route)) = parse_host_proxy_route(request_host(request), domains) { + return Some((route.sandbox_id, route.target_port)); + } } Some(( @@ -181,7 +179,7 @@ pub(crate) async fn has_valid_envd_access_token( pub(crate) fn is_sandbox_proxy_request(request: &Request, domains: &[String]) -> bool { let path = request.uri().path(); - if path == PROXY_ROUTE || path.starts_with("/proxy/") { + if has_proxy_prefix(path) { return true; } @@ -216,7 +214,7 @@ where I: AsRef + Clone + Send + Sync + 'static, { let path = request.uri().path(); - if path == PROXY_ROUTE || path.starts_with("/proxy/") { + if has_proxy_prefix(path) { return next.run(request).await; } @@ -338,6 +336,10 @@ fn strip_proxy_prefix(path: &str) -> &str { path.strip_prefix(PROXY_ROUTE).unwrap_or("") } +fn has_proxy_prefix(path: &str) -> bool { + path == PROXY_ROUTE || path.starts_with("/proxy/") +} + fn parse_host_proxy_route( raw_host: Option<&str>, domains: &[String], @@ -437,11 +439,9 @@ async fn proxy_http_request( sandbox_id, upstream_uri, original_host, - target_port, - envd_port, } = resolved; - sanitize_request_headers(&mut parts.headers, target_port, envd_port); + sanitize_request_headers(&mut parts.headers); inject_forwarded_headers( &mut parts.headers, original_host.as_ref(), @@ -622,11 +622,9 @@ async fn proxy_websocket_request( sandbox_id, upstream_uri, original_host, - target_port, - envd_port, } = resolved; - sanitize_websocket_request_headers(&mut parts.headers, target_port, envd_port); + sanitize_websocket_request_headers(&mut parts.headers); inject_forwarded_headers( &mut parts.headers, original_host.as_ref(), @@ -807,14 +805,6 @@ async fn resolve_proxy_request( } }; - let metadata = api_impl - .orchestrator() - .get_sandbox(&sandbox_id) - .await - .map_err(|_| StatusCode::INTERNAL_SERVER_ERROR.into_response())? - .ok_or_else(|| proxy_error_response(&ProxyRequestError::SandboxNotFound(sandbox_id)))?; - let envd_port = effective_envd_port(&metadata); - let upstream_uri = if is_websocket_request { build_upstream_uri_with_scheme("ws", &target, target_port, proxy_path, parts.uri.query()) } else { @@ -826,8 +816,6 @@ async fn resolve_proxy_request( sandbox_id, upstream_uri, original_host: parts.headers.get(header::HOST).cloned(), - target_port, - envd_port, }) } @@ -1045,7 +1033,7 @@ fn build_upstream_uri_with_scheme( .map_err(|_| StatusCode::BAD_REQUEST) } -fn sanitize_request_headers(headers: &mut HeaderMap, target_port: u16, envd_port: u16) { +fn sanitize_request_headers(headers: &mut HeaderMap) { // These headers are only for the control-plane hop between the client and // AgentENV. Upstream sandbox services should not see them. headers.remove(SANDBOX_ID_HEADER); @@ -1055,14 +1043,11 @@ fn sanitize_request_headers(headers: &mut HeaderMap, target_port: u16, envd_port headers.remove(API_KEY_HEADER); headers.remove(TRAFFIC_ACCESS_TOKEN_HEADER); headers.remove(header::HOST); - if target_port != envd_port { - headers.remove(ENVD_ACCESS_TOKEN_HEADER); - } remove_hop_by_hop_headers(headers); } -fn sanitize_websocket_request_headers(headers: &mut HeaderMap, target_port: u16, envd_port: u16) { - sanitize_request_headers(headers, target_port, envd_port); +fn sanitize_websocket_request_headers(headers: &mut HeaderMap) { + sanitize_request_headers(headers); headers.remove(header::SEC_WEBSOCKET_ACCEPT); headers.remove(header::SEC_WEBSOCKET_EXTENSIONS); headers.remove(header::SEC_WEBSOCKET_KEY); @@ -1841,11 +1826,7 @@ mod tests { HeaderValue::from_static("keep"), ); - sanitize_request_headers( - &mut headers, - 8080, - ConfigManager::global_config().tools.control_plane_port, - ); + sanitize_request_headers(&mut headers); assert!(headers.get(SANDBOX_ID_HEADER).is_none()); assert!(headers.get(E2B_SANDBOX_ID_HEADER).is_none()); diff --git a/src/api_key.rs b/src/api_key.rs index ec59037e5..96ddc7048 100644 --- a/src/api_key.rs +++ b/src/api_key.rs @@ -1,6 +1,6 @@ use std::ffi::OsStr; use std::fs::{self, File}; -use std::io::{self, Write}; +use std::io::{self, Read}; use std::path::Path; use anyhow::{bail, Context, Result}; @@ -8,10 +8,13 @@ use rand::{rngs::SysRng, TryRng}; use tracing::info; use crate::cfg::AppConfig; +use crate::managed_secret::{self, CreateOutcome}; const API_KEY_ENV: &str = "AENV_API_KEY"; const EXTERNAL_API_KEY_PATH: &str = "/run/secrets/api-key"; const MANAGED_API_KEY_RELATIVE_PATH: &str = "secrets/api-key"; +const API_KEY_MAX_LEN: usize = 4096; +const API_KEY_FILE_MAX_LEN: usize = API_KEY_MAX_LEN + 2; const GENERATED_API_KEY_PREFIX: &str = "e2b_"; pub fn resolve(config: &AppConfig) -> Result { @@ -36,7 +39,7 @@ fn resolve_from( .context("invalid AENV_API_KEY"); } - match read(external_path) { + match read_external(external_path) { Ok(key) => { info!(path = %external_path.display(), "loaded API key from external secret"); return Ok(key); @@ -46,80 +49,77 @@ fn resolve_from( } let managed_path = home_path.join(MANAGED_API_KEY_RELATIVE_PATH); - match read(&managed_path) { - Ok(key) => return Ok(key), - Err(error) if error.kind() == io::ErrorKind::NotFound => {} - Err(error) => return Err(error).context("load managed API key"), + match fs::symlink_metadata(&managed_path) { + Err(error) if error.kind() == io::ErrorKind::NotFound => return create(&managed_path), + Err(error) => { + return Err(error) + .with_context(|| format!("inspect managed API key {}", managed_path.display())); + } + Ok(_) => {} + } + if let Some(value) = + managed_secret::read(&managed_path, API_KEY_FILE_MAX_LEN).context("load managed API key")? + { + return validate_file_contents(&value).context("invalid managed API key"); } create(&managed_path) } -fn read(path: &Path) -> Result { - let value = fs::read_to_string(path)?; - validate(&value).map_err(io::Error::other) +fn read_external(path: &Path) -> Result { + let value = read_bounded(File::open(path)?)?; + validate_file_contents(&value).map_err(io::Error::other) +} + +fn read_bounded(file: File) -> Result { + let mut value = String::with_capacity(API_KEY_FILE_MAX_LEN); + file.take((API_KEY_FILE_MAX_LEN + 1) as u64) + .read_to_string(&mut value)?; + if value.len() > API_KEY_FILE_MAX_LEN { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + format!("API key file must be at most {API_KEY_FILE_MAX_LEN} bytes"), + )); + } + Ok(value) } fn validate(value: &str) -> Result { - let value = value.trim(); - if value.len() < 32 + if !(32..=API_KEY_MAX_LEN).contains(&value.len()) || !value .bytes() .all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'.' | b'_' | b'~' | b'-')) { - bail!("API key must contain at least 32 URL-safe characters"); + bail!("API key must contain between 32 and {API_KEY_MAX_LEN} URL-safe characters"); } Ok(value.to_owned()) } -fn create(path: &Path) -> Result { - let parent = path - .parent() - .context("managed API key path has no parent")?; - fs::create_dir_all(parent) - .with_context(|| format!("create managed secret directory {}", parent.display()))?; - set_permissions(parent, 0o700)?; +fn validate_file_contents(value: &str) -> Result { + let value = value.strip_suffix('\n').unwrap_or(value); + validate(value.strip_suffix('\r').unwrap_or(value)) +} +fn create(path: &Path) -> Result { let mut random = [0_u8; 32]; SysRng .try_fill_bytes(&mut random) .context("generate managed API key")?; let key = format!("{GENERATED_API_KEY_PREFIX}{}", hex::encode(random)); - let mut temporary = tempfile::NamedTempFile::new_in(parent) - .with_context(|| format!("create temporary API key in {}", parent.display()))?; - set_permissions(temporary.path(), 0o600)?; - writeln!(temporary, "{key}")?; - temporary.as_file().sync_all()?; - - match temporary.persist_noclobber(path) { - Ok(_) => { - File::open(parent)?.sync_all()?; + match managed_secret::create(path, format!("{key}\n").as_bytes())? { + CreateOutcome::Created => { info!(path = %path.display(), "generated managed API key"); Ok(key) } - Err(error) if error.error.kind() == io::ErrorKind::AlreadyExists => { - read(path).context("load concurrently generated API key") - } - Err(error) => { - Err(error.error).with_context(|| format!("persist managed API key {}", path.display())) + CreateOutcome::Existing(file) => { + let value = managed_secret::read_file(path, file, API_KEY_FILE_MAX_LEN) + .context("load concurrently generated API key")?; + validate_file_contents(&value).context("invalid concurrently generated API key") } } } -#[cfg(unix)] -fn set_permissions(path: &Path, mode: u32) -> Result<()> { - use std::os::unix::fs::PermissionsExt; - - fs::set_permissions(path, fs::Permissions::from_mode(mode)) - .with_context(|| format!("set permissions on {}", path.display())) -} - -#[cfg(not(unix))] -fn set_permissions(_path: &Path, _mode: u32) -> Result<()> { - Ok(()) -} - #[cfg(test)] mod tests { use super::*; diff --git a/src/lib.rs b/src/lib.rs index 14d02f7e8..9223e1bfa 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -6,6 +6,7 @@ pub mod identity; pub mod image; mod local_store; pub mod logging; +mod managed_secret; pub mod observability; pub mod orchestrator; pub mod overlaybd; diff --git a/src/managed_secret.rs b/src/managed_secret.rs new file mode 100644 index 000000000..4c9f7b955 --- /dev/null +++ b/src/managed_secret.rs @@ -0,0 +1,233 @@ +#[cfg(unix)] +use std::fs::OpenOptions; +use std::fs::{self, File}; +use std::io::{self, Read, Write}; +use std::path::Path; + +use anyhow::{bail, Context, Result}; + +pub(crate) enum CreateOutcome { + Created, + Existing(File), +} + +pub(crate) fn read(path: &Path, max_len: usize) -> Result> { + let parent = path.parent().context("managed secret path has no parent")?; + match validate_directory(parent) { + Ok(()) => {} + Err(error) if error.kind() == io::ErrorKind::NotFound => return Ok(None), + Err(error) => { + return Err(error).with_context(|| { + format!("validate managed secret directory {}", parent.display()) + }); + } + } + + match open(path) { + Ok(file) => read_file(path, file, max_len).map(Some), + Err(error) if error.kind() == io::ErrorKind::NotFound => Ok(None), + Err(error) => Err(error).with_context(|| format!("open managed secret {}", path.display())), + } +} + +pub(crate) fn read_file(path: &Path, mut file: File, max_len: usize) -> Result { + let metadata = file + .metadata() + .with_context(|| format!("inspect managed secret {}", path.display()))?; + if !metadata.is_file() { + bail!("managed secret {} must be a regular file", path.display()); + } + + #[cfg(unix)] + { + use std::os::unix::fs::{MetadataExt, PermissionsExt}; + + let mode = metadata.permissions().mode() & 0o777; + if mode != 0o600 { + bail!( + "managed secret {} must have permissions 0600, found {mode:04o}", + path.display() + ); + } + let expected_uid = nix::unistd::Uid::effective().as_raw(); + if metadata.uid() != expected_uid { + bail!( + "managed secret {} must be owned by uid {expected_uid}, found uid {}", + path.display(), + metadata.uid() + ); + } + } + + if metadata.len() > max_len as u64 { + bail!( + "managed secret {} must be at most {max_len} bytes", + path.display() + ); + } + + let mut contents = String::with_capacity(max_len); + Read::by_ref(&mut file) + .take((max_len + 1) as u64) + .read_to_string(&mut contents) + .with_context(|| format!("read managed secret {}", path.display()))?; + if contents.len() > max_len { + bail!( + "managed secret {} must be at most {max_len} bytes", + path.display() + ); + } + Ok(contents) +} + +pub(crate) fn create(path: &Path, contents: &[u8]) -> Result { + ensure_supported()?; + let parent = path.parent().context("managed secret path has no parent")?; + create_directory(parent)?; + validate_directory_identity(parent).with_context(|| { + format!( + "validate managed secret directory ownership {}", + parent.display() + ) + })?; + set_permissions(parent, 0o700)?; + validate_directory(parent) + .with_context(|| format!("validate managed secret directory {}", parent.display()))?; + + let mut temporary = tempfile::NamedTempFile::new_in(parent) + .with_context(|| format!("create temporary secret in {}", parent.display()))?; + set_permissions(temporary.path(), 0o600)?; + temporary + .write_all(contents) + .with_context(|| format!("write temporary secret in {}", parent.display()))?; + temporary + .as_file() + .sync_all() + .with_context(|| format!("sync temporary secret in {}", parent.display()))?; + + match temporary.persist_noclobber(path) { + Ok(_) => { + File::open(parent) + .and_then(|directory| directory.sync_all()) + .with_context(|| format!("sync managed secret directory {}", parent.display()))?; + Ok(CreateOutcome::Created) + } + Err(error) if error.error.kind() == io::ErrorKind::AlreadyExists => { + validate_directory(parent).with_context(|| { + format!("validate managed secret directory {}", parent.display()) + })?; + open(path) + .map(CreateOutcome::Existing) + .with_context(|| format!("open managed secret {}", path.display())) + } + Err(error) => { + Err(error.error).with_context(|| format!("persist managed secret {}", path.display())) + } + } +} + +fn create_directory(path: &Path) -> Result<()> { + let mut builder = fs::DirBuilder::new(); + builder.recursive(true); + + #[cfg(unix)] + { + use std::os::unix::fs::DirBuilderExt; + + builder.mode(0o700); + } + + builder + .create(path) + .with_context(|| format!("create managed secret directory {}", path.display())) +} + +#[cfg(unix)] +fn open(path: &Path) -> io::Result { + let mut options = OpenOptions::new(); + options.read(true); + + use std::os::unix::fs::OpenOptionsExt; + + options.custom_flags(libc::O_NOFOLLOW | libc::O_NONBLOCK); + + options.open(path) +} + +#[cfg(not(unix))] +fn open(_path: &Path) -> io::Result { + Err(io::Error::new( + io::ErrorKind::Unsupported, + "managed secrets require Unix no-follow file semantics", + )) +} + +#[cfg(unix)] +fn ensure_supported() -> Result<()> { + Ok(()) +} + +#[cfg(not(unix))] +fn ensure_supported() -> Result<()> { + bail!("managed secrets require Unix no-follow file semantics") +} + +fn validate_directory(path: &Path) -> io::Result<()> { + let metadata = validate_directory_identity(path)?; + + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + + let mode = metadata.permissions().mode() & 0o777; + if mode != 0o700 { + return Err(io::Error::new( + io::ErrorKind::PermissionDenied, + format!("must have permissions 0700, found {mode:04o}"), + )); + } + } + + Ok(()) +} + +fn validate_directory_identity(path: &Path) -> io::Result { + let metadata = fs::symlink_metadata(path)?; + if metadata.file_type().is_symlink() || !metadata.is_dir() { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + "must be a directory and not a symbolic link", + )); + } + + #[cfg(unix)] + { + use std::os::unix::fs::MetadataExt; + + let expected_uid = nix::unistd::Uid::effective().as_raw(); + if metadata.uid() != expected_uid { + return Err(io::Error::new( + io::ErrorKind::PermissionDenied, + format!( + "must be owned by uid {expected_uid}, found uid {}", + metadata.uid() + ), + )); + } + } + + Ok(metadata) +} + +#[cfg(unix)] +fn set_permissions(path: &Path, mode: u32) -> Result<()> { + use std::os::unix::fs::PermissionsExt; + + fs::set_permissions(path, fs::Permissions::from_mode(mode)) + .with_context(|| format!("set permissions on {}", path.display())) +} + +#[cfg(not(unix))] +fn set_permissions(_path: &Path, _mode: u32) -> Result<()> { + Ok(()) +} diff --git a/src/sandbox/access.rs b/src/sandbox/access.rs index abf13788e..b6de8b0d4 100644 --- a/src/sandbox/access.rs +++ b/src/sandbox/access.rs @@ -1,6 +1,4 @@ use std::fmt; -use std::fs::{self, File, OpenOptions}; -use std::io::{self, Read, Write}; use std::path::Path; use anyhow::{bail, Context, Result}; @@ -10,6 +8,7 @@ use sha2::Sha256; use tracing::{info, warn}; use crate::cfg::AppConfig; +use crate::managed_secret::{self, CreateOutcome}; use crate::types::SandboxId; type HmacSha256 = Hmac; @@ -116,30 +115,8 @@ fn validate_explicit_seed(seed: &str) -> Result<&str> { } fn resolve_seed(managed_path: &Path, managed_seed_must_exist: bool) -> Result { - let parent = managed_path - .parent() - .context("managed sandbox access-token seed path has no parent")?; - match validate_managed_seed_directory(parent) { - Ok(()) => {} - Err(error) if error.kind() == io::ErrorKind::NotFound => {} - Err(error) => { - return Err(error).with_context(|| { - format!("validate managed secret directory {}", parent.display()) - }); - } - } - - match open_managed_seed(managed_path) { - Ok(file) => return read_managed_seed(managed_path, file), - Err(error) if error.kind() == io::ErrorKind::NotFound => {} - Err(error) => { - return Err(error).with_context(|| { - format!( - "open managed sandbox access-token seed {}", - managed_path.display() - ) - }); - } + if let Some(contents) = managed_secret::read(managed_path, MANAGED_SEED_FILE_MAX_LEN)? { + return validate_managed_seed(managed_path, &contents); } if managed_seed_must_exist { @@ -152,80 +129,8 @@ fn resolve_seed(managed_path: &Path, managed_seed_must_exist: bool) -> Result io::Result { - let mut options = OpenOptions::new(); - options.read(true); - - #[cfg(unix)] - { - use std::os::unix::fs::OpenOptionsExt; - - options.custom_flags(libc::O_NOFOLLOW | libc::O_NONBLOCK); - } - - options.open(path) -} - -fn validate_managed_seed_file(path: &Path, file: &File) -> Result { - let metadata = file.metadata().with_context(|| { - format!( - "inspect managed sandbox access-token seed {}", - path.display() - ) - })?; - if !metadata.is_file() { - bail!( - "managed sandbox access-token seed {} must be a regular file", - path.display() - ); - } - - #[cfg(unix)] - { - use std::os::unix::fs::{MetadataExt, PermissionsExt}; - - let mode = metadata.permissions().mode() & 0o777; - if mode != 0o600 { - bail!( - "managed sandbox access-token seed {} must have permissions 0600, found {mode:04o}", - path.display() - ); - } - let expected_uid = nix::unistd::Uid::effective().as_raw(); - if metadata.uid() != expected_uid { - bail!( - "managed sandbox access-token seed {} must be owned by uid {expected_uid}, found uid {}", - path.display(), - metadata.uid() - ); - } - } - - Ok(metadata) -} - -fn read_managed_seed(path: &Path, mut file: File) -> Result { - let metadata = validate_managed_seed_file(path, &file)?; - - if metadata.len() > MANAGED_SEED_FILE_MAX_LEN as u64 { - bail!( - "managed sandbox access-token seed {} must be at most {MANAGED_SEED_FILE_MAX_LEN} bytes", - path.display() - ); - } - - let mut contents = String::with_capacity(MANAGED_SEED_FILE_MAX_LEN); - Read::by_ref(&mut file) - .take((MANAGED_SEED_FILE_MAX_LEN + 1) as u64) - .read_to_string(&mut contents) - .with_context(|| format!("read managed sandbox access-token seed {}", path.display()))?; - if contents.len() > MANAGED_SEED_FILE_MAX_LEN { - bail!( - "managed sandbox access-token seed {} must be at most {MANAGED_SEED_FILE_MAX_LEN} bytes", - path.display() - ); - } - let seed = contents.strip_suffix('\n').unwrap_or(&contents); +fn validate_managed_seed(path: &Path, contents: &str) -> Result { + let seed = contents.strip_suffix('\n').unwrap_or(contents); if !is_valid_managed_seed(seed) { bail!( "managed sandbox access-token seed {} must contain exactly {SEED_HEX_LEN} lowercase hexadecimal characters, optionally followed by a newline", @@ -237,57 +142,21 @@ fn read_managed_seed(path: &Path, mut file: File) -> Result { } fn create_managed_seed(path: &Path) -> Result { - let parent = path - .parent() - .context("managed sandbox access-token seed path has no parent")?; - fs::create_dir_all(parent) - .with_context(|| format!("create managed secret directory {}", parent.display()))?; - validate_managed_seed_directory_identity(parent).with_context(|| { - format!( - "validate managed secret directory ownership {}", - parent.display() - ) - })?; - set_permissions(parent, 0o700)?; - validate_managed_seed_directory(parent) - .with_context(|| format!("validate managed secret directory {}", parent.display()))?; - let mut random = [0_u8; MANAGED_SEED_BYTES]; SysRng .try_fill_bytes(&mut random) .context("generate managed sandbox access-token seed")?; let seed = hex::encode(random); - let mut temporary = tempfile::NamedTempFile::new_in(parent) - .with_context(|| format!("create temporary seed file in {}", parent.display()))?; - set_permissions(temporary.path(), 0o600)?; - writeln!(temporary, "{seed}") - .with_context(|| format!("write temporary seed file in {}", parent.display()))?; - temporary - .as_file() - .sync_all() - .with_context(|| format!("sync temporary seed file in {}", parent.display()))?; - - match temporary.persist_noclobber(path) { - Ok(_) => { - fs::File::open(parent) - .and_then(|directory| directory.sync_all()) - .with_context(|| format!("sync managed secret directory {}", parent.display()))?; + match managed_secret::create(path, format!("{seed}\n").as_bytes())? { + CreateOutcome::Created => { info!(path = %path.display(), "generated managed sandbox access-token seed"); Ok(seed) } - Err(error) if error.error.kind() == io::ErrorKind::AlreadyExists => { - let file = open_managed_seed(path).with_context(|| { - format!("open managed sandbox access-token seed {}", path.display()) - })?; - read_managed_seed(path, file) - } - Err(error) => Err(error.error).with_context(|| { - format!( - "persist managed sandbox access-token seed {}", - path.display() - ) - }), + CreateOutcome::Existing(file) => validate_managed_seed( + path, + &managed_secret::read_file(path, file, MANAGED_SEED_FILE_MAX_LEN)?, + ), } } @@ -298,66 +167,6 @@ fn is_valid_managed_seed(seed: &str) -> bool { .all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte)) } -fn validate_managed_seed_directory(path: &Path) -> io::Result<()> { - let metadata = validate_managed_seed_directory_identity(path)?; - - #[cfg(unix)] - { - use std::os::unix::fs::PermissionsExt; - - let mode = metadata.permissions().mode() & 0o777; - if mode != 0o700 { - return Err(io::Error::new( - io::ErrorKind::PermissionDenied, - format!("must have permissions 0700, found {mode:04o}"), - )); - } - } - - Ok(()) -} - -fn validate_managed_seed_directory_identity(path: &Path) -> io::Result { - let metadata = fs::symlink_metadata(path)?; - if metadata.file_type().is_symlink() || !metadata.is_dir() { - return Err(io::Error::new( - io::ErrorKind::InvalidInput, - "must be a directory and not a symbolic link", - )); - } - - #[cfg(unix)] - { - use std::os::unix::fs::MetadataExt; - - let expected_uid = nix::unistd::Uid::effective().as_raw(); - if metadata.uid() != expected_uid { - return Err(io::Error::new( - io::ErrorKind::PermissionDenied, - format!( - "must be owned by uid {expected_uid}, found uid {}", - metadata.uid() - ), - )); - } - } - - Ok(metadata) -} - -#[cfg(unix)] -fn set_permissions(path: &Path, mode: u32) -> Result<()> { - use std::os::unix::fs::PermissionsExt; - - fs::set_permissions(path, fs::Permissions::from_mode(mode)) - .with_context(|| format!("set permissions on {}", path.display())) -} - -#[cfg(not(unix))] -fn set_permissions(_path: &Path, _mode: u32) -> Result<()> { - Ok(()) -} - impl fmt::Debug for SandboxAccessTokenGenerator { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { f.write_str("SandboxAccessTokenGenerator()") @@ -367,13 +176,20 @@ impl fmt::Debug for SandboxAccessTokenGenerator { #[cfg(test)] mod tests { use super::*; + use std::fs; use std::sync::{Arc, Barrier}; use tempfile::TempDir; fn create_private_managed_seed_directory(path: &Path) -> Result<()> { fs::create_dir_all(path)?; - set_permissions(path, 0o700) + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + + fs::set_permissions(path, fs::Permissions::from_mode(0o700))?; + } + Ok(()) } #[test] @@ -505,7 +321,7 @@ mod tests { for contents in ["", "invalid\n"] { fs::write(&managed_path, contents)?; - set_permissions(&managed_path, 0o600)?; + set_test_permissions(&managed_path, 0o600)?; let error = resolve_seed(&managed_path, false).unwrap_err(); @@ -522,7 +338,7 @@ mod tests { let managed_path = temp.path().join(MANAGED_SEED_RELATIVE_PATH); create_private_managed_seed_directory(managed_path.parent().unwrap())?; fs::write(&managed_path, format!("{}\n", "a".repeat(64)))?; - set_permissions(&managed_path, 0o640)?; + set_test_permissions(&managed_path, 0o640)?; let error = resolve_seed(&managed_path, false).unwrap_err(); @@ -540,14 +356,12 @@ mod tests { let target_path = temp.path().join("seed-target"); create_private_managed_seed_directory(managed_path.parent().unwrap())?; fs::write(&target_path, format!("{}\n", "a".repeat(64)))?; - set_permissions(&target_path, 0o600)?; + set_test_permissions(&target_path, 0o600)?; symlink(&target_path, &managed_path)?; let error = resolve_seed(&managed_path, false).unwrap_err(); - assert!(error - .to_string() - .contains("open managed sandbox access-token seed")); + assert!(error.to_string().contains("open managed secret")); Ok(()) } @@ -556,9 +370,9 @@ mod tests { let temp = TempDir::new()?; let managed_path = temp.path().join(MANAGED_SEED_RELATIVE_PATH); create_private_managed_seed_directory(managed_path.parent().unwrap())?; - let file = File::create(&managed_path)?; + let file = fs::File::create(&managed_path)?; file.set_len(1024 * 1024)?; - set_permissions(&managed_path, 0o600)?; + set_test_permissions(&managed_path, 0o600)?; let error = resolve_seed(&managed_path, false).unwrap_err(); @@ -572,7 +386,7 @@ mod tests { let temp = TempDir::new()?; let managed_path = temp.path().join(MANAGED_SEED_RELATIVE_PATH); fs::create_dir_all(managed_path.parent().unwrap())?; - set_permissions(managed_path.parent().unwrap(), 0o770)?; + set_test_permissions(managed_path.parent().unwrap(), 0o770)?; let error = resolve_seed(&managed_path, false).unwrap_err(); @@ -608,4 +422,17 @@ mod tests { assert!(!managed_path.exists()); Ok(()) } + + #[cfg(unix)] + fn set_test_permissions(path: &Path, mode: u32) -> Result<()> { + use std::os::unix::fs::PermissionsExt; + + fs::set_permissions(path, fs::Permissions::from_mode(mode))?; + Ok(()) + } + + #[cfg(not(unix))] + fn set_test_permissions(_path: &Path, _mode: u32) -> Result<()> { + Ok(()) + } } From 52a66c71d476d59c8ff995dc8add241a00e12469 Mon Sep 17 00:00:00 2001 From: Yingdi Shan <5491399+yingdi-shan@users.noreply.github.com> Date: Sun, 16 Aug 2026 08:02:40 +0000 Subject: [PATCH 6/7] fix: enforce authentication boundaries --- crates/aenv/src/client/files.rs | 21 +- crates/aenv/src/client/mod.rs | 2 +- crates/aenv/src/grpc/mod.rs | 22 +- deploy/docker-compose.yml | 2 + deploy/k8s/run.sh | 140 +++++-- docs/src/concepts/proxy.md | 17 + docs/src/configuration/authentication.md | 145 ++----- docs/src/configuration/reference.md | 2 +- docs/src/deployment/kubernetes.md | 4 +- docs/src/integration/e2b.md | 13 +- .../persistence-artifact-inventory.md | 4 +- docs/src/internals/proxy-design.md | 29 +- docs/src/security/secure-sandboxes.md | 2 +- scripts/install.sh | 2 +- scripts/tests/e2e/lib/helpers.sh | 42 +- scripts/tests/e2e/lib/runtime.sh | 37 +- scripts/tests/e2e/lib/server.sh | 2 +- scripts/tests/e2e/suites/06_proxy.sh | 16 +- scripts/tests/e2e/suites/08_auth.sh | 105 ++++- scripts/tests/e2e/suites/11_node_metrics.sh | 22 +- services/README.md | 11 +- services/gateway/cmd/main.go | 24 +- services/gateway/cmd/main_test.go | 34 ++ services/gateway/internal/server.go | 60 ++- services/gateway/internal/server_test.go | 96 +++-- src/api/generated/src/models.rs | 2 +- src/api/impls/auth.rs | 73 +++- src/api/impls/sandbox.rs | 37 +- src/api/openapi.yml | 2 +- src/api/proxy.rs | 389 +++++++++--------- src/api_key.rs | 90 ++-- src/bin/server.rs | 4 +- src/managed_secret.rs | 108 +++-- src/orchestrator/service.rs | 24 +- src/orchestrator/tests.rs | 24 +- src/sandbox/access.rs | 20 +- src/sandbox/firecracker/sandbox.rs | 3 +- src/sandbox/network/policy.rs | 40 +- 38 files changed, 1034 insertions(+), 636 deletions(-) diff --git a/crates/aenv/src/client/files.rs b/crates/aenv/src/client/files.rs index ac5173ae5..39be84900 100644 --- a/crates/aenv/src/client/files.rs +++ b/crates/aenv/src/client/files.rs @@ -17,7 +17,6 @@ use super::Client; use crate::grpc::{RpcError, Transport, ENVD_PORT_STR}; use crate::progress::TransferProgress; -const API_KEY_HEADER: &str = "X-API-Key"; const SANDBOX_ID_HEADER: &str = "x-agentenv-sandbox-id"; const TARGET_PORT_HEADER: &str = "x-agentenv-target-port"; const ACCESS_TOKEN_HEADER: &str = "X-Access-Token"; @@ -32,17 +31,8 @@ pub struct EnvdFilesClient { } impl EnvdFilesClient { - fn new( - base_url: &str, - api_key: &str, - sandbox_id: &str, - envd_access_token: Option<&str>, - ) -> Result { + fn new(base_url: &str, sandbox_id: &str, envd_access_token: Option<&str>) -> Result { let mut headers = HeaderMap::new(); - headers.insert( - API_KEY_HEADER, - HeaderValue::from_str(api_key).context("invalid API key header value")?, - ); headers.insert( SANDBOX_ID_HEADER, HeaderValue::from_str(sandbox_id).context("invalid sandbox ID header value")?, @@ -72,7 +62,7 @@ impl EnvdFilesClient { Ok(Self { base_url: base_url.trim_end_matches('/').to_string(), http: client, - transport: Transport::new(base_url, api_key, sandbox_id, envd_access_token)?, + transport: Transport::new(base_url, sandbox_id, envd_access_token)?, }) } @@ -311,12 +301,7 @@ fn format_envd_response_error(status: reqwest::StatusCode, content: &str) -> any impl Client { pub fn files(&self, sandbox_id: &str) -> Result { let sandbox = self.get_sandbox(sandbox_id)?; - EnvdFilesClient::new( - &self.base, - &self.api_key, - sandbox_id, - sandbox.envd_access_token.as_deref(), - ) + EnvdFilesClient::new(&self.base, sandbox_id, sandbox.envd_access_token.as_deref()) } } diff --git a/crates/aenv/src/client/mod.rs b/crates/aenv/src/client/mod.rs index 55dba491e..c384e9648 100644 --- a/crates/aenv/src/client/mod.rs +++ b/crates/aenv/src/client/mod.rs @@ -40,7 +40,7 @@ impl Client { sandbox_id: &str, envd_access_token: Option<&str>, ) -> Result { - Transport::new(&self.base, &self.api_key, sandbox_id, envd_access_token) + Transport::new(&self.base, sandbox_id, envd_access_token) } fn url(&self, path: &str) -> String { diff --git a/crates/aenv/src/grpc/mod.rs b/crates/aenv/src/grpc/mod.rs index 64336c333..cdc24e83b 100644 --- a/crates/aenv/src/grpc/mod.rs +++ b/crates/aenv/src/grpc/mod.rs @@ -54,23 +54,16 @@ impl std::error::Error for RpcError {} pub struct Transport { http: HttpClient, base_url: String, - api_key: String, sandbox_id: String, envd_access_token: Option, } impl Transport { - pub fn new( - base_url: &str, - api_key: &str, - sandbox_id: &str, - envd_access_token: Option<&str>, - ) -> Result { + pub fn new(base_url: &str, sandbox_id: &str, envd_access_token: Option<&str>) -> Result { let http = Self::http_client(base_url).context("building Connect-RPC HTTP client")?; Ok(Self { http, base_url: base_url.trim_end_matches('/').to_string(), - api_key: api_key.to_string(), sandbox_id: sandbox_id.to_string(), envd_access_token: envd_access_token.map(str::to_owned), }) @@ -102,7 +95,6 @@ impl Transport { fn auth(&self, builder: reqwest::RequestBuilder) -> reqwest::RequestBuilder { let builder = builder - .header("X-API-Key", &self.api_key) .header("x-agentenv-sandbox-id", &self.sandbox_id) .header("x-agentenv-target-port", ENVD_PORT_STR) .header("Connect-Protocol-Version", "1"); @@ -508,7 +500,7 @@ mod tests { #[test] fn unary_user_is_sent_as_basic_auth() { - let transport = Transport::new("http://127.0.0.1", "api-key", "sandbox-id", None).unwrap(); + let transport = Transport::new("http://127.0.0.1", "sandbox-id", None).unwrap(); let request = transport .unary_request("filesystem.Filesystem", "Stat", Some("app")) .build() @@ -518,6 +510,7 @@ mod tests { request.headers().get(AUTHORIZATION).unwrap(), "Basic YXBwOg==" ); + assert!(!request.headers().contains_key("x-api-key")); let request = transport .unary_request("filesystem.Filesystem", "Stat", None) @@ -528,13 +521,8 @@ mod tests { #[test] fn envd_access_token_is_sent_on_connect_requests() { - let transport = Transport::new( - "http://127.0.0.1", - "api-key", - "sandbox-id", - Some("envd-token"), - ) - .unwrap(); + let transport = + Transport::new("http://127.0.0.1", "sandbox-id", Some("envd-token")).unwrap(); let request = transport .unary_request("process.Process", "List", None) diff --git a/deploy/docker-compose.yml b/deploy/docker-compose.yml index 8f60d7bef..7f06bc38f 100644 --- a/deploy/docker-compose.yml +++ b/deploy/docker-compose.yml @@ -3,6 +3,7 @@ x-agentenv-base: &agentenv-base init: true working_dir: /workspace environment: &agentenv-environment + AENV_API_KEY: AENV_CONFIG_PATH: /workspace/config/default.toml AENV_VIRTUALIZATION_MODE: ${AENV_VIRTUALIZATION_MODE:-kvm} API_ADDR: 0.0.0.0:8000 @@ -72,6 +73,7 @@ services: - ./docker/config/default.json:/config/default.json:ro - agentenv-auth:/run/secrets:ro environment: + AENV_API_KEY: GATEWAY_HTTP_LISTEN_ADDR: :8080 GATEWAY_SCHEDULER_ADDR: scheduler:9090 GATEWAY_SANDBOX_PROXY_DOMAINS: ${SANDBOX_PROXY_DOMAINS:-} diff --git a/deploy/k8s/run.sh b/deploy/k8s/run.sh index 93d9cdd63..940384fe4 100644 --- a/deploy/k8s/run.sh +++ b/deploy/k8s/run.sh @@ -8,6 +8,14 @@ fi MODE="$1" shift +case "${MODE}" in + render|apply|delete) ;; + *) + echo "unsupported mode: ${MODE}" >&2 + exit 1 + ;; +esac + KUBECTL_BIN="${KUBECTL:-kubectl}" OVERLAY_NAME="${K8S_OVERLAY:-default}" NAMESPACE="${K8S_NAMESPACE:-agentenv-system}" @@ -16,6 +24,33 @@ if [[ ${#NAMESPACE} -gt 63 || ! "${NAMESPACE}" =~ ^[a-z0-9]([-a-z0-9]*[a-z0-9])? exit 1 fi +KUBECTL_TARGET_ARGS=() +DRY_RUN=0 +ARGS=("$@") +for ((i = 0; i < ${#ARGS[@]}; i++)); do + arg="${ARGS[i]}" + case "${arg}" in + --context|--kubeconfig) + if ((i + 1 >= ${#ARGS[@]})); then + echo "${arg} requires a value" >&2 + exit 1 + fi + KUBECTL_TARGET_ARGS+=("${arg}" "${ARGS[i + 1]}") + i=$((i + 1)) + ;; + --context=*|--kubeconfig=*) KUBECTL_TARGET_ARGS+=("${arg}") ;; + --dry-run) + if ((i + 1 < ${#ARGS[@]})) && [[ "${ARGS[i + 1]}" == "none" ]]; then + DRY_RUN=0 + else + DRY_RUN=1 + fi + ;; + --dry-run=client|--dry-run=server) DRY_RUN=1 ;; + --dry-run=none) DRY_RUN=0 ;; + esac +done + SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" REPO_ROOT="$(cd "${SCRIPT_DIR}/../.." && pwd)" TEMP_DIR="$(mktemp -d)" @@ -62,27 +97,56 @@ sed_in_place "s#^namespace: agentenv-system#namespace: ${NAMESPACE}#" "${OVERLAY sed_in_place "s# name: agentenv-system# name: ${NAMESPACE}#" "${TEMP_DIR}/k8s/base/namespace.yaml" sed_in_place "s#\"namespace\": \"agentenv-system\"#\"namespace\": \"${NAMESPACE}\"#" "${TEMP_DIR}/k8s/base/config/scheduler.json" -namespace_name="" -if [[ "${MODE}" == "apply" ]]; then - if ! namespace_name="$("${KUBECTL_BIN}" get namespace "${NAMESPACE}" --ignore-not-found -o name)"; then - echo "failed to check namespace ${NAMESPACE}" >&2 - exit 1 +read_existing_api_key() { + local value="" + + if ! value="$("${KUBECTL_BIN}" "${KUBECTL_TARGET_ARGS[@]}" -n "${NAMESPACE}" get secret agentenv-auth \ + --ignore-not-found -o 'go-template={{index .data "AENV_API_KEY" | base64decode}}')"; then + echo "failed to read AENV_API_KEY from Secret ${NAMESPACE}/agentenv-auth" >&2 + return 1 fi -fi + printf '%s' "${value}" +} -read_existing_api_key() { - local encoded_value="" +ensure_namespace() { + if ! "${KUBECTL_BIN}" "${KUBECTL_TARGET_ARGS[@]}" create namespace "${NAMESPACE}" \ + --dry-run=client -o yaml | "${KUBECTL_BIN}" "${KUBECTL_TARGET_ARGS[@]}" apply -f - >/dev/null; then + echo "failed to create or verify namespace ${NAMESPACE}" >&2 + return 1 + fi +} + +generate_api_key() { + printf 'e2b_%s' "$(od -An -N32 -tx1 /dev/urandom | tr -d '[:space:]')" +} - if [[ -z "${namespace_name}" ]]; then +bootstrap_api_key() { + local create_error secret_file + + if ! API_KEY_VALUE="$(read_existing_api_key)"; then + return 1 + fi + if [[ -n "${API_KEY_VALUE}" ]]; then return 0 fi - if ! encoded_value="$("${KUBECTL_BIN}" -n "${NAMESPACE}" get secret agentenv-auth \ - --ignore-not-found -o 'go-template={{index .data "AENV_API_KEY"}}')"; then - echo "failed to read AENV_API_KEY from Secret ${NAMESPACE}/agentenv-auth" >&2 + + secret_file="${TEMP_DIR}/bootstrap-api-key" + create_error="${TEMP_DIR}/bootstrap-api-key.err" + generate_api_key >"${secret_file}" + chmod 0600 "${secret_file}" + + # A concurrent apply can win this create. The persisted reread below is + # authoritative whether this command succeeds or reports AlreadyExists. + "${KUBECTL_BIN}" "${KUBECTL_TARGET_ARGS[@]}" -n "${NAMESPACE}" create secret generic agentenv-auth \ + --from-file="AENV_API_KEY=${secret_file}" >/dev/null 2>"${create_error}" || true + + if ! API_KEY_VALUE="$(read_existing_api_key)"; then return 1 fi - if [[ -n "${encoded_value}" ]]; then - printf '%s' "${encoded_value}" | base64 -d + if [[ -z "${API_KEY_VALUE}" ]]; then + cat "${create_error}" >&2 + echo "failed to bootstrap Secret ${NAMESPACE}/agentenv-auth" >&2 + return 1 fi } @@ -93,18 +157,25 @@ if [[ "${MODE}" != "delete" ]]; then set +x fi API_KEY_VALUE="" - if [[ "${AENV_API_KEY+x}" == "x" ]]; then - API_KEY_VALUE="${AENV_API_KEY}" - elif ! API_KEY_VALUE="$(read_existing_api_key)"; then - exit 1 - fi - - if [[ -z "${API_KEY_VALUE}" ]]; then - API_KEY_VALUE="e2b_$(od -An -N32 -tx1 /dev/urandom | tr -d '[:space:]')" - fi - if [[ ! "${API_KEY_VALUE}" =~ ^[A-Za-z0-9._~-]{32,4096}$ ]]; then - echo "AENV_API_KEY must contain between 32 and 4096 URL-safe characters" >&2 - exit 1 + if [[ "${MODE}" == "render" ]]; then + API_KEY_VALUE="REDACTED" + else + if [[ "${AENV_API_KEY+x}" == "x" ]]; then + if [[ -z "${AENV_API_KEY}" ]]; then + echo "AENV_API_KEY must not be empty" >&2 + exit 1 + fi + API_KEY_VALUE="${AENV_API_KEY}" + elif [[ "${DRY_RUN}" == "1" ]]; then + API_KEY_VALUE="$(generate_api_key)" + else + ensure_namespace || exit 1 + bootstrap_api_key || exit 1 + fi + if [[ ! "${API_KEY_VALUE}" =~ ^[A-Za-z0-9._~-]{32,4096}$ ]]; then + echo "AENV_API_KEY must contain between 32 and 4096 URL-safe characters" >&2 + exit 1 + fi fi render_api_key "${TEMP_DIR}/k8s/base/kustomization.yaml" @@ -141,15 +212,18 @@ case "${MODE}" in ;; apply) "${KUBECTL_BIN}" apply -k "${OVERLAY_PATH}" "$@" - echo "AgentENV API key stored in Secret ${NAMESPACE}/agentenv-auth." >&2 - echo "Read it with:" >&2 - echo " ${KUBECTL_BIN} -n ${NAMESPACE} get secret agentenv-auth -o go-template='{{index .data \"AENV_API_KEY\" | base64decode}}{{\"\\n\"}}'" >&2 + if [[ "${DRY_RUN}" == "0" ]]; then + "${KUBECTL_BIN}" "${KUBECTL_TARGET_ARGS[@]}" -n "${NAMESPACE}" rollout restart \ + deployment/agentenv-gateway daemonset/agentenv-node + echo "AgentENV API key stored in Secret ${NAMESPACE}/agentenv-auth." >&2 + echo "Read it with:" >&2 + printf ' %q' "${KUBECTL_BIN}" "${KUBECTL_TARGET_ARGS[@]}" >&2 + printf " -n %q get secret agentenv-auth -o go-template='%s'\n" \ + "${NAMESPACE}" \ + '{{index .data "AENV_API_KEY" | base64decode}}{{"\n"}}' >&2 + fi ;; delete) "${KUBECTL_BIN}" delete --ignore-not-found -k "${OVERLAY_PATH}" "$@" ;; - *) - echo "unsupported mode: ${MODE}" >&2 - exit 1 - ;; esac diff --git a/docs/src/concepts/proxy.md b/docs/src/concepts/proxy.md index 20ebff43a..aca0824bd 100644 --- a/docs/src/concepts/proxy.md +++ b/docs/src/concepts/proxy.md @@ -33,6 +33,23 @@ E2B-compatible aliases are also accepted: These routing headers are stripped before the request is forwarded to the sandbox. +## Access Control + +Proxy authentication is independent from AgentENV API authentication: + +- Public application ingress (`allowPublicTraffic: true`, the default) requires + no AgentENV credential. +- Private application ingress (`allowPublicTraffic: false`) requires the + sandbox's `trafficAccessToken` in `e2b-traffic-access-token`. +- Secure envd traffic requires the sandbox's `envdAccessToken` in + `X-Access-Token`. Insecure envd traffic has no envd token. + +`X-API-Key` authenticates AgentENV control-plane APIs only. It does not grant +access to private application ingress or secure envd. A matching platform key +is stripped on proxy requests; other `X-API-Key` values remain available to +sandbox applications. AgentENV also strips the traffic token, and forwards +`X-Access-Token` only to the matching secure envd port. + Host-based proxy requests derive both values from `Host`, for example `http://8080-.sandbox.example.com/health` targets port `8080`. The configured domain must route to the AgentENV server in single-node mode or diff --git a/docs/src/configuration/authentication.md b/docs/src/configuration/authentication.md index 4d4349f5a..c8e0589b2 100644 --- a/docs/src/configuration/authentication.md +++ b/docs/src/configuration/authentication.md @@ -1,124 +1,61 @@ # Authentication AgentENV uses one shared API key for a single-tenant deployment. The gateway -and every runtime node in a cluster must resolve the same key. - -Clients authenticate API requests with: - -```text -X-API-Key: -``` - -`Authorization`, `X-Admin-Token`, and `X-Team-ID` do not authenticate -AgentENV. The `Authorization` header is left unchanged when a request is -proxied into a sandbox, so applications inside a sandbox can use it normally. -`GET /health` is public for load balancer and container health checks. - -E2B SDK users set `E2B_API_KEY` to the same value. Sandbox create responses -include an independent `trafficAccessToken`; send it as -`e2b-traffic-access-token` on application proxy requests. The token is scoped to -the sandbox and is not accepted for control-plane API calls. - -For secure sandboxes, `envdAccessToken` is a separate credential for envd -control traffic and must be sent as `X-Access-Token` only when targeting the -envd control-plane port. It is absent for insecure sandboxes. - -Both sandbox credentials are derived from the sandbox ID and one independent -`AENV_SANDBOX_ACCESS_TOKEN_HASH_SEED`. They are not derived from the API key. +and every runtime node must use the same value. + +| Credential | Scope | Header | +|---|---|---| +| API key | AgentENV lifecycle and management APIs | `X-API-Key` | +| `trafficAccessToken` | Application ingress when `allowPublicTraffic` is `false` | `e2b-traffic-access-token` | +| `envdAccessToken` | Direct envd access for secure sandboxes | `X-Access-Token` | + +The credentials are not interchangeable. Public application ingress and envd +in insecure sandboxes need no AgentENV credential. `Authorization` remains an +application header and does not authenticate AgentENV. On sandbox routes, +`X-API-Key` is also treated as application data unless it exactly matches the +AgentENV API key, in which case it is removed to avoid forwarding the platform +credential. + +`GET /health` and node `GET /metrics` are outside API-key authentication. The +gateway exposes Prometheus metrics on its separate metrics listener. Protect +these endpoints with the network and authentication controls used by your +Prometheus deployment. They are distinct from E2B's authenticated sandbox +metrics API. + +E2B SDK users set `E2B_API_KEY` to the AgentENV API key. Sandbox credentials +are derived from the sandbox ID and +`AENV_SANDBOX_ACCESS_TOKEN_HASH_SEED`, independently of the API key. ## Key Resolution -On normal startup, a runtime node uses the first available source: +A runtime node checks these sources in order: 1. `AENV_API_KEY` 2. `/run/secrets/api-key` 3. `$AENV_HOME/secrets/api-key` -If neither an environment value nor an external secret exists, the server -generates a 256-bit key and atomically stores it in the managed path with -`0600` permissions. It reuses that key on later starts. Dependency and host -setup modes do not create a key. - -The gateway uses `AENV_API_KEY` or `/run/secrets/api-key`; it never generates a -key. Runtime nodes validate sandbox-scoped tokens, so the gateway does not need -the sandbox seed. - -## Installation Methods - -For a native installation, start the service once and read the managed key: - -```bash -sudo cat /var/lib/aenv/secrets/api-key -``` - -When upgrading an installation that already has `AENV_API_KEY` in -`/etc/default/aenv`, the installer preserves that entry and the server keeps -using it. Fresh installations leave key creation to the server. - -For a single Docker container, no auth volume is required. The server creates -the key in its writable container layer: +If none exists, normal server startup generates and atomically stores a key at +the managed path. The gateway checks only the first two sources and never +generates a key. -```bash -docker exec aenv-server cat /workspace/env/secrets/api-key -``` - -Removing the container removes this generated key. Supply an explicit key or -mount a secret at `/run/secrets/api-key` when it must remain stable across -container replacements. - -The checked-in Compose deployment mounts one named volume read-write on both -runtime nodes and read-only at `/run/secrets` on the gateway. Concurrent node -startup is safe: atomic creation makes both nodes converge on the same key and -sandbox seed. The gateway reads only the API key from that volume. Read it with: - -```bash -docker compose -f deploy/docker-compose.yml exec -T agentenv-a \ - cat /workspace/env/secrets/api-key -``` - -`docker compose down` preserves the key. `docker compose down -v` removes the -auth volume, so the next startup generates a new key. - -`make k8s-apply` creates `Secret/agentenv-auth` with an API key on the first -apply, then reuses it. Read the key with: - -```bash -kubectl -n agentenv-system get secret agentenv-auth \ - -o go-template='{{index .data "AENV_API_KEY" | base64decode}}{{"\n"}}' -``` - -For a single-node manual build, start the server and read -`$AENV_HOME/secrets/api-key`. To provide your own key instead, export it before -startup: +Custom keys must contain 32 to 4096 URL-safe characters. Generated keys use an +E2B-compatible `e2b_` prefix. For example: ```bash export AENV_API_KEY="e2b_$(openssl rand -hex 32)" -make start-server ``` -Custom keys must contain between 32 and 4096 URL-safe characters. In a multi-node -deployment, use exactly the same value for the gateway and every runtime node. -The generated keys use `e2b_` followed by hexadecimal characters so they pass -the E2B SDK default API-key validation. Use that format for custom keys when -you need E2B SDK compatibility. - -Docker Compose secrets can supply a pre-existing key without another AgentENV -configuration variable. In an override file, define a file-backed secret and -mount it with `target: api-key` on the gateway and every runtime node. Compose -then exposes the standard `/run/secrets/api-key` path. Compose secret sources -must already exist, so the named-volume setup remains the zero-configuration -default that allows Rust to generate the key during startup. - -## Transport Security +Docker Compose shares one managed-secret volume between runtime nodes and +mounts it read-only on the gateway. Kubernetes stores the key in +`Secret/agentenv-auth`. See the corresponding deployment guide for commands to +read or supply those values. Multi-node deployments must also share one +`AENV_SANDBOX_ACCESS_TOKEN_HASH_SEED` across runtime nodes. -API key authentication does not encrypt HTTP traffic. Do not send the key over -an untrusted plaintext network. Keep AgentENV on loopback or a trusted private -network, use a VPN, or terminate HTTPS at a reverse proxy or load balancer. +## Security and Rotation -## Rotation +API-key authentication does not encrypt traffic. Use HTTPS termination, a VPN, +loopback, or a trusted private network. -Changing `AENV_API_KEY` invalidates existing client API credentials without -changing sandbox credentials; apply it to the gateway and every runtime node -together. Changing -`AENV_SANDBOX_ACCESS_TOKEN_HASH_SEED` rotates both `trafficAccessToken` and -`envdAccessToken` values and must be changed on every runtime node together. +Changing `AENV_API_KEY` invalidates API clients without changing sandbox +credentials. Changing `AENV_SANDBOX_ACCESS_TOKEN_HASH_SEED` rotates both +sandbox token types. Apply either change to all relevant processes together. diff --git a/docs/src/configuration/reference.md b/docs/src/configuration/reference.md index 6f6465763..7c4b609d4 100644 --- a/docs/src/configuration/reference.md +++ b/docs/src/configuration/reference.md @@ -259,7 +259,7 @@ Sandbox control communication settings. |-----|------|---------|-------------| | `access_token_hash_seed` | string | auto-generated | Optional override for the secret used to derive sandbox envd and traffic access tokens. When unset, normal server startup creates and reuses `$AENV_HOME/secrets/sandbox-access-token-hash-seed`. Configure an explicit shared value for clustered deployments. | -The managed seed is node-local persistent state and must be included in backups of `$AENV_HOME`. AgentENV refuses to generate a replacement when persisted sandboxes exist. An explicit environment or TOML value takes precedence over the managed file; changing that effective value invalidates existing sandbox access tokens. +The managed seed is node-local persistent state and must be included in backups of `$AENV_HOME`. AgentENV refuses to generate a replacement when persisted secure or private-ingress sandboxes exist. An explicit environment or TOML value takes precedence over the managed file; changing that effective value invalidates existing sandbox access tokens. Configure `AENV_SANDBOX_ACCESS_TOKEN_HASH_SEED` with the same value on every runtime node in a clustered deployment. Standalone runtime nodes use their managed seed when it is unset. diff --git a/docs/src/deployment/kubernetes.md b/docs/src/deployment/kubernetes.md index f3133a516..63c058549 100644 --- a/docs/src/deployment/kubernetes.md +++ b/docs/src/deployment/kubernetes.md @@ -71,8 +71,8 @@ kubectl -n agentenv-system get secret agentenv-auth \ ``` Set `AENV_API_KEY` when applying to supply your own value. A standalone -`make k8s-render` uses a temporary generated value because it does not modify or -read cluster state. The optional runtime seed keeps its existing +`make k8s-render` uses an invalid `REDACTED` placeholder so preview output never +contains a deployable API key. The optional runtime seed keeps its existing `agentenv-runtime-secrets` contract described in [Secure Sandboxes](../security/secure-sandboxes.md). diff --git a/docs/src/integration/e2b.md b/docs/src/integration/e2b.md index 653f30371..619bd958e 100644 --- a/docs/src/integration/e2b.md +++ b/docs/src/integration/e2b.md @@ -15,12 +15,13 @@ export E2B_SANDBOX_URL=${E2B_API_URL} export E2B_API_KEY=${AENV_API_KEY} ``` -No `E2B_ACCESS_TOKEN` is needed. AgentENV returns `trafficAccessToken` for -application proxy traffic and (for secure sandboxes) `envdAccessToken` for envd -control traffic. These credentials have different headers and trust boundaries: -use `e2b-traffic-access-token` for application routes and `X-Access-Token` only -for envd. This is transport data, not the deprecated user-supplied -`E2B_ACCESS_TOKEN`. +No `E2B_ACCESS_TOKEN` is needed. AgentENV returns `trafficAccessToken` when +`network.allowPublicTraffic` is false and (for secure sandboxes) +`envdAccessToken` for envd control traffic. These credentials have different +headers and trust boundaries: use `e2b-traffic-access-token` for private +application routes and `X-Access-Token` only for envd. Public application +routes require neither token. This is transport data, not the deprecated +user-supplied `E2B_ACCESS_TOKEN`. ### TypeScript SDK diff --git a/docs/src/internals/persistence-artifact-inventory.md b/docs/src/internals/persistence-artifact-inventory.md index 8cf259048..e1291be90 100644 --- a/docs/src/internals/persistence-artifact-inventory.md +++ b/docs/src/internals/persistence-artifact-inventory.md @@ -9,7 +9,7 @@ This document lists AgentENV artifacts that can remain on disk or in object stor | `home_path` | `/var/lib/aenv` | `src/cfg.rs` | Base for paths containing the literal `$AENV_HOME` placeholder. `AENV_HOME_PATH` overrides it before placeholder expansion. | | `runtime_path` | `/run/aenv` | `src/cfg.rs`, `src/sandbox/network/*` | Base for transient namespace mount points and daemon sockets. `AENV_RUNTIME_PATH` overrides it. | | `deps_path` | `$AENV_HOME/deps` | `src/cfg.rs`, `src/setup/*` | Base for downloaded runtime dependencies. `AENV_DEPS_PATH` can place these rebuildable assets outside `home_path`. | -| Managed sandbox access-token seed | `$AENV_HOME/secrets/sandbox-access-token-hash-seed` | `src/sandbox/access.rs` | Node-local secret used to derive envd and traffic tokens when `[sandbox].access_token_hash_seed` is unset. It must be preserved with persisted sandboxes. | +| Managed sandbox access-token seed | `$AENV_HOME/secrets/sandbox-access-token-hash-seed` | `src/sandbox/access.rs` | Node-local secret used to derive envd and traffic tokens when `[sandbox].access_token_hash_seed` is unset. It must be preserved with persisted secure or private-ingress sandboxes. | | Firecracker sandbox work dirs | `$AENV_HOME/firecracker-work` with `agentenv-fc-` children | `src/sandbox/firecracker/*` | Per-sandbox runtime directories for sockets, symlinks, ublk runtime dirs, local logs, and writable OverlayBD upper layer data (`overlaybd/upper.data`, `overlaybd/upper.index`). An explicit `[firecracker].work_dir` overrides the root. | | `firecracker.serial_dir` | `$AENV_HOME/logs/serial` | `src/sandbox/firecracker/*` | Durable Firecracker stdout/stderr root, grouped by sandbox ID. An explicit `[firecracker].serial_dir` overrides the root. | | `managed_snapshot_root` | `/managed-snapshots` | `src/sandbox/firecracker/*` | In-process live snapshot artifact root used to keep captured snapshots alive until publish or drop. | @@ -37,7 +37,7 @@ Owned by `src/setup/*` and `src/cfg.rs`. | Overlaybd package downloads | `/overlaybd/downloads/*` | Temporary downloaded package archives | Setup staging for overlaybd release packages | Removed after a successful install. | | Generated overlaybd config | `$AENV_HOME/overlaybd/overlaybd-global.json`, `$AENV_HOME/overlaybd/mem-overlaybd-global.json`, `$AENV_HOME/overlaybd/convert-overlaybd-global.json`, `$AENV_HOME/overlaybd/resize-overlaybd-global.json` | Runtime global config, cache path, credentials config | Configures overlaybd runtime, memory snapshot overlaybd access, and the offline C++ tools (`overlaybd-apply`, `overlaybd-resize`), which get dedicated configs with isolated cacheDirs (`convert-blocks`, `resize-blocks`) and download disabled | Rewritten during setup/startup. | | Overlaybd runtime log | `$AENV_HOME/overlaybd/overlaybd.log` | Overlaybd runtime logs | Debugging | Appended by overlaybd runtime; no automatic GC. | -| Managed sandbox access-token seed | `$AENV_HOME/secrets/sandbox-access-token-hash-seed` | 32 random bytes encoded as lowercase hexadecimal | Derives stable per-sandbox envd and traffic access tokens when no explicit seed is configured | Atomically created with mode `0600` during normal startup and reused thereafter. Must not be deleted while sandboxes are persisted. | +| Managed sandbox access-token seed | `$AENV_HOME/secrets/sandbox-access-token-hash-seed` | 32 random bytes encoded as lowercase hexadecimal | Derives stable per-sandbox envd and traffic access tokens when no explicit seed is configured | Atomically created with mode `0600` during normal startup and reused thereafter. Must not be deleted while secure or private-ingress sandboxes are persisted. | ## Firecracker Sandbox diff --git a/docs/src/internals/proxy-design.md b/docs/src/internals/proxy-design.md index 3fff71daf..822f91897 100644 --- a/docs/src/internals/proxy-design.md +++ b/docs/src/internals/proxy-design.md @@ -43,6 +43,23 @@ Validation: - Sandbox ID must be a valid UUID format. - Target port must parse as `u16` and be greater than `0`. +Authorization is evaluated by the owning runtime after route parsing: + +- Control-plane routes require the deployment `X-API-Key` and do not accept + sandbox credentials. +- Node Prometheus `/metrics` and health `/health` are public to application + auth and should be protected separately at the deployment boundary. +- Non-envd application routes require `e2b-traffic-access-token` only when the + sandbox has private ingress (`allowPublicTraffic: false`). +- The envd port requires `X-Access-Token` only for secure sandboxes. +- `X-API-Key` is never a data-plane credential. + +The distributed gateway deliberately does not make sandbox authorization +decisions. It routes data-plane requests, including public ingress and insecure +envd requests with no credential, to the owning runtime. The runtime has the +sandbox metadata needed to apply the policy and performs the authoritative +token validation. + Host-based routing derives both fields from `Host`. The configured domain must match exactly after lowercase normalization and optional trailing-dot removal. Sandbox IDs in host routes must be valid UUIDs and the target port must fit in @@ -75,7 +92,7 @@ This keeps hot-path reads lock-light and avoids reading sandbox instance interna `/proxy` can auto-resume paused sandboxes when lifecycle policy enables it. -- Proxy never reads sandbox metadata directly. +- Proxy route resolution does not read sandbox instance internals. - Orchestrator lookup returns `Paused { auto_resume }`, and proxy decides behavior from that signal. - Auto-resume is attempted once per request. - Resume timeout update uses `EnsureMinimum(5 minutes)`: @@ -135,6 +152,14 @@ Control-plane routing headers are stripped before forwarding upstream: - `e2b-sandbox-id` - `e2b-sandbox-port` +Sandbox credential handling: + +- `e2b-traffic-access-token` is stripped before forwarding. +- A successfully validated secure-envd `X-Access-Token` is forwarded to envd; + otherwise that header is stripped. +- A value matching the platform `X-API-Key` is stripped; other values are + forwarded as application headers. + Hop-by-hop headers are stripped on both request and response paths, including: - Standard hop-by-hop headers (`Connection`, `Upgrade`, `TE`, `Trailer`, `Transfer-Encoding`, `Proxy-Authenticate`, `Proxy-Authorization`, `Keep-Alive`) @@ -191,7 +216,7 @@ Handshake failure behavior: ```bash curl -i \ - -H 'X-API-Key: test-key' \ + -H 'e2b-traffic-access-token: ' \ -H 'x-agentenv-sandbox-id: ' \ -H 'x-agentenv-target-port: 8080' \ 'http://127.0.0.1:8000/proxy/health?full=true' diff --git a/docs/src/security/secure-sandboxes.md b/docs/src/security/secure-sandboxes.md index 32e387501..764a8771a 100644 --- a/docs/src/security/secure-sandboxes.md +++ b/docs/src/security/secure-sandboxes.md @@ -13,7 +13,7 @@ Set `secure: true` when creating a sandbox through API or E2B-compatible SDKs to aenv start --secure ``` -The API and SDKs return the sandbox's `envdAccessToken` where appropriate and attach it to envd requests automatically. The application proxy credential is independent and is sent as `e2b-traffic-access-token`. Forked sandboxes get independent credentials. Secure mode is preserved across pause, restart, and resume; legacy sandboxes remain non-secure unless created with `secure: true`. +The API and SDKs return the sandbox's `envdAccessToken` where appropriate and attach it to envd requests automatically. Private application ingress has an independent `trafficAccessToken`, sent as `e2b-traffic-access-token`; public application ingress has no AgentENV credential. Forked sandboxes get independent credentials. Secure mode is preserved across pause, restart, and resume; legacy sandboxes remain non-secure unless created with `secure: true`. ## Access-Token Seed diff --git a/scripts/install.sh b/scripts/install.sh index 9ccff8e0a..254bd17c3 100644 --- a/scripts/install.sh +++ b/scripts/install.sh @@ -371,7 +371,7 @@ echo " CLI : ${INSTALL_DIR}/aenv" echo " Server : ${INSTALL_DIR}/server" echo " Data : ${DATA_DIR}" echo " Config : ${CONFIG_PATH}" -echo " API key: generated on first server start in ${DATA_DIR}/secrets/api-key" +echo " API key: ${DATA_DIR}/secrets/api-key (auto-generated when no API key is configured)" echo " Mode : ${VIRTUALIZATION_MODE}" if [[ -d /run/systemd/system ]]; then if [[ "$ENV_FILE_STATUS" == "written" ]]; then diff --git a/scripts/tests/e2e/lib/helpers.sh b/scripts/tests/e2e/lib/helpers.sh index f5803b707..e5c088f46 100644 --- a/scripts/tests/e2e/lib/helpers.sh +++ b/scripts/tests/e2e/lib/helpers.sh @@ -8,6 +8,7 @@ if [[ -z "${E2E_HELPERS_SH_LOADED:-}" ]]; then : "${AENV_API_KEY:=e2b_0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef}" : "${AENV_TEMPLATE_ID:=ubuntu}" : "${AENV_PROXY_URL:=${AENV_URL}/proxy}" + : "${AENV_ENVD_PORT:=49983}" : "${E2E_MODE:=single-node}" : "${E2E_DEFAULT_USER_IMAGE:=ghcr.io/linuxserver/baseimage-ubuntu:noble}" : "${E2E_TEMPLATE_USER_IMAGE:=${E2E_DEFAULT_USER_IMAGE}}" @@ -120,44 +121,6 @@ if [[ -z "${E2E_HELPERS_SH_LOADED:-}" ]]; then HTTP_HEADERS=$(<"$_E2E_HEADERS") } - api_admin_get() { - local path="$1" - _curl_do -s \ - -H "X-API-Key: ${AENV_API_KEY}" \ - "${AENV_URL}${path}" - } - - api_admin_get_at() { - local base_url="$1" - local path="$2" - _curl_do -s \ - -H "X-API-Key: ${AENV_API_KEY}" \ - "${base_url}${path}" - } - - api_admin_get_with_headers() { - local path="$1" - [[ -z "$_E2E_HEADERS" ]] && _E2E_HEADERS=$(mktemp) - curl -s -o "$_E2E_BODY" -D "$_E2E_HEADERS" -w '%{http_code}' \ - -H "X-API-Key: ${AENV_API_KEY}" \ - "${AENV_URL}${path}" > "$_E2E_STATUS" 2>/dev/null || true - HTTP_STATUS=$(<"$_E2E_STATUS") - HTTP_BODY=$(<"$_E2E_BODY") - HTTP_HEADERS=$(<"$_E2E_HEADERS") - } - - api_admin_get_with_headers_at() { - local base_url="$1" - local path="$2" - [[ -z "$_E2E_HEADERS" ]] && _E2E_HEADERS=$(mktemp) - curl -s -o "$_E2E_BODY" -D "$_E2E_HEADERS" -w '%{http_code}' \ - -H "X-API-Key: ${AENV_API_KEY}" \ - "${base_url}${path}" > "$_E2E_STATUS" 2>/dev/null || true - HTTP_STATUS=$(<"$_E2E_STATUS") - HTTP_BODY=$(<"$_E2E_BODY") - HTTP_HEADERS=$(<"$_E2E_HEADERS") - } - api_post() { local path="$1" local body="${2:-}" @@ -208,9 +171,8 @@ if [[ -z "${E2E_HELPERS_SH_LOADED:-}" ]]; then proxy_get_with_sandbox() { local sandbox_id="$1" local path="${2:-/health}" - local target_port="${3:-49983}" + local target_port="${3:-${AENV_ENVD_PORT}}" _curl_do -s --max-time 5 \ - -H "X-API-Key: ${AENV_API_KEY}" \ -H "x-agentenv-sandbox-id: ${sandbox_id}" \ -H "x-agentenv-target-port: ${target_port}" \ "${AENV_PROXY_URL}${path}" diff --git a/scripts/tests/e2e/lib/runtime.sh b/scripts/tests/e2e/lib/runtime.sh index 3d51d31d5..3bfa0c759 100644 --- a/scripts/tests/e2e/lib/runtime.sh +++ b/scripts/tests/e2e/lib/runtime.sh @@ -7,11 +7,19 @@ if [[ -z "${E2E_RUNTIME_SH_LOADED:-}" ]]; then E2E_RUNTIME_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" E2E_REPO_ROOT="$(cd "${E2E_RUNTIME_DIR}/../../../.." && pwd)" + _E2E_API_KEY_FROM_USER=0 + if [[ "${AENV_API_KEY+x}" == "x" ]]; then + if [[ -z "${AENV_API_KEY}" ]]; then + echo "AENV_API_KEY must not be empty" >&2 + return 1 + fi + _E2E_API_KEY_FROM_USER=1 + fi + # shellcheck source=/dev/null source "${E2E_RUNTIME_DIR}/server.sh" : "${E2E_MODE:=single-node}" - : "${AENV_API_KEY:=e2b_0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef}" export AENV_API_KEY : "${E2E_COMPOSE_FILE:=deploy/docker-compose.yml}" : "${E2E_COMPOSE_OVERRIDE_FILE:=scripts/tests/e2e/docker-compose.e2e.yml}" @@ -61,7 +69,11 @@ if [[ -z "${E2E_RUNTIME_SH_LOADED:-}" ]]; then } _deploy_make_cmd() { - make --no-print-directory -C "${E2E_REPO_ROOT}" "$@" + if [[ "${_E2E_API_KEY_FROM_USER}" == "1" ]]; then + make --no-print-directory -C "${E2E_REPO_ROOT}" "$@" + else + env -u AENV_API_KEY make --no-print-directory -C "${E2E_REPO_ROOT}" "$@" + fi } _run_deploy_target() { @@ -156,7 +168,12 @@ if [[ -z "${E2E_RUNTIME_SH_LOADED:-}" ]]; then _run_k8s_target() { local target="${1:?usage: _run_k8s_target }" [[ -n "${target}" && "${target}" != "none" ]] || return 0 - make --no-print-directory -C "${E2E_REPO_ROOT}" "${target}" + _deploy_make_cmd "${target}" + } + + _read_k8s_api_key() { + kubectl -n "${E2E_K8S_NAMESPACE}" get secret agentenv-auth \ + -o 'go-template={{index .data "AENV_API_KEY" | base64decode}}' } _resolve_runtime_path() { @@ -398,8 +415,10 @@ if [[ -z "${E2E_RUNTIME_SH_LOADED:-}" ]]; then _wait_for_health_url "agentenv-b" "${AENV_NODE_B_URL}" "${timeout}" || die "agentenv-b failed to become ready within ${timeout}s" - AENV_API_KEY="$(_compose_cmd exec -T agentenv-a cat /workspace/env/secrets/api-key)" || - die "Failed to read the Compose deployment API key" + if [[ "${_E2E_API_KEY_FROM_USER}" != "1" ]]; then + AENV_API_KEY="$(_compose_cmd exec -T agentenv-a cat /workspace/env/secrets/api-key)" || + die "Failed to read the Compose deployment API key" + fi [[ "${AENV_API_KEY}" =~ ^[A-Za-z0-9._~-]{32,4096}$ ]] || die "Compose deployment returned an invalid API key" export AENV_API_KEY @@ -440,6 +459,14 @@ if [[ -z "${E2E_RUNTIME_SH_LOADED:-}" ]]; then die "${label} failed to become ready within ${timeout}s" done < <(printf '%s\n' "${AENV_NODE_URLS}" | tr ' ' '\n') + if [[ "${_E2E_API_KEY_FROM_USER}" != "1" ]]; then + AENV_API_KEY="$(_read_k8s_api_key)" || + die "Failed to read the Kubernetes deployment API key" + fi + [[ "${AENV_API_KEY}" =~ ^[A-Za-z0-9._~-]{32,4096}$ ]] || + die "Kubernetes deployment returned an invalid API key" + export AENV_API_KEY + expected_nodes="$(_runtime_node_count)" [[ "${expected_nodes}" -gt 0 ]] || expected_nodes=1 _wait_for_scheduler_ready_nodes "${timeout}" "${expected_nodes}" || diff --git a/scripts/tests/e2e/lib/server.sh b/scripts/tests/e2e/lib/server.sh index 1def30252..bc3997ebc 100644 --- a/scripts/tests/e2e/lib/server.sh +++ b/scripts/tests/e2e/lib/server.sh @@ -15,9 +15,9 @@ if [[ -z "${E2E_SERVER_SH_LOADED:-}" ]]; then local binary="${1:?usage: start_server [config_path]}" local config="${2:-}" + export AENV_API_KEY local env_vars=( "API_ADDR=127.0.0.1:${AENV_PORT}" - "AENV_API_KEY=${AENV_API_KEY}" "RUST_LOG=agentenv=info,envd=info" ) [[ -n "$config" ]] && env_vars+=("AENV_CONFIG_PATH=${config}") diff --git a/scripts/tests/e2e/suites/06_proxy.sh b/scripts/tests/e2e/suites/06_proxy.sh index da9903e8a..e500e67cd 100755 --- a/scripts/tests/e2e/suites/06_proxy.sh +++ b/scripts/tests/e2e/suites/06_proxy.sh @@ -12,24 +12,23 @@ log "Suite: Proxy Routing" sandbox_id=$(create_sandbox); _sync_http assert_status "$HTTP_STATUS" "201" "create sandbox for proxy test" assert_not_empty "$sandbox_id" "sandboxID present" +assert_json_field "$HTTP_BODY" '.trafficAccessToken' "null" "public sandbox omits traffic token" track_sandbox "$sandbox_id" wait_for_sandbox_state "$sandbox_id" "running" 30 # -- Proxy request using AgentENV headers -- -# The envd health endpoint runs on port 49983 inside the sandbox. +# The envd health endpoint runs on the configured control-plane port. _curl_do -s --max-time 5 \ - -H "X-API-Key: ${AENV_API_KEY}" \ -H "x-agentenv-sandbox-id: ${sandbox_id}" \ - -H "x-agentenv-target-port: 49983" \ + -H "x-agentenv-target-port: ${AENV_ENVD_PORT}" \ "${AENV_PROXY_URL}/health" log "Proxy (agentenv headers) returned HTTP ${HTTP_STATUS}" assert_status "$HTTP_STATUS" "204" "proxy with agentenv headers" # -- Proxy request using E2B compat headers -- _curl_do -s --max-time 5 \ - -H "X-API-Key: ${AENV_API_KEY}" \ -H "e2b-sandbox-id: ${sandbox_id}" \ - -H "e2b-sandbox-port: 49983" \ + -H "e2b-sandbox-port: ${AENV_ENVD_PORT}" \ "${AENV_PROXY_URL}/health" log "Proxy (e2b headers) returned HTTP ${HTTP_STATUS}" assert_status "$HTTP_STATUS" "204" "proxy with e2b headers" @@ -38,7 +37,6 @@ assert_status "$HTTP_STATUS" "204" "proxy with e2b headers" # Use the explicit /proxy path so both single-node and compose reach the # node-local proxy entrypoint before header validation. _curl_do -s --max-time 5 \ - -H "X-API-Key: ${AENV_API_KEY}" \ "${AENV_URL}/proxy/health" log "Proxy (no sandbox header) returned HTTP ${HTTP_STATUS}" assert_status "$HTTP_STATUS" "400" "proxy without sandbox header" @@ -59,9 +57,8 @@ else fi _curl_do -s --max-time 10 \ - -H "X-API-Key: ${AENV_API_KEY}" \ -H "x-agentenv-sandbox-id: ${paused_no_resume_id}" \ - -H "x-agentenv-target-port: 49983" \ + -H "x-agentenv-target-port: ${AENV_ENVD_PORT}" \ "${AENV_PROXY_URL}/health" log "Proxy (paused + auto-resume disabled) returned HTTP ${HTTP_STATUS}" assert_status "$HTTP_STATUS" "410" "paused sandbox without auto-resume returns 410" @@ -83,9 +80,8 @@ fi # Auto-resume may wait up to 60s in non-test runtime. Keep client timeout above that. _curl_do -s --max-time 75 \ - -H "X-API-Key: ${AENV_API_KEY}" \ -H "e2b-sandbox-id: ${paused_auto_resume_id}" \ - -H "e2b-sandbox-port: 49983" \ + -H "e2b-sandbox-port: ${AENV_ENVD_PORT}" \ "${AENV_PROXY_URL}/health" log "Proxy (paused + auto-resume enabled) returned HTTP ${HTTP_STATUS}" assert_status "$HTTP_STATUS" "204" "paused sandbox auto-resumes on proxy request" diff --git a/scripts/tests/e2e/suites/08_auth.sh b/scripts/tests/e2e/suites/08_auth.sh index 478375176..0c87dab4e 100755 --- a/scripts/tests/e2e/suites/08_auth.sh +++ b/scripts/tests/e2e/suites/08_auth.sh @@ -8,12 +8,23 @@ init_suite "08_auth" log "Suite: Authentication" +proxy_envd_health() { + local sandbox_id="$1" + local header_name="${2:-}" + local header_value="${3:-}" + local args=(-s --max-time 5 + -H "x-agentenv-sandbox-id: ${sandbox_id}" + -H "x-agentenv-target-port: ${AENV_ENVD_PORT}") + [[ -z "${header_name}" ]] || args+=(-H "${header_name}: ${header_value}") + _curl_do "${args[@]}" "${AENV_PROXY_URL}/health" +} + # -- Request without auth header returns 401 -- api_get_no_auth "/sandboxes" assert_status "$HTTP_STATUS" "401" "no auth header returns 401" # -- Alternative and malformed credentials are rejected -- -_curl_do -s -H "X-API-Key: wrong-key" "${AENV_URL}/sandboxes" +_curl_do -s -H "X-API-Key: ${AENV_API_KEY}x" "${AENV_URL}/sandboxes" assert_status "$HTTP_STATUS" "401" "wrong API key returns 401" _curl_do -s -H "Authorization: Bearer ${AENV_API_KEY}" "${AENV_URL}/sandboxes" @@ -27,16 +38,100 @@ assert_status "$HTTP_STATUS" "401" "legacy team key does not authenticate AgentE _curl_do -s \ -H "X-API-Key: ${AENV_API_KEY}" \ - -H "X-API-Key: ${AENV_API_KEY}" \ + -H "X-API-Key: ${AENV_API_KEY}x" \ + "${AENV_URL}/sandboxes" +assert_status "$HTTP_STATUS" "401" "valid and invalid API key headers return 401" + +_curl_do -s \ + -H "X-API-Key: ${AENV_API_KEY}x" \ + -H "X-API-Key: ${AENV_API_KEY}y" \ "${AENV_URL}/sandboxes" -assert_status "$HTTP_STATUS" "401" "duplicate API key headers return 401" +assert_status "$HTTP_STATUS" "401" "conflicting invalid API key headers return 401" # -- Request with valid API key succeeds -- api_get "/sandboxes" -assert_not_eq "$HTTP_STATUS" "401" "valid API key does not return 401" +assert_status "$HTTP_STATUS" "200" "valid API key authenticates successfully" + +# -- Sandbox-scoped credentials cannot authenticate the control plane -- +secure_sandbox_id=$(create_sandbox "$AENV_TEMPLATE_ID" 60 \ + '{"secure":true,"network":{"allowPublicTraffic":false}}'); _sync_http +if [[ -n "$secure_sandbox_id" ]]; then + track_sandbox "$secure_sandbox_id" +fi +assert_status "$HTTP_STATUS" "201" "create private secure sandbox" +assert_not_empty "$secure_sandbox_id" "private secure sandbox ID present" + +if [[ "$HTTP_STATUS" != "201" || -z "$secure_sandbox_id" ]]; then + suite_summary "08_auth" || true + exit 1 +fi + +traffic_access_token=$(echo "$HTTP_BODY" | jq -r '.trafficAccessToken // empty') +envd_access_token=$(echo "$HTTP_BODY" | jq -r '.envdAccessToken // empty') +assert_not_empty "$traffic_access_token" "private sandbox returns traffic token" +assert_not_empty "$envd_access_token" "secure sandbox returns envd token" +if [[ -z "$traffic_access_token" || -z "$envd_access_token" ]]; then + suite_summary "08_auth" || true + exit 1 +fi +wait_for_sandbox_state "$secure_sandbox_id" "running" 30 + +_curl_do -s \ + -H "e2b-traffic-access-token: ${traffic_access_token}" \ + "${AENV_URL}/sandboxes" +assert_status "$HTTP_STATUS" "401" "traffic token does not authenticate control plane" -# -- Health endpoint works without auth -- +_curl_do -s \ + -H "X-Access-Token: ${envd_access_token}" \ + "${AENV_URL}/sandboxes" +assert_status "$HTTP_STATUS" "401" "envd token does not authenticate control plane" + +# -- Secure envd accepts only its envd access token -- +proxy_envd_health "${secure_sandbox_id}" +assert_status "$HTTP_STATUS" "401" "secure envd rejects missing token" + +proxy_envd_health "${secure_sandbox_id}" "X-API-Key" "${AENV_API_KEY}" +assert_status "$HTTP_STATUS" "401" "secure envd rejects API key" + +proxy_envd_health \ + "${secure_sandbox_id}" "e2b-traffic-access-token" "${traffic_access_token}" +assert_status "$HTTP_STATUS" "401" "secure envd rejects traffic token" + +proxy_envd_health "${secure_sandbox_id}" "X-Access-Token" "${envd_access_token}" +assert_status "$HTTP_STATUS" "204" "secure envd accepts envd token" + +# -- Envd tokens are scoped to one sandbox -- +other_secure_sandbox_id=$(create_sandbox "$AENV_TEMPLATE_ID" 60 \ + '{"secure":true,"network":{"allowPublicTraffic":false}}'); _sync_http +if [[ -n "$other_secure_sandbox_id" ]]; then + track_sandbox "$other_secure_sandbox_id" +fi +assert_status "$HTTP_STATUS" "201" "create second private secure sandbox" +assert_not_empty "$other_secure_sandbox_id" "second private secure sandbox ID present" +if [[ "$HTTP_STATUS" != "201" || -z "$other_secure_sandbox_id" ]]; then + suite_summary "08_auth" || true + exit 1 +fi +wait_for_sandbox_state "$other_secure_sandbox_id" "running" 30 + +proxy_envd_health "${other_secure_sandbox_id}" "X-Access-Token" "${envd_access_token}" +assert_status "$HTTP_STATUS" "401" "envd token cannot authenticate another sandbox" + +# -- Health and Prometheus metrics work without application auth -- api_get_no_auth "/health" assert_status "$HTTP_STATUS" "204" "/health works without auth" +api_get_no_auth "/metrics" +if e2e_mode_is_clustered; then + assert_status "$HTTP_STATUS" "404" "gateway client listener does not expose /metrics" +else + assert_status "$HTTP_STATUS" "200" "/metrics works without auth" +fi + +while IFS= read -r node_url; do + [[ -z "${node_url}" ]] && continue + api_get_no_auth_at "${node_url}" "/metrics" + assert_status "$HTTP_STATUS" "200" "node /metrics works without auth at ${node_url}" +done < <(printf '%s\n' "${AENV_NODE_URLS:-}" | tr ' ' '\n') + suite_summary "08_auth" diff --git a/scripts/tests/e2e/suites/11_node_metrics.sh b/scripts/tests/e2e/suites/11_node_metrics.sh index d547ed485..d2df68c78 100644 --- a/scripts/tests/e2e/suites/11_node_metrics.sh +++ b/scripts/tests/e2e/suites/11_node_metrics.sh @@ -11,7 +11,7 @@ log "Suite: Node Metrics" readonly NODE_METRICS_SANDBOX_TIMEOUT_SECONDS=60 readonly SCHEDULER_BINDING_CLEANUP_TIMEOUT_SECONDS=75 readonly PROXY_HEALTH_PATH="/health" -readonly PROXY_HEALTH_PORT=49983 +readonly PROXY_HEALTH_PORT="${AENV_ENVD_PORT}" wait_for_global_sandboxes_quiesced() { local timeout="${1:-20}" @@ -55,7 +55,7 @@ wait_for_node_runtime_allocations_quiesced() { local attempt for ((attempt = 0; attempt < timeout * 2; attempt++)); do - api_admin_get "/nodes" + api_get "/nodes" if [[ "${HTTP_STATUS}" == "200" ]] && echo "${HTTP_BODY}" | jq -e 'all(.[]; (.sandboxCount == 0 and .metrics.allocatedCPU == 0 and .metrics.allocatedMemoryBytes == 0))' >/dev/null 2>&1; then return 0 @@ -97,7 +97,7 @@ quiesce_runtime_state_before_baseline() { fetch_admin_nodes() { local base_url="${1:-${AENV_URL}}" - api_admin_get_at "${base_url}" "/nodes" + api_get_at "${base_url}" "/nodes" [[ "${HTTP_STATUS}" == "200" ]] || return 1 printf '%s\n' "${HTTP_BODY}" } @@ -138,7 +138,7 @@ wait_for_node_snapshot() { local create_successes for ((attempt = 0; attempt < timeout * 2; attempt++)); do - api_admin_get "/nodes" + api_get "/nodes" if [[ "${HTTP_STATUS}" == "200" ]]; then body="${HTTP_BODY}" sandbox_count="$(echo "${body}" | jq -r --arg id "${node_id}" '.[] | select(.id == $id) | .sandboxCount // empty' 2>/dev/null || true)" @@ -164,7 +164,7 @@ wait_for_node_snapshot() { node_detail_sandbox_count() { local base_url="$1" local node_id="$2" - api_admin_get_at "${base_url}" "/nodes/${node_id}" + api_get_at "${base_url}" "/nodes/${node_id}" [[ "${HTTP_STATUS}" == "200" ]] || return 1 echo "${HTTP_BODY}" | jq '.sandboxCount' } @@ -192,7 +192,7 @@ quiesce_runtime_state_before_baseline || wait_for_admin_nodes_count "${AENV_URL}" "${expected_nodes}" 60 || die "Timed out waiting for gateway/admin nodes endpoint" -api_admin_get "/nodes" +api_get "/nodes" assert_status "${HTTP_STATUS}" "200" "admin /nodes returns 200" baseline_nodes_json="${HTTP_BODY}" @@ -206,7 +206,7 @@ while IFS=$'\t' read -r node_id sandbox_count allocated_cpu allocated_memory cre BASELINE_ALLOCATED_MEMORY["${node_id}"]="${allocated_memory}" BASELINE_CREATE_SUCCESSES["${node_id}"]="${create_successes}" - api_admin_get "/nodes/${node_id}" + api_get "/nodes/${node_id}" assert_status "${HTTP_STATUS}" "200" "admin /nodes/${node_id} returns 200" BASELINE_DETAIL_SANDBOX_COUNT["${node_id}"]="$(echo "${HTTP_BODY}" | jq '.sandboxCount')" done < <(echo "${baseline_nodes_json}" | jq -r '.[] | [ @@ -222,7 +222,7 @@ if e2e_mode_is_clustered; then [[ -n "${node_url}" ]] || continue wait_for_admin_nodes_count "${node_url}" 1 45 || die "Timed out waiting for ${node_url}/nodes" - api_admin_get_at "${node_url}" "/nodes" + api_get_at "${node_url}" "/nodes" assert_status "${HTTP_STATUS}" "200" "node-local admin /nodes returns 200 for $(node_label_for_url "${node_url}")" local_node_id="$(echo "${HTTP_BODY}" | jq -r '.[0].id')" assert_not_empty "${local_node_id}" "node-local admin /nodes exposes node id for $(node_label_for_url "${node_url}")" @@ -316,14 +316,14 @@ for node_id in "${!BASELINE_SANDBOX_COUNT[@]}"; do 30; then _pass "gateway /nodes metrics converge for ${node_id}" else - api_admin_get "/nodes" + api_get "/nodes" _fail \ "gateway /nodes metrics converge for ${node_id}" \ "sandboxCount=${expected_sandbox_count}, allocatedCPU=${expected_allocated_cpu}, allocatedMemoryBytes=${expected_allocated_memory}, createSuccesses>=${expected_create_successes_min}" \ "${HTTP_BODY}" fi - api_admin_get "/nodes/${node_id}" + api_get "/nodes/${node_id}" assert_status "${HTTP_STATUS}" "200" "gateway /nodes/${node_id} returns 200 after workload" detail_sandboxes="$(echo "${HTTP_BODY}" | jq '.sandboxCount')" expected_detail_sandboxes=$((BASELINE_DETAIL_SANDBOX_COUNT["${node_id}"] + owned_count)) @@ -370,7 +370,7 @@ for node_id in "${!BASELINE_SANDBOX_COUNT[@]}"; do 30; then _pass "gateway /nodes runtime allocation resets for ${node_id} after cleanup" else - api_admin_get "/nodes" + api_get "/nodes" _fail \ "gateway /nodes runtime allocation resets for ${node_id} after cleanup" \ "sandboxCount=${BASELINE_SANDBOX_COUNT[${node_id}]}, allocatedCPU=${BASELINE_ALLOCATED_CPU[${node_id}]}, allocatedMemoryBytes=${BASELINE_ALLOCATED_MEMORY[${node_id}]}, createSuccesses>=${expected_create_successes_min_after_cleanup}" \ diff --git a/services/README.md b/services/README.md index a5fe1869d..59fe1194e 100644 --- a/services/README.md +++ b/services/README.md @@ -78,10 +78,13 @@ make run-gateway The default local config uses `127.0.0.1:9090` for the scheduler. -The gateway and runtime nodes require the same API key. The gateway reads -`AENV_API_KEY` or `/run/secrets/api-key`; it does not generate a key. -Application proxy requests may additionally use the sandbox response's -`trafficAccessToken` in the `e2b-traffic-access-token` header. +The gateway and runtime nodes require the same API key for control-plane APIs. +The gateway reads `AENV_API_KEY` or `/run/secrets/api-key`; it does not generate +a key. The gateway routes data-plane requests without authenticating them +because only the owning runtime has the sandbox policy needed to distinguish +public ingress, private ingress, and secure envd. Private application proxy +requests use the sandbox response's `trafficAccessToken` in the +`e2b-traffic-access-token` header; secure envd requests use `X-Access-Token`. ## Scheduler configuration diff --git a/services/gateway/cmd/main.go b/services/gateway/cmd/main.go index a0cfacbb7..ea6448ea5 100644 --- a/services/gateway/cmd/main.go +++ b/services/gateway/cmd/main.go @@ -48,7 +48,7 @@ func loadAPIKeyFrom(lookupEnv func(string) (string, bool), secretPath string) (s if explicit, present := lookupEnv(apiKeyEnv); present { value = explicit } else { - file, err := os.Open(secretPath) + file, err := openSecretFile(secretPath) if err != nil { if os.IsNotExist(err) { return "", fmt.Errorf("%s must be set or %s must exist", apiKeyEnv, secretPath) @@ -66,6 +66,28 @@ func loadAPIKeyFrom(lookupEnv func(string) (string, bool), secretPath string) (s return validateAPIKey(value, source) } +func openSecretFile(path string) (*os.File, error) { + fd, err := syscall.Open(path, syscall.O_RDONLY|syscall.O_NONBLOCK, 0) + if err != nil { + return nil, err + } + file := os.NewFile(uintptr(fd), path) + if file == nil { + _ = syscall.Close(fd) + return nil, fmt.Errorf("open returned an invalid file descriptor") + } + info, err := file.Stat() + if err != nil { + _ = file.Close() + return nil, err + } + if !info.Mode().IsRegular() { + _ = file.Close() + return nil, fmt.Errorf("must be a regular file") + } + return file, nil +} + func validateAPIKey(value, source string) (string, error) { if len(value) < 32 || len(value) > maxAPIKeyLen { return "", fmt.Errorf("API key from %s must contain between 32 and %d URL-safe characters", source, maxAPIKeyLen) diff --git a/services/gateway/cmd/main_test.go b/services/gateway/cmd/main_test.go index bc88faebe..f384460e5 100644 --- a/services/gateway/cmd/main_test.go +++ b/services/gateway/cmd/main_test.go @@ -4,6 +4,7 @@ import ( "os" "path/filepath" "strings" + "syscall" "testing" ) @@ -75,3 +76,36 @@ func TestLoadAPIKeyRejectsMissingFile(t *testing.T) { t.Fatal("loadAPIKeyFrom() unexpectedly accepted a missing secret") } } + +func TestLoadAPIKeyRejectsNonRegularFile(t *testing.T) { + t.Parallel() + + path := filepath.Join(t.TempDir(), "api-key") + if err := syscall.Mkfifo(path, 0o600); err != nil { + t.Fatal(err) + } + if _, err := loadAPIKeyFrom(func(string) (string, bool) { return "", false }, path); err == nil { + t.Fatal("loadAPIKeyFrom() unexpectedly accepted a FIFO") + } +} + +func TestLoadAPIKeyAllowsSymlinkedSecret(t *testing.T) { + t.Parallel() + + dir := t.TempDir() + target := filepath.Join(dir, "..data-api-key") + path := filepath.Join(dir, "api-key") + if err := os.WriteFile(target, []byte(testAPIKey+"\n"), 0o444); err != nil { + t.Fatal(err) + } + if err := os.Symlink(filepath.Base(target), path); err != nil { + t.Fatal(err) + } + got, err := loadAPIKeyFrom(func(string) (string, bool) { return "", false }, path) + if err != nil { + t.Fatalf("loadAPIKeyFrom() error = %v", err) + } + if got != testAPIKey { + t.Fatalf("loadAPIKeyFrom() = %q, want %q", got, testAPIKey) + } +} diff --git a/services/gateway/internal/server.go b/services/gateway/internal/server.go index 796d8f69a..d94498c28 100644 --- a/services/gateway/internal/server.go +++ b/services/gateway/internal/server.go @@ -3,6 +3,7 @@ package gateway import ( "bytes" "context" + "crypto/subtle" "encoding/json" "errors" "io" @@ -68,8 +69,7 @@ type Server struct { } func NewServer(logger *zap.Logger, schedulerClient schedulerv1.SchedulerClient, options ServerOptions) (*Server, error) { - apiKey := strings.TrimSpace(options.APIKey) - if apiKey == "" { + if options.APIKey == "" { return nil, errors.New("API key is required") } sandboxProxyDomains, err := normalizeProxyDomains(options.SandboxProxyDomains) @@ -89,7 +89,7 @@ func NewServer(logger *zap.Logger, schedulerClient schedulerv1.SchedulerClient, httpClient: &http.Client{}, requestTimeout: options.RequestTimeout, maxRespSize: options.MaxResponseSize, - apiKey: []byte(apiKey), + apiKey: []byte(options.APIKey), debugMode: options.DebugMode, sandboxProxyDomains: sandboxProxyDomains, }, nil @@ -104,7 +104,20 @@ func (s *Server) Handler() http.Handler { // decoding %2F → / and issuing 301 redirects), which breaks proxy // forwarding of percent-encoded path segments such as /files/%2F. core := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if r.URL.Path == "/health" || r.URL.Path == "/metrics" { + if isExplicitProxyPath(r.URL.Path) && !hasCompleteProxyRouteHeaders(r.Header) { + setGatewayRouteSource(w, routeSourceHeader) + if _, hasSandbox := sandboxIDFromHeaders(r.Header); !hasSandbox { + http.Error(w, "sandbox id header required", http.StatusBadRequest) + return + } + http.Error(w, "target port header required", http.StatusBadRequest) + return + } + if r.URL.Path == "/metrics" { + http.NotFound(w, r) + return + } + if r.URL.Path == "/health" { hostRoute, hostRouteErr := parseHostRoute(r.Host, s.sandboxProxyDomains) if hostRoute != nil || hostRouteErr != nil { s.handleProxy(w, r) @@ -542,6 +555,12 @@ func hasProxyRoutingHeaders(h http.Header) bool { return false } +func hasCompleteProxyRouteHeaders(h http.Header) bool { + _, hasSandbox := sandboxIDFromHeaders(h) + _, hasTargetPort := targetPortFromHeaders(h) + return hasSandbox && hasTargetPort +} + func targetPortFromHeaders(h http.Header) (string, bool) { for _, name := range []string{headerTargetPort, headerE2BTargetPort} { v := strings.TrimSpace(h.Get(name)) @@ -825,42 +844,45 @@ func extractSandboxIDsFromResponse(body []byte) []string { func singleHeaderMatches(headers http.Header, name string, expected []byte) bool { values := headers.Values(name) - return len(values) == 1 && bytes.Equal([]byte(values[0]), expected) + if len(values) != 1 || len(values[0]) != len(expected) { + return false + } + return subtle.ConstantTimeCompare([]byte(values[0]), expected) == 1 } func (s *Server) isSandboxDataPlaneRequest(r *http.Request) bool { - if strings.TrimRight(r.URL.Path, "/") == "/proxy" || strings.HasPrefix(r.URL.Path, "/proxy/") { + if isExplicitProxyPath(r.URL.Path) { + // The explicit proxy prefix cannot dispatch to a control-plane handler. + // Let the core handler return a stable 400 for incomplete routing data. return true } hostRoute, err := parseHostRoute(r.Host, s.sandboxProxyDomains) - if hostRoute != nil || err != nil { + if hostRoute != nil { return true } + if err != nil { + return false + } - return !isSandboxControlPlaneRequest(r) && hasProxyRoutingHeaders(r.Header) + return !isSandboxControlPlaneRequest(r) && hasCompleteProxyRouteHeaders(r.Header) } -func hasSingleNonEmptyHeader(headers http.Header, name string) bool { - values := headers.Values(name) - return len(values) == 1 && strings.TrimSpace(values[0]) != "" +func isExplicitProxyPath(path string) bool { + return path == "/proxy" || strings.HasPrefix(path, "/proxy/") } func (s *Server) authenticate(next http.Handler) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { dataPlane := s.isSandboxDataPlaneRequest(r) - if r.URL.Path == "/health" && !dataPlane { + if dataPlane || r.URL.Path == "/health" || r.URL.Path == "/metrics" { + // Sandbox-scoped ingress and envd authorization depend on runtime + // metadata and are enforced by the owning runtime node. next.ServeHTTP(w, r) return } - authorized := singleHeaderMatches(r.Header, headerAPIKey, s.apiKey) - if !authorized && dataPlane { - // Runtime nodes perform the definitive sandbox-scoped token validation. - authorized = hasSingleNonEmptyHeader(r.Header, headerTrafficToken) || - hasSingleNonEmptyHeader(r.Header, headerEnvdAccessToken) - } - if !authorized { + if !singleHeaderMatches(r.Header, headerAPIKey, s.apiKey) { w.WriteHeader(http.StatusUnauthorized) return } diff --git a/services/gateway/internal/server_test.go b/services/gateway/internal/server_test.go index 7818628a7..c9d4be87c 100644 --- a/services/gateway/internal/server_test.go +++ b/services/gateway/internal/server_test.go @@ -221,7 +221,7 @@ func TestGatewayRequiresExactAPIKey(t *testing.T) { addHeaders: func(headers http.Header) { headers.Set(headerAPIKey, testAPIKey) }, - wantStatus: http.StatusNotFound, + wantStatus: http.StatusBadGateway, }, { name: "duplicate", @@ -235,7 +235,7 @@ func TestGatewayRequiresExactAPIKey(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - req := httptest.NewRequest(http.MethodGet, "/metrics", nil) + req := httptest.NewRequest(http.MethodGet, "/nodes", nil) tt.addHeaders(req.Header) recorder := httptest.NewRecorder() @@ -248,7 +248,18 @@ func TestGatewayRequiresExactAPIKey(t *testing.T) { } } -func TestGatewayForwardsSandboxTokensOnlyOnDataPlane(t *testing.T) { +func TestGatewayMetricsPathDoesNotRequireAPIKey(t *testing.T) { + server := newTestServer(t, stubSchedulerClient{}, time.Second, 1024) + recorder := httptest.NewRecorder() + + server.Handler().ServeHTTP(recorder, httptest.NewRequest(http.MethodGet, "/metrics", nil)) + + if recorder.Code != http.StatusNotFound { + t.Fatalf("status = %d, want %d", recorder.Code, http.StatusNotFound) + } +} + +func TestGatewayLeavesDataPlaneAuthorizationToRuntime(t *testing.T) { const sandboxID = "0191f4d0-7b2a-7c11-9c2d-0123456789ab" lookupCalls := 0 server := newTestServer(t, stubSchedulerClient{ @@ -259,44 +270,63 @@ func TestGatewayForwardsSandboxTokensOnlyOnDataPlane(t *testing.T) { }, time.Second, 1024) handler := server.Handler() - req := httptest.NewRequest(http.MethodGet, "/proxy", nil) - req.Header.Set(headerE2BSandboxID, sandboxID) - req.Header.Set(headerE2BTargetPort, "49983") - req.Header.Set(headerTrafficToken, "runtime-validates-this-token") - recorder := httptest.NewRecorder() - handler.ServeHTTP(recorder, req) - if recorder.Code == http.StatusUnauthorized || lookupCalls != 1 { - t.Fatalf("valid scoped token: status=%d lookup calls=%d", recorder.Code, lookupCalls) + for i, tt := range []struct { + port, header, value string + }{ + {port: "8080"}, + {port: "49983", header: headerTrafficToken, value: "runtime-validates-this-token"}, + {port: "49983", header: headerTrafficToken, value: "wrong-token"}, + {port: "8080", header: headerEnvdAccessToken, value: "runtime-validates-this-token"}, + } { + req := httptest.NewRequest(http.MethodGet, "/proxy", nil) + req.Header.Set(headerE2BSandboxID, sandboxID) + req.Header.Set(headerE2BTargetPort, tt.port) + if tt.header != "" { + req.Header.Set(tt.header, tt.value) + } + recorder := httptest.NewRecorder() + handler.ServeHTTP(recorder, req) + if recorder.Code == http.StatusUnauthorized || lookupCalls != i+1 { + t.Fatalf("data plane case %d: status=%d lookup calls=%d", i, recorder.Code, lookupCalls) + } } - req = httptest.NewRequest(http.MethodGet, "/proxy", nil) + req := httptest.NewRequest(http.MethodPost, "/sandboxes/"+sandboxID+"/pause", nil) req.Header.Set(headerE2BSandboxID, sandboxID) - req.Header.Set(headerE2BTargetPort, "49983") - req.Header.Set(headerTrafficToken, "wrong-token") - recorder = httptest.NewRecorder() + req.Header.Set(headerTrafficToken, "runtime-validates-this-token") + recorder := httptest.NewRecorder() handler.ServeHTTP(recorder, req) - if recorder.Code == http.StatusUnauthorized || lookupCalls != 2 { - t.Fatalf("runtime-scoped token: status=%d lookup calls=%d", recorder.Code, lookupCalls) - } - req = httptest.NewRequest(http.MethodGet, "/proxy", nil) - req.Header.Set(headerE2BSandboxID, sandboxID) - req.Header.Set(headerE2BTargetPort, "8080") - req.Header.Set("X-Access-Token", "runtime-validates-this-token") - recorder = httptest.NewRecorder() - handler.ServeHTTP(recorder, req) - if recorder.Code == http.StatusUnauthorized || lookupCalls != 3 { - t.Fatalf("runtime-scoped envd token: status=%d lookup calls=%d", recorder.Code, lookupCalls) + if recorder.Code != http.StatusUnauthorized || lookupCalls != 4 { + t.Fatalf("scoped token reached control plane: status=%d lookup calls=%d", recorder.Code, lookupCalls) } +} - req = httptest.NewRequest(http.MethodPost, "/sandboxes/"+sandboxID+"/pause", nil) - req.Header.Set(headerE2BSandboxID, sandboxID) - req.Header.Set(headerTrafficToken, "runtime-validates-this-token") - recorder = httptest.NewRecorder() - handler.ServeHTTP(recorder, req) +func TestGatewayRejectsIncompleteProxyRouteBeforeScheduling(t *testing.T) { + scheduleCalls := 0 + server := newTestServer(t, stubSchedulerClient{ + scheduleFunc: func(context.Context, *schedulerv1.ScheduleRequest, ...grpc.CallOption) (*schedulerv1.ScheduleResponse, error) { + scheduleCalls++ + return nil, fmt.Errorf("schedule reached") + }, + }, time.Second, 1024) + handler := server.Handler() - if recorder.Code != http.StatusUnauthorized || lookupCalls != 3 { - t.Fatalf("scoped token reached control plane: status=%d lookup calls=%d", recorder.Code, lookupCalls) + for _, headers := range []http.Header{ + {}, + {headerE2BSandboxID: []string{"sandbox-only"}}, + {headerE2BTargetPort: []string{"8080"}}, + } { + req := httptest.NewRequest(http.MethodGet, "/proxy", nil) + req.Header = headers + recorder := httptest.NewRecorder() + handler.ServeHTTP(recorder, req) + if recorder.Code != http.StatusBadRequest { + t.Fatalf("status = %d, want %d", recorder.Code, http.StatusBadRequest) + } + } + if scheduleCalls != 0 { + t.Fatalf("schedule calls = %d, want 0", scheduleCalls) } } diff --git a/src/api/generated/src/models.rs b/src/api/generated/src/models.rs index 434da3440..def67a2aa 100644 --- a/src/api/generated/src/models.rs +++ b/src/api/generated/src/models.rs @@ -5722,7 +5722,7 @@ impl std::convert::TryFrom for header::IntoHeaderValue, diff --git a/src/api/impls/auth.rs b/src/api/impls/auth.rs index 435ea1188..905c8fd8c 100644 --- a/src/api/impls/auth.rs +++ b/src/api/impls/auth.rs @@ -27,7 +27,9 @@ impl ApiImpl { single_header(headers, API_KEY_HEADER).is_some_and(|value| { let candidate = value.as_bytes(); let expected = self.api_key.as_bytes(); - candidate.len() == expected.len() && bool::from(candidate.ct_eq(expected)) + !expected.is_empty() + && candidate.len() == expected.len() + && bool::from(candidate.ct_eq(expected)) }) } @@ -55,32 +57,59 @@ where { let proxy_request = proxy::is_sandbox_proxy_request(&request, api_impl.as_ref().sandbox_proxy_domains()); - if request.uri().path() == "/health" && !proxy_request { + if matches!(request.uri().path(), "/health" | "/metrics") && !proxy_request { return next.run(request).await; } let api_impl = api_impl.as_ref(); - let mut authorized = api_impl.has_valid_api_key(request.headers()); - let mut envd_authorized = false; - if proxy_request { - if let Some((sandbox_id, target_port)) = - proxy::route_for_auth(&request, api_impl.sandbox_proxy_domains()) - { - authorized |= api_impl.has_valid_traffic_access_token(request.headers(), sandbox_id); - let envd_candidate = single_header(request.headers(), ENVD_ACCESS_TOKEN_HEADER) - .and_then(|value| value.to_str().ok()); - if let Some(candidate) = envd_candidate { - envd_authorized = proxy::has_valid_envd_access_token( - api_impl, - sandbox_id, - target_port, - candidate, - ) - .await; - authorized |= envd_authorized; - } - } + if !proxy_request { + return if api_impl.has_valid_api_key(request.headers()) { + next.run(request).await + } else { + StatusCode::UNAUTHORIZED.into_response() + }; + } + + let has_api_key = api_impl.has_valid_api_key(request.headers()); + + let Some((sandbox_id, target_port)) = + proxy::route_for_auth(&request, api_impl.sandbox_proxy_domains()) + else { + request.headers_mut().remove(ENVD_ACCESS_TOKEN_HEADER); + return if proxy::has_proxy_prefix(request.uri().path()) || has_api_key { + next.run(request).await + } else { + StatusCode::UNAUTHORIZED.into_response() + }; + }; + if has_api_key { + request.headers_mut().remove(API_KEY_HEADER); } + let metadata = match api_impl.orchestrator().get_sandbox(&sandbox_id).await { + Ok(Some(metadata)) => metadata, + Ok(None) => { + request.headers_mut().remove(ENVD_ACCESS_TOKEN_HEADER); + return proxy::sandbox_not_found_response(sandbox_id); + } + Err(_) => return StatusCode::INTERNAL_SERVER_ERROR.into_response(), + }; + + let envd_request = target_port == proxy::effective_envd_port(&metadata); + let envd_authorized = envd_request + && metadata.secure + && single_header(request.headers(), ENVD_ACCESS_TOKEN_HEADER) + .and_then(|value| value.to_str().ok()) + .is_some_and(|candidate| { + api_impl + .orchestrator() + .validate_envd_access_token(sandbox_id, candidate) + }); + let authorized = if envd_request { + !metadata.secure || envd_authorized + } else { + metadata.network_policy.allow_public_traffic + || api_impl.has_valid_traffic_access_token(request.headers(), sandbox_id) + }; if !authorized { return StatusCode::UNAUTHORIZED.into_response(); diff --git a/src/api/impls/sandbox.rs b/src/api/impls/sandbox.rs index 48648dc8a..73c14311e 100644 --- a/src/api/impls/sandbox.rs +++ b/src/api/impls/sandbox.rs @@ -145,7 +145,7 @@ impl From<&SandboxNetworkPolicy> for models::SandboxNetworkConfig { fn from(policy: &SandboxNetworkPolicy) -> Self { let egress = &policy.egress; Self { - allow_public_traffic: Some(true), + allow_public_traffic: Some(policy.allow_public_traffic), allow_out: (!egress.allowed_cidrs.is_empty() || !egress.allowed_domains.is_empty()) .then(|| { egress @@ -179,10 +179,9 @@ fn allow_internet_access_from_base_policy(policy: BaseSandboxNetworkPolicy) -> N impl From for models::SandboxDetail { fn from(m: SandboxMetadata) -> Self { - let network = m - .network_policy - .has_explicit_egress_rules() - .then(|| models::SandboxNetworkConfig::from(&m.network_policy)); + let network = (!m.network_policy.allow_public_traffic + || m.network_policy.has_explicit_egress_rules()) + .then(|| models::SandboxNetworkConfig::from(&m.network_policy)); let allow_internet_access = Some(allow_internet_access_from_base_policy( m.network_policy.base_policy, )); @@ -214,14 +213,18 @@ impl From for models::SandboxDetail { impl ApiImpl { fn sandbox_model(&self, metadata: SandboxMetadata) -> models::Sandbox { - let traffic_access_token = self.traffic_access_token(metadata.id); + let traffic_access_token = (!metadata.network_policy.allow_public_traffic) + .then(|| self.traffic_access_token(metadata.id)); let envd_access_token = self .orchestrator .get_envd_access_token(&metadata) .map(|token| token.expose().to_owned()); let mut sandbox = models::Sandbox::from(metadata); sandbox.envd_access_token = envd_access_token; - sandbox.traffic_access_token = Some(Nullable::Present(traffic_access_token)); + sandbox.traffic_access_token = Some(match traffic_access_token { + Some(token) => Nullable::Present(token), + None => Nullable::Null, + }); sandbox.domain = self .sandbox_proxy_domains() .first() @@ -366,7 +369,11 @@ fn network_policy_from_create( let allow_out = network.and_then(|network| network.allow_out.clone()); let deny_out = network.and_then(|network| network.deny_out.clone()); let egress = SandboxNetworkEgressPolicy::new(allow_out, deny_out)?; - let policy = SandboxNetworkPolicy::new(base_policy, egress); + let allow_public_traffic = network + .and_then(|network| network.allow_public_traffic) + .unwrap_or(true); + let policy = SandboxNetworkPolicy::new(base_policy, egress) + .with_allow_public_traffic(allow_public_traffic); if policy.has_domain_allow_rules() { anyhow::bail!( "domain entries in allowOut are not supported until TCP egress proxy is enabled" @@ -1505,6 +1512,20 @@ mod tests { assert_eq!(policy.egress.denied_cidrs, ["203.0.113.0/24"]); } + #[test] + fn network_create_preserves_private_ingress() { + let mut network = models::SandboxNetworkConfig::new(); + network.allow_public_traffic = Some(false); + + let policy = network_policy_from_create(None, Some(&network)).unwrap(); + + assert!(!policy.allow_public_traffic); + assert_eq!( + models::SandboxNetworkConfig::from(&policy).allow_public_traffic, + Some(false) + ); + } + #[test] fn empty_network_update_clears_base_policy_and_egress() { let policy = network_policy_from_update(&models::SandboxNetworkUpdateConfig::new()) diff --git a/src/api/openapi.yml b/src/api/openapi.yml index 32a4c74b7..4c56086fc 100644 --- a/src/api/openapi.yml +++ b/src/api/openapi.yml @@ -302,7 +302,7 @@ components: allowPublicTraffic: type: boolean default: true - description: Specify if the sandbox URLs should be accessible only with authentication. + description: Specify if the sandbox URLs should be accessible without a traffic access token. allowOut: type: array description: List of allowed destinations for egress traffic. Each entry can be a CIDR block (e.g. "8.8.8.8/32"), a bare IP address (e.g. "8.8.8.8"), or a domain name (e.g. "example.com", "*.example.com"). Allowed entries always take precedence over denied entries. diff --git a/src/api/proxy.rs b/src/api/proxy.rs index d275ed4d3..73661306d 100644 --- a/src/api/proxy.rs +++ b/src/api/proxy.rs @@ -35,10 +35,7 @@ use tokio_tungstenite::{ use tracing::{debug, info, trace, warn}; use crate::{ - api::{ - impls::auth::{API_KEY_HEADER, ENVD_ACCESS_TOKEN_HEADER, TRAFFIC_ACCESS_TOKEN_HEADER}, - ApiImpl, - }, + api::{impls::auth::TRAFFIC_ACCESS_TOKEN_HEADER, ApiImpl}, cfg::ConfigManager, observability::prometheus::HttpRouteSource, orchestrator::{ @@ -48,6 +45,9 @@ use crate::{ types::SandboxId, }; +#[cfg(test)] +use crate::api::impls::auth::{API_KEY_HEADER, ENVD_ACCESS_TOKEN_HEADER}; + /// Shared outbound HTTP client for the client-facing reverse proxy. pub(crate) type ProxyClient = Client; type UpstreamWebSocket = WebSocketStream>; @@ -148,8 +148,10 @@ where pub(crate) fn route_for_auth(request: &Request, domains: &[String]) -> Option<(SandboxId, u16)> { if !has_proxy_prefix(request.uri().path()) { - if let Ok(Some(route)) = parse_host_proxy_route(request_host(request), domains) { - return Some((route.sandbox_id, route.target_port)); + match parse_host_proxy_route(request_host(request), domains) { + Ok(Some(route)) => return Some((route.sandbox_id, route.target_port)), + Ok(None) => {} + Err(_) => return None, } } @@ -159,24 +161,6 @@ pub(crate) fn route_for_auth(request: &Request, domains: &[String]) -> Option<(S )) } -pub(crate) async fn has_valid_envd_access_token( - api_impl: &ApiImpl, - sandbox_id: SandboxId, - target_port: u16, - candidate: &str, -) -> bool { - let Ok(Some(metadata)) = api_impl.orchestrator().get_sandbox(&sandbox_id).await else { - return false; - }; - if !metadata.secure || target_port != effective_envd_port(&metadata) { - return false; - } - - api_impl - .orchestrator() - .validate_envd_access_token(sandbox_id, candidate) -} - pub(crate) fn is_sandbox_proxy_request(request: &Request, domains: &[String]) -> bool { let path = request.uri().path(); if has_proxy_prefix(path) { @@ -336,7 +320,7 @@ fn strip_proxy_prefix(path: &str) -> &str { path.strip_prefix(PROXY_ROUTE).unwrap_or("") } -fn has_proxy_prefix(path: &str) -> bool { +pub(crate) fn has_proxy_prefix(path: &str) -> bool { path == PROXY_ROUTE || path.starts_with("/proxy/") } @@ -420,7 +404,7 @@ fn has_routing_header(headers: &HeaderMap) -> bool { headers.get(SANDBOX_ID_HEADER).is_some() || headers.get(E2B_SANDBOX_ID_HEADER).is_some() } -fn effective_envd_port(metadata: &SandboxMetadata) -> u16 { +pub(crate) fn effective_envd_port(metadata: &SandboxMetadata) -> u16 { metadata .paused_state .as_ref() @@ -750,13 +734,6 @@ async fn resolve_proxy_request( sandbox_id, ))); } - authorize_secure_envd_auto_resume( - api_impl, - sandbox_id, - target_port, - &parts.headers, - ) - .await?; try_auto_resume(api_impl, sandbox_id).await?; auto_resume_attempted = true; continue; @@ -819,45 +796,6 @@ async fn resolve_proxy_request( }) } -async fn authorize_secure_envd_auto_resume( - api_impl: &ApiImpl, - sandbox_id: SandboxId, - target_port: u16, - headers: &HeaderMap, -) -> Result<(), Response> { - let metadata = api_impl - .orchestrator() - .get_sandbox(&sandbox_id) - .await - .map_err(|_| StatusCode::INTERNAL_SERVER_ERROR.into_response())? - .ok_or_else(|| proxy_error_response(&ProxyRequestError::SandboxNotFound(sandbox_id)))?; - if target_port != effective_envd_port(&metadata) { - return Ok(()); - } - if !metadata.secure { - return Ok(()); - } - let candidate = headers - .get(ENVD_ACCESS_TOKEN_HEADER) - .and_then(|value| value.to_str().ok()) - .unwrap_or_default(); - if api_impl - .orchestrator() - .validate_envd_access_token(sandbox_id, candidate) - { - return Ok(()); - } - - Err(Response::builder() - .status(StatusCode::UNAUTHORIZED) - .header( - header::CONTENT_TYPE, - HeaderValue::from_static("text/plain; charset=utf-8"), - ) - .body(Body::from("invalid or missing envd access token")) - .expect("static unauthorized proxy response is valid")) -} - async fn try_auto_resume(api_impl: &ApiImpl, sandbox_id: SandboxId) -> Result<(), Response> { match timeout( PROXY_AUTO_RESUME_TIMEOUT, @@ -909,6 +847,10 @@ fn parse_target_port_header(headers: &HeaderMap) -> Result Response { + proxy_error_response(&ProxyRequestError::SandboxNotFound(sandbox_id)) +} + fn proxy_error_response(error: &ProxyRequestError) -> Response { let (status, message) = match error { ProxyRequestError::MissingSandboxId => { @@ -1040,7 +982,6 @@ fn sanitize_request_headers(headers: &mut HeaderMap) { headers.remove(E2B_SANDBOX_ID_HEADER); headers.remove(TARGET_PORT_HEADER); headers.remove(E2B_TARGET_PORT_HEADER); - headers.remove(API_KEY_HEADER); headers.remove(TRAFFIC_ACCESS_TOKEN_HEADER); headers.remove(header::HOST); remove_hop_by_hop_headers(headers); @@ -1681,6 +1622,10 @@ mod tests { } async fn build_api_with_sandbox_proxy_domains(domains: Vec) -> Arc { + build_api_with_auth(domains, "test-key").await + } + + async fn build_api_with_auth(domains: Vec, api_key: &str) -> Arc { let root = tempfile::tempdir().unwrap(); let orchestrator = Orchestrator::new( crate::orchestrator::InMemoryMetadataStore::new(), @@ -1699,7 +1644,7 @@ mod tests { image_resolver, None, domains, - "test-key".to_string(), + api_key.to_string(), )) } @@ -1740,6 +1685,10 @@ mod tests { crate::orchestrator::SandboxState::Running, ) .await; + api.orchestrator() + .set_allow_public_traffic_for_test(sandbox_id, false) + .await + .unwrap(); (server::new(api), access_token) } @@ -1756,6 +1705,10 @@ mod tests { crate::orchestrator::SandboxState::Running, ) .await; + api.orchestrator() + .set_allow_public_traffic_for_test(sandbox_id, false) + .await + .unwrap(); (server::new(api), access_token) } @@ -1775,6 +1728,20 @@ mod tests { spawn_upstream(proxy_app_for_sandbox(sandbox_id).await).await } + async fn get_status(app: &axum::Router, uri: &str, headers: &[(&str, &str)]) -> StatusCode { + let mut request = Request::builder() + .uri(uri) + .header(header::HOST, "localhost"); + for (name, value) in headers { + request = request.header(*name, *value); + } + app.clone() + .oneshot(request.body(Body::empty()).unwrap()) + .await + .unwrap() + .status() + } + #[test] fn parses_agentenv_headers() { let sandbox_id = SandboxId::new().to_string(); @@ -1819,6 +1786,11 @@ mod tests { ); headers.insert(TARGET_PORT_HEADER, HeaderValue::from_static("8080")); headers.insert(E2B_TARGET_PORT_HEADER, HeaderValue::from_static("8080")); + headers.insert(API_KEY_HEADER, HeaderValue::from_static("application-key")); + headers.insert( + TRAFFIC_ACCESS_TOKEN_HEADER, + HeaderValue::from_static("traffic-token"), + ); headers.insert(HOST, HeaderValue::from_static("client.example")); headers.insert(header::CONNECTION, HeaderValue::from_static("keep-alive")); headers.insert( @@ -1832,6 +1804,8 @@ mod tests { assert!(headers.get(E2B_SANDBOX_ID_HEADER).is_none()); assert!(headers.get(TARGET_PORT_HEADER).is_none()); assert!(headers.get(E2B_TARGET_PORT_HEADER).is_none()); + assert_eq!(headers.get(API_KEY_HEADER).unwrap(), "application-key"); + assert!(headers.get(TRAFFIC_ACCESS_TOKEN_HEADER).is_none()); assert!(headers.get(HOST).is_none()); assert!(headers.get(header::CONNECTION).is_none()); assert_eq!(headers.get("x-extra").unwrap(), "keep"); @@ -1881,117 +1855,174 @@ mod tests { } #[tokio::test] - async fn server_requires_exact_api_key_and_leaves_health_public() { + async fn control_plane_auth_is_separate_from_sandbox_auth() { let app = server::new(build_api().await); - for request in [ - Request::builder() - .uri("/nonexistent/path") - .body(Body::empty()) - .unwrap(), - Request::builder() - .uri("/nonexistent/path") - .header(header::AUTHORIZATION, "Bearer test-key") - .body(Body::empty()) - .unwrap(), - Request::builder() - .uri("/nonexistent/path") - .header(API_KEY_HEADER, "wrong-key") - .body(Body::empty()) - .unwrap(), - Request::builder() - .uri("/nonexistent/path") - .header(API_KEY_HEADER, "test-key") - .header(API_KEY_HEADER, "test-key") - .body(Body::empty()) - .unwrap(), + for headers in [ + vec![], + vec![(header::AUTHORIZATION.as_str(), "Bearer test-key")], + vec![(API_KEY_HEADER, "wrong-key")], + vec![(API_KEY_HEADER, "test-key"), (API_KEY_HEADER, "test-key")], ] { - let response = app.clone().oneshot(request).await.unwrap(); - assert_eq!(response.status(), StatusCode::UNAUTHORIZED); + assert_eq!( + get_status(&app, "/nonexistent/path", &headers).await, + StatusCode::UNAUTHORIZED + ); } - let response = app - .clone() - .oneshot( - Request::builder() - .uri("/nonexistent/path") - .header(API_KEY_HEADER, "test-key") - .body(Body::empty()) - .unwrap(), - ) - .await - .unwrap(); - assert_eq!(response.status(), StatusCode::NOT_FOUND); + for (path, headers, expected) in [ + ( + "/nonexistent/path", + vec![(API_KEY_HEADER, "test-key")], + StatusCode::NOT_FOUND, + ), + ("/health", vec![], StatusCode::NO_CONTENT), + ] { + assert_eq!(get_status(&app, path, &headers).await, expected); + } + assert_ne!( + get_status(&app, "/metrics", &[]).await, + StatusCode::UNAUTHORIZED + ); - let response = app - .clone() - .oneshot( - Request::builder() - .uri("/health") - .header(header::HOST, "localhost") - .body(Body::empty()) - .unwrap(), - ) - .await - .unwrap(); - assert_eq!(response.status(), StatusCode::NO_CONTENT); + let empty_key_app = server::new(build_api_with_auth(Vec::new(), "").await); + assert_eq!( + get_status(&empty_key_app, "/sandboxes", &[(API_KEY_HEADER, "")]).await, + StatusCode::UNAUTHORIZED + ); - let response = app - .oneshot( - Request::builder() - .uri("/health") - .header(header::HOST, "localhost") - .header(SANDBOX_ID_HEADER, SandboxId::new().to_string()) - .body(Body::empty()) - .unwrap(), - ) - .await - .unwrap(); - assert_eq!(response.status(), StatusCode::NO_CONTENT); + let sandbox_id = SandboxId::new().to_string(); + let route = [ + (SANDBOX_ID_HEADER, sandbox_id.as_str()), + (TARGET_PORT_HEADER, "8080"), + ]; + assert_eq!( + get_status(&app, "/sandboxes", &route).await, + StatusCode::UNAUTHORIZED + ); + assert_eq!( + get_status(&app, "/proxy/health", &route).await, + StatusCode::NOT_FOUND + ); } #[tokio::test] - async fn traffic_token_cannot_authenticate_control_plane() { + async fn application_proxy_auth_respects_public_and_private_ingress() { + let upstream_addr = start_upstream_server().await; let api = build_api().await; let sandbox_id = SandboxId::new(); - let traffic_token = api.traffic_access_token(sandbox_id); - let app = server::new(api); - let response = app - .oneshot( - Request::builder() - .method(Method::POST) - .uri(format!("/sandboxes/{sandbox_id}/pause")) - .header(TRAFFIC_ACCESS_TOKEN_HEADER, traffic_token) - .header(SANDBOX_ID_HEADER, sandbox_id.to_string()) - .header(TARGET_PORT_HEADER, "80") - .body(Body::empty()) - .unwrap(), + api.orchestrator() + .set_proxy_target_for_test( + sandbox_id, + ProxyTarget::new(Ipv4Addr::LOCALHOST), + crate::orchestrator::SandboxState::Running, ) + .await; + + let app = server::new(Arc::clone(&api)); + let sandbox_id_text = sandbox_id.to_string(); + let port = upstream_addr.port().to_string(); + let route = [ + (SANDBOX_ID_HEADER, sandbox_id_text.as_str()), + (TARGET_PORT_HEADER, port.as_str()), + ]; + assert_eq!( + get_status(&app, "/proxy/public", &route).await, + StatusCode::OK + ); + + api.orchestrator() + .set_allow_public_traffic_for_test(&sandbox_id, false) .await .unwrap(); + let traffic_token = api.traffic_access_token(sandbox_id); - assert_eq!(response.status(), StatusCode::UNAUTHORIZED); + for credential in [ + None, + Some((API_KEY_HEADER, "test-key")), + Some((TRAFFIC_ACCESS_TOKEN_HEADER, "incorrect")), + Some((ENVD_ACCESS_TOKEN_HEADER, "envd-token")), + ] { + let mut headers = route.to_vec(); + if let Some((header_name, value)) = credential { + headers.push((header_name, value)); + } + assert_eq!( + get_status(&app, "/proxy/private", &headers).await, + StatusCode::UNAUTHORIZED + ); + } + + let mut headers = route.to_vec(); + headers.push((TRAFFIC_ACCESS_TOKEN_HEADER, traffic_token.as_str())); + headers.push((API_KEY_HEADER, "application-api-key")); + assert_eq!( + get_status(&app, "/proxy/private", &headers).await, + StatusCode::OK + ); } #[tokio::test] - async fn envd_token_cannot_authenticate_application_proxy() { + async fn envd_proxy_auth_depends_only_on_secure_mode_and_envd_token() { let api = build_api().await; let sandbox_id = SandboxId::new(); - let traffic_token = api.traffic_access_token(sandbox_id); - let response = server::new(api) - .oneshot( - Request::builder() - .uri("/proxy/hello") - .header(ENVD_ACCESS_TOKEN_HEADER, traffic_token) - .header(SANDBOX_ID_HEADER, sandbox_id.to_string()) - .header(TARGET_PORT_HEADER, "80") - .body(Body::empty()) - .unwrap(), + api.orchestrator() + .set_proxy_target_for_test( + sandbox_id, + ProxyTarget::new(Ipv4Addr::LOCALHOST), + crate::orchestrator::SandboxState::Running, ) + .await; + let target_port = ConfigManager::global_config() + .tools + .control_plane_port + .to_string(); + let app = server::new(Arc::clone(&api)); + let sandbox_id_text = sandbox_id.to_string(); + let route = [ + (SANDBOX_ID_HEADER, sandbox_id_text.as_str()), + (TARGET_PORT_HEADER, target_port.as_str()), + ]; + assert_ne!( + get_status(&app, "/proxy/health", &route).await, + StatusCode::UNAUTHORIZED + ); + + api.orchestrator() + .set_secure_for_test(&sandbox_id, true) .await .unwrap(); + let metadata = api + .orchestrator() + .get_sandbox(&sandbox_id) + .await + .unwrap() + .unwrap(); + let envd_token = api.orchestrator().get_envd_access_token(&metadata).unwrap(); + let traffic_token = api.traffic_access_token(sandbox_id); - assert_eq!(response.status(), StatusCode::UNAUTHORIZED); + for credential in [ + None, + Some((API_KEY_HEADER, "test-key")), + Some((TRAFFIC_ACCESS_TOKEN_HEADER, traffic_token.as_str())), + Some((ENVD_ACCESS_TOKEN_HEADER, "incorrect")), + ] { + let mut headers = route.to_vec(); + if let Some((header_name, value)) = credential { + headers.push((header_name, value)); + } + assert_eq!( + get_status(&app, "/proxy/health", &headers).await, + StatusCode::UNAUTHORIZED + ); + } + + let mut headers = route.to_vec(); + headers.push((ENVD_ACCESS_TOKEN_HEADER, envd_token.expose())); + assert_ne!( + get_status(&app, "/proxy/health", &headers).await, + StatusCode::UNAUTHORIZED + ); } #[tokio::test] @@ -2003,7 +2034,6 @@ mod tests { .oneshot( Request::builder() .uri("/proxy/hello") - .header("x-api-key", "test-key") .body(Body::empty()) .unwrap(), ) @@ -2033,7 +2063,7 @@ mod tests { Request::builder() .method(Method::GET) .uri("/proxy/health") - .header("x-api-key", "test-key") + .header("x-api-key", "application-key") .header(SANDBOX_ID_HEADER, sandbox_id.to_string()) .header(TARGET_PORT_HEADER, upstream_addr.port().to_string()) .body(Body::empty()) @@ -2066,7 +2096,6 @@ mod tests { Request::builder() .method(Method::GET) .uri("/proxy/health") - .header("x-api-key", "test-key") .header(SANDBOX_ID_HEADER, sandbox_id.to_string()) .header(TARGET_PORT_HEADER, upstream_addr.port().to_string()) .body(Body::empty()) @@ -2115,7 +2144,6 @@ mod tests { let mut request = Request::builder() .method(Method::GET) .uri("/proxy/health") - .header("x-api-key", "test-key") .header(SANDBOX_ID_HEADER, sandbox_id.to_string()) .header( TARGET_PORT_HEADER, @@ -2140,7 +2168,6 @@ mod tests { Request::builder() .method(Method::GET) .uri("/proxy/health") - .header("x-api-key", "test-key") .header(SANDBOX_ID_HEADER, sandbox_id.to_string()) .header( TARGET_PORT_HEADER, @@ -2169,7 +2196,6 @@ mod tests { Request::builder() .method(Method::GET) .uri("/proxy/health") - .header("x-api-key", "test-key") .header(SANDBOX_ID_HEADER, sandbox_id.to_string()) .header(TARGET_PORT_HEADER, upstream_addr.port().to_string()) .body(Body::empty()) @@ -2196,7 +2222,6 @@ mod tests { Request::builder() .method(Method::GET) .uri("/proxy/health") - .header("x-api-key", "test-key") .header(SANDBOX_ID_HEADER, sandbox_id.to_string()) .header(TARGET_PORT_HEADER, "0") .body(Body::empty()) @@ -2224,7 +2249,6 @@ mod tests { Request::builder() .method(Method::GET) .uri("/proxy?foo=bar") - .header("x-api-key", "test-key") .header(SANDBOX_ID_HEADER, sandbox_id.to_string()) .header(TARGET_PORT_HEADER, upstream_addr.port().to_string()) .body(Body::empty()) @@ -2242,7 +2266,7 @@ mod tests { } #[tokio::test] - async fn proxy_forwards_request_and_strips_internal_headers() { + async fn proxy_forwards_application_headers_and_strips_internal_headers() { let upstream_addr = start_upstream_server().await; let sandbox_id = SandboxId::new(); let app = proxy_app_for_sandbox(&sandbox_id).await; @@ -2253,7 +2277,7 @@ mod tests { .method(Method::GET) .uri("/proxy/echo/test?foo=bar".to_string()) .header("host", "client.example") - .header("x-api-key", "test-key") + .header(API_KEY_HEADER, "application-key") .header(header::AUTHORIZATION, "Bearer application-token") .header(SANDBOX_ID_HEADER, sandbox_id.to_string()) .header(TARGET_PORT_HEADER, upstream_addr.port().to_string()) @@ -2278,7 +2302,7 @@ mod tests { assert!(payload["access_token"].is_null()); assert_eq!(payload["traffic_token_header_seen"], false); assert_eq!(payload["forwarded_host"], "client.example"); - assert_eq!(payload["api_key_header_seen"], false); + assert_eq!(payload["api_key_header_seen"], true); assert_eq!(payload["authorization"], "Bearer application-token"); } @@ -2293,7 +2317,6 @@ mod tests { Request::builder() .method(Method::GET) .uri("/proxy/a%2Fb/%2525") - .header("x-api-key", "test-key") .header(SANDBOX_ID_HEADER, sandbox_id.to_string()) .header(TARGET_PORT_HEADER, upstream_addr.port().to_string()) .body(Body::empty()) @@ -2320,7 +2343,6 @@ mod tests { Request::builder() .method(Method::GET) .uri("/proxy//api") - .header("x-api-key", "test-key") .header(SANDBOX_ID_HEADER, sandbox_id.to_string()) .header(TARGET_PORT_HEADER, upstream_addr.port().to_string()) .body(Body::empty()) @@ -2347,7 +2369,6 @@ mod tests { Request::builder() .method(Method::GET) .uri("/proxy/check") - .header("x-api-key", "test-key") .header(SANDBOX_ID_HEADER, sandbox_id.to_string()) .header(TARGET_PORT_HEADER, upstream_addr.port().to_string()) .header(header::CONNECTION, "foo") @@ -2383,7 +2404,6 @@ mod tests { Request::builder() .method(Method::GET) .uri("/proxy/reject") - .header("x-api-key", "test-key") .header(SANDBOX_ID_HEADER, sandbox_id.to_string()) .header(TARGET_PORT_HEADER, upstream_addr.port().to_string()) .body(Body::empty()) @@ -2417,7 +2437,6 @@ mod tests { Request::builder() .method(Method::GET) .uri("/api/files?path=/") - .header("x-api-key", "test-key") .header(SANDBOX_ID_HEADER, sandbox_id.to_string()) .header(TARGET_PORT_HEADER, upstream_addr.port().to_string()) .body(Body::empty()) @@ -2446,7 +2465,6 @@ mod tests { Request::builder() .method(Method::GET) .uri("/envd/health") - .header("x-api-key", "test-key") .header(E2B_SANDBOX_ID_HEADER, sandbox_id.to_string()) .header(E2B_TARGET_PORT_HEADER, upstream_addr.port().to_string()) .body(Body::empty()) @@ -2469,7 +2487,7 @@ mod tests { .oneshot( Request::builder() .uri("/nonexistent/path") - .header("x-api-key", "test-key") + .header(API_KEY_HEADER, "test-key") .body(Body::empty()) .unwrap(), ) @@ -2534,7 +2552,7 @@ mod tests { Request::builder() .method(Method::GET) .uri("/health?foo=bar") - .header(TRAFFIC_ACCESS_TOKEN_HEADER, access_token) + .header(TRAFFIC_ACCESS_TOKEN_HEADER, &access_token) .header( "host", format!( @@ -2567,7 +2585,7 @@ mod tests { upstream_addr.port(), sandbox_id )) - .header("x-api-key", "test-key") + .header(TRAFFIC_ACCESS_TOKEN_HEADER, &access_token) .body(Body::empty()) .unwrap(), ) @@ -2579,7 +2597,7 @@ mod tests { let payload: Value = serde_json::from_slice(&body).unwrap(); assert_eq!(payload["path"], "/authority"); - let (app, _) = proxy_app_for_sandbox_with_domains( + let (app, access_token) = proxy_app_for_sandbox_with_domains( &sandbox_id, vec!["sandbox.example.invalid".to_string()], ) @@ -2590,7 +2608,7 @@ mod tests { Request::builder() .method(Method::GET) .uri("/proxy/health") - .header("x-api-key", "test-key") + .header(TRAFFIC_ACCESS_TOKEN_HEADER, &access_token) .header( "host", format!( @@ -2612,7 +2630,7 @@ mod tests { Request::builder() .method(Method::GET) .uri("/proxy/health") - .header("x-api-key", "test-key") + .header(TRAFFIC_ACCESS_TOKEN_HEADER, access_token) .header( "host", format!( @@ -2681,7 +2699,6 @@ mod tests { Request::builder() .method(Method::GET) .uri("/proxy/events") - .header("x-api-key", "test-key") .header(SANDBOX_ID_HEADER, sandbox_id.to_string()) .header(TARGET_PORT_HEADER, upstream_addr.port().to_string()) .body(Body::empty()) @@ -2730,7 +2747,6 @@ mod tests { Request::builder() .method(Method::POST) .uri("/proxy/upload") - .header("x-api-key", "test-key") .header(SANDBOX_ID_HEADER, sandbox_id.to_string()) .header(TARGET_PORT_HEADER, upstream_addr.port().to_string()) .body(Body::from_stream(body_stream)) @@ -2771,7 +2787,6 @@ mod tests { Request::builder() .method(Method::POST) .uri("/proxy/upload") - .header("x-api-key", "test-key") .header(SANDBOX_ID_HEADER, sandbox_id.to_string()) .header(TARGET_PORT_HEADER, upstream_addr.port().to_string()) .body(Body::from_stream(body_stream)) @@ -2798,7 +2813,6 @@ mod tests { Request::builder() .method(Method::GET) .uri("/proxy/slow") - .header("x-api-key", "test-key") .header(SANDBOX_ID_HEADER, sandbox_id.to_string()) .header(TARGET_PORT_HEADER, upstream_addr.port().to_string()) .body(Body::empty()) @@ -2835,7 +2849,6 @@ mod tests { Request::builder() .method(Method::POST) .uri(ENVD_STREAM_INPUT_PATH) - .header("x-api-key", "test-key") .header(SANDBOX_ID_HEADER, sandbox_id.to_string()) .header(TARGET_PORT_HEADER, upstream_addr.port().to_string()) .body(Body::from_stream(body_stream)) diff --git a/src/api_key.rs b/src/api_key.rs index 96ddc7048..246ee7767 100644 --- a/src/api_key.rs +++ b/src/api_key.rs @@ -1,5 +1,5 @@ use std::ffi::OsStr; -use std::fs::{self, File}; +use std::fs::{File, OpenOptions}; use std::io::{self, Read}; use std::path::Path; @@ -49,14 +49,6 @@ fn resolve_from( } let managed_path = home_path.join(MANAGED_API_KEY_RELATIVE_PATH); - match fs::symlink_metadata(&managed_path) { - Err(error) if error.kind() == io::ErrorKind::NotFound => return create(&managed_path), - Err(error) => { - return Err(error) - .with_context(|| format!("inspect managed API key {}", managed_path.display())); - } - Ok(_) => {} - } if let Some(value) = managed_secret::read(&managed_path, API_KEY_FILE_MAX_LEN).context("load managed API key")? { @@ -67,10 +59,31 @@ fn resolve_from( } fn read_external(path: &Path) -> Result { - let value = read_bounded(File::open(path)?)?; + let file = open_external(path)?; + if !file.metadata()?.is_file() { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "API key secret must be a regular file", + )); + } + let value = read_bounded(file)?; validate_file_contents(&value).map_err(io::Error::other) } +fn open_external(path: &Path) -> Result { + let mut options = OpenOptions::new(); + options.read(true); + + #[cfg(unix)] + { + use std::os::unix::fs::OpenOptionsExt; + + options.custom_flags(libc::O_NONBLOCK); + } + + options.open(path) +} + fn read_bounded(file: File) -> Result { let mut value = String::with_capacity(API_KEY_FILE_MAX_LEN); file.take((API_KEY_FILE_MAX_LEN + 1) as u64) @@ -123,7 +136,7 @@ fn create(path: &Path) -> Result { #[cfg(test)] mod tests { use super::*; - use std::sync::{Arc, Barrier}; + use std::fs; use tempfile::TempDir; @@ -144,6 +157,34 @@ mod tests { Ok(()) } + #[test] + fn external_secret_must_be_a_regular_file() -> Result<()> { + let temp = TempDir::new()?; + let external_path = temp.path().join("external"); + fs::create_dir(&external_path)?; + + let error = resolve_from(None, &external_path, temp.path()).unwrap_err(); + + assert!(error.to_string().contains("load external API key")); + assert!(format!("{error:#}").contains("must be a regular file")); + Ok(()) + } + + #[cfg(unix)] + #[test] + fn external_secret_allows_kubernetes_style_symlinks() -> Result<()> { + use std::os::unix::fs::symlink; + + let temp = TempDir::new()?; + let target_path = temp.path().join("target"); + let external_path = temp.path().join("external"); + fs::write(&target_path, format!("{TEST_KEY}\n"))?; + symlink(&target_path, &external_path)?; + + assert_eq!(resolve_from(None, &external_path, temp.path())?, TEST_KEY); + Ok(()) + } + #[test] fn managed_key_is_private_and_stable() -> Result<()> { let temp = TempDir::new()?; @@ -165,31 +206,4 @@ mod tests { } Ok(()) } - - #[test] - fn concurrent_creation_converges() -> Result<()> { - const THREADS: usize = 8; - let temp = TempDir::new()?; - let home_path = Arc::new(temp.path().to_owned()); - let external_path = Arc::new(temp.path().join("missing")); - let barrier = Arc::new(Barrier::new(THREADS)); - let handles = (0..THREADS) - .map(|_| { - let home_path = Arc::clone(&home_path); - let external_path = Arc::clone(&external_path); - let barrier = Arc::clone(&barrier); - std::thread::spawn(move || { - barrier.wait(); - resolve_from(None, &external_path, &home_path) - }) - }) - .collect::>(); - - let keys = handles - .into_iter() - .map(|handle| handle.join().expect("API key creation thread panicked")) - .collect::>>()?; - assert!(keys.iter().all(|key| key == &keys[0])); - Ok(()) - } } diff --git a/src/bin/server.rs b/src/bin/server.rs index f755bad90..f9f7d4ed7 100644 --- a/src/bin/server.rs +++ b/src/bin/server.rs @@ -76,11 +76,11 @@ async fn main() -> anyhow::Result<()> { return Ok(()); } - let api_key = agentenv::api_key::resolve(config)?; - agentenv::privileges::require_runtime_capabilities()?; agentenv::privileges::clear_ambient_capabilities()?; + let api_key = agentenv::api_key::resolve(config)?; + let addr = std::env::var("API_ADDR").unwrap_or_else(|_| "0.0.0.0:8000".to_string()); let identity = NodeIdentity::from_config(&config.node_identity); let p2p_transport = agentenv::p2p::transport_from_config(config, &identity).await?; diff --git a/src/managed_secret.rs b/src/managed_secret.rs index 4c9f7b955..cb365a2b3 100644 --- a/src/managed_secret.rs +++ b/src/managed_secret.rs @@ -12,22 +12,18 @@ pub(crate) enum CreateOutcome { } pub(crate) fn read(path: &Path, max_len: usize) -> Result> { - let parent = path.parent().context("managed secret path has no parent")?; - match validate_directory(parent) { - Ok(()) => {} + ensure_supported()?; + let parent = managed_parent(path)?; + let file = match open_secret(path) { + Ok(file) => file, Err(error) if error.kind() == io::ErrorKind::NotFound => return Ok(None), Err(error) => { - return Err(error).with_context(|| { - format!("validate managed secret directory {}", parent.display()) - }); + return Err(error).with_context(|| format!("open managed secret {}", path.display())); } - } - - match open(path) { - Ok(file) => read_file(path, file, max_len).map(Some), - Err(error) if error.kind() == io::ErrorKind::NotFound => Ok(None), - Err(error) => Err(error).with_context(|| format!("open managed secret {}", path.display())), - } + }; + validate_directory(parent) + .with_context(|| format!("validate managed secret directory {}", parent.display()))?; + read_file(path, file, max_len).map(Some) } pub(crate) fn read_file(path: &Path, mut file: File, max_len: usize) -> Result { @@ -42,7 +38,7 @@ pub(crate) fn read_file(path: &Path, mut file: File, max_len: usize) -> Result Result max_len { @@ -82,7 +80,7 @@ pub(crate) fn read_file(path: &Path, mut file: File, max_len: usize) -> Result Result { ensure_supported()?; - let parent = path.parent().context("managed secret path has no parent")?; + let parent = managed_parent(path)?; create_directory(parent)?; validate_directory_identity(parent).with_context(|| { format!( @@ -116,7 +114,7 @@ pub(crate) fn create(path: &Path, contents: &[u8]) -> Result { validate_directory(parent).with_context(|| { format!("validate managed secret directory {}", parent.display()) })?; - open(path) + open_secret(path) .map(CreateOutcome::Existing) .with_context(|| format!("open managed secret {}", path.display())) } @@ -126,36 +124,42 @@ pub(crate) fn create(path: &Path, contents: &[u8]) -> Result { } } +fn managed_parent(path: &Path) -> Result<&Path> { + let parent = path.parent().context("managed secret path has no parent")?; + if parent.file_name().is_none_or(|name| name != "secrets") { + bail!( + "managed secret parent {} must be a dedicated directory named secrets", + parent.display() + ); + } + Ok(parent) +} + fn create_directory(path: &Path) -> Result<()> { let mut builder = fs::DirBuilder::new(); builder.recursive(true); - #[cfg(unix)] { use std::os::unix::fs::DirBuilderExt; - builder.mode(0o700); } - builder .create(path) .with_context(|| format!("create managed secret directory {}", path.display())) } #[cfg(unix)] -fn open(path: &Path) -> io::Result { - let mut options = OpenOptions::new(); - options.read(true); - +fn open_secret(path: &Path) -> io::Result { use std::os::unix::fs::OpenOptionsExt; - options.custom_flags(libc::O_NOFOLLOW | libc::O_NONBLOCK); - - options.open(path) + OpenOptions::new() + .read(true) + .custom_flags(libc::O_NOFOLLOW | libc::O_NONBLOCK) + .open(path) } #[cfg(not(unix))] -fn open(_path: &Path) -> io::Result { +fn open_secret(_path: &Path) -> io::Result { Err(io::Error::new( io::ErrorKind::Unsupported, "managed secrets require Unix no-follow file semantics", @@ -174,12 +178,11 @@ fn ensure_supported() -> Result<()> { fn validate_directory(path: &Path) -> io::Result<()> { let metadata = validate_directory_identity(path)?; - #[cfg(unix)] { use std::os::unix::fs::PermissionsExt; - let mode = metadata.permissions().mode() & 0o777; + let mode = metadata.permissions().mode() & 0o7777; if mode != 0o700 { return Err(io::Error::new( io::ErrorKind::PermissionDenied, @@ -187,7 +190,6 @@ fn validate_directory(path: &Path) -> io::Result<()> { )); } } - Ok(()) } @@ -199,7 +201,6 @@ fn validate_directory_identity(path: &Path) -> io::Result { "must be a directory and not a symbolic link", )); } - #[cfg(unix)] { use std::os::unix::fs::MetadataExt; @@ -215,7 +216,6 @@ fn validate_directory_identity(path: &Path) -> io::Result { )); } } - Ok(metadata) } @@ -231,3 +231,43 @@ fn set_permissions(path: &Path, mode: u32) -> Result<()> { fn set_permissions(_path: &Path, _mode: u32) -> Result<()> { Ok(()) } + +#[cfg(all(test, unix))] +mod tests { + use std::os::unix::fs::PermissionsExt; + + use super::*; + + #[test] + fn creation_tightens_an_empty_volume_directory() -> Result<()> { + let root = tempfile::tempdir()?; + let directory = root.path().join("secrets"); + let secret = directory.join("api-key"); + fs::create_dir(&directory)?; + fs::set_permissions(&directory, fs::Permissions::from_mode(0o755))?; + + assert!(read(&secret, 64)?.is_none()); + assert!(matches!( + create(&secret, b"secret"), + Ok(CreateOutcome::Created) + )); + assert_eq!( + fs::metadata(directory)?.permissions().mode() & 0o7777, + 0o700 + ); + Ok(()) + } + + #[test] + fn creation_rejects_a_non_dedicated_parent() -> Result<()> { + let parent = tempfile::tempdir()?; + fs::set_permissions(parent.path(), fs::Permissions::from_mode(0o750))?; + + assert!(create(&parent.path().join("api-key"), b"secret").is_err()); + assert_eq!( + fs::metadata(parent.path())?.permissions().mode() & 0o7777, + 0o750 + ); + Ok(()) + } +} diff --git a/src/orchestrator/service.rs b/src/orchestrator/service.rs index 9f4a0e65c..542f145bf 100644 --- a/src/orchestrator/service.rs +++ b/src/orchestrator/service.rs @@ -162,7 +162,7 @@ where // Restore persisted sandboxes from the previous run, keeping the paused // ones (with their state) for the paused-protection reconcile below. let persisted = persister.load_all(&factory).await?; - let managed_seed_must_exist = !persisted.is_empty(); + let managed_seed_must_exist = persisted_sandboxes_require_managed_seed(&persisted); let access_tokens = tokio::task::spawn_blocking(move || { SandboxAccessTokenGenerator::load_or_create(app_config, managed_seed_must_exist) }) @@ -1548,7 +1548,7 @@ where async fn replace_sandbox_network_policy_inner( &self, sandbox_id: SandboxId, - network_policy: SandboxNetworkPolicy, + mut network_policy: SandboxNetworkPolicy, ) -> Result<()> { let metadata = self .store @@ -1561,6 +1561,7 @@ where state: metadata.state, }); } + network_policy.allow_public_traffic = metadata.network_policy.allow_public_traffic; let sandbox = { let sandboxes = self.sandboxes.read().await; @@ -2458,6 +2459,12 @@ where } } +fn persisted_sandboxes_require_managed_seed(persisted: &[SandboxMetadata]) -> bool { + persisted + .iter() + .any(|metadata| metadata.secure || !metadata.network_policy.allow_public_traffic) +} + #[cfg(test)] impl Orchestrator where @@ -2534,6 +2541,19 @@ where Ok(()) } + pub(crate) async fn set_allow_public_traffic_for_test( + &self, + sandbox_id: &SandboxId, + allow_public_traffic: bool, + ) -> Result<()> { + let Some(mut metadata) = self.store.get(sandbox_id).await? else { + return Err(OrchestratorError::SandboxNotFound(*sandbox_id)); + }; + metadata.network_policy.allow_public_traffic = allow_public_traffic; + self.store.update(metadata).await?; + Ok(()) + } + pub(crate) async fn remove_proxy_route_for_test(&self, sandbox_id: &SandboxId) { let _ = self.proxy_routes.write().await.remove(sandbox_id); } diff --git a/src/orchestrator/tests.rs b/src/orchestrator/tests.rs index af4d24ae8..58e4d0680 100644 --- a/src/orchestrator/tests.rs +++ b/src/orchestrator/tests.rs @@ -677,6 +677,22 @@ async fn new_loads_persisted_sandboxes_into_store() -> Result<()> { Ok(()) } +#[test] +fn managed_seed_continuity_only_applies_to_token_protected_sandboxes() { + let public_insecure = SandboxMetadata::default(); + assert!(!persisted_sandboxes_require_managed_seed( + std::slice::from_ref(&public_insecure) + )); + + let mut secure = public_insecure.clone(); + secure.secure = true; + assert!(persisted_sandboxes_require_managed_seed(&[secure])); + + let mut private = public_insecure; + private.network_policy.allow_public_traffic = false; + assert!(persisted_sandboxes_require_managed_seed(&[private])); +} + #[tokio::test] async fn new_returns_error_when_loading_persisted_sandboxes_fails() { setup(); @@ -1241,7 +1257,8 @@ async fn sandbox_network_policy_is_applied_and_persisted() -> Result<()> { Some(vec!["8.8.8.8".to_string()]), Some(vec!["203.0.113.0/24".to_string()]), )?, - ); + ) + .with_allow_public_traffic(false); let mut request = create_request(Some(60), &[]); request.network_policy = initial_policy.clone(); @@ -1252,8 +1269,9 @@ async fn sandbox_network_policy_is_applied_and_persisted() -> Result<()> { BaseSandboxNetworkPolicy::Allow, SandboxNetworkEgressPolicy::new(None, Some(vec!["198.51.100.0/24".to_string()]))?, ); + let expected_policy = updated_policy.clone().with_allow_public_traffic(false); orchestrator - .replace_sandbox_network_policy(created.id, updated_policy.clone()) + .replace_sandbox_network_policy(created.id, updated_policy) .await?; assert_eq!(behavior.update_network_calls(), 1); @@ -1261,7 +1279,7 @@ async fn sandbox_network_policy_is_applied_and_persisted() -> Result<()> { .get_sandbox(&created.id) .await? .expect("sandbox metadata should exist"); - assert_eq!(updated.network_policy, updated_policy); + assert_eq!(updated.network_policy, expected_policy); Ok(()) } diff --git a/src/sandbox/access.rs b/src/sandbox/access.rs index b6de8b0d4..d00e5fd9e 100644 --- a/src/sandbox/access.rs +++ b/src/sandbox/access.rs @@ -58,9 +58,7 @@ impl SandboxAccessTokenGenerator { let managed_seed_path = config.home_path.join(MANAGED_SEED_RELATIVE_PATH); let seed = resolve_seed(&managed_seed_path, managed_seed_must_exist)?; - if config.sandbox.access_token_hash_seed.is_none() - && config.cluster.scheduler_endpoint.is_some() - { + if config.cluster.scheduler_endpoint.is_some() { warn!( path = %managed_seed_path.display(), "using a node-local managed sandbox access-token seed; configure AENV_SANDBOX_ACCESS_TOKEN_HASH_SEED with the same value on every node in a clustered deployment" @@ -121,7 +119,7 @@ fn resolve_seed(managed_path: &Path, managed_seed_must_exist: bool) -> Result Result<()> { - let temp = TempDir::new()?; - let managed_path = temp.path().join(MANAGED_SEED_RELATIVE_PATH); - fs::create_dir_all(managed_path.parent().unwrap())?; - set_test_permissions(managed_path.parent().unwrap(), 0o770)?; - - let error = resolve_seed(&managed_path, false).unwrap_err(); - - assert!(format!("{error:#}").contains("permissions 0700")); - Ok(()) - } - #[cfg(unix)] #[test] fn managed_seed_directory_symlink_is_rejected() -> Result<()> { diff --git a/src/sandbox/firecracker/sandbox.rs b/src/sandbox/firecracker/sandbox.rs index a4b943cdd..52eb2e565 100644 --- a/src/sandbox/firecracker/sandbox.rs +++ b/src/sandbox/firecracker/sandbox.rs @@ -230,7 +230,8 @@ impl FirecrackerPausedState { impl PausedSandboxState for FirecrackerPausedState { fn control_plane_port(&self) -> Option { - Some(self.snapshot_config.common.control_plane_port) + let port = self.snapshot_config.common.control_plane_port; + (port != 0).then_some(port) } fn encode(&self) -> Result { diff --git a/src/sandbox/network/policy.rs b/src/sandbox/network/policy.rs index 205578de3..6c42a0210 100644 --- a/src/sandbox/network/policy.rs +++ b/src/sandbox/network/policy.rs @@ -73,20 +73,42 @@ impl SandboxNetworkEgressPolicy { } } -#[derive(Clone, Debug, Default, Eq, PartialEq, Serialize, Deserialize)] +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] pub struct SandboxNetworkPolicy { + #[serde(default = "default_allow_public_traffic")] + pub allow_public_traffic: bool, pub base_policy: BaseSandboxNetworkPolicy, pub egress: SandboxNetworkEgressPolicy, } +fn default_allow_public_traffic() -> bool { + true +} + +impl Default for SandboxNetworkPolicy { + fn default() -> Self { + Self { + allow_public_traffic: true, + base_policy: BaseSandboxNetworkPolicy::default(), + egress: SandboxNetworkEgressPolicy::default(), + } + } +} + impl SandboxNetworkPolicy { pub fn new(base_policy: BaseSandboxNetworkPolicy, egress: SandboxNetworkEgressPolicy) -> Self { Self { + allow_public_traffic: true, base_policy, egress, } } + pub fn with_allow_public_traffic(mut self, allow_public_traffic: bool) -> Self { + self.allow_public_traffic = allow_public_traffic; + self + } + pub(crate) fn runtime_policy(&self) -> Option { self.has_runtime_egress_rules().then(|| self.clone()) } @@ -335,14 +357,29 @@ mod tests { SandboxNetworkEgressPolicy::new(Some(vec!["8.8.8.8/32".to_string()]), None).unwrap(), ); + assert!(policy.allow_public_traffic); assert_eq!(policy.base_policy, BaseSandboxNetworkPolicy::Deny); assert!(policy.egress.denied_cidrs.is_empty()); assert!(policy.has_runtime_egress_rules()); } + #[test] + fn missing_ingress_policy_deserializes_as_public() { + let mut value = serde_json::to_value(SandboxNetworkPolicy::default()).unwrap(); + value + .as_object_mut() + .unwrap() + .remove("allow_public_traffic"); + + let policy: SandboxNetworkPolicy = serde_json::from_value(value).unwrap(); + + assert!(policy.allow_public_traffic); + } + #[test] fn build_rules_keeps_allow_before_deny() { let policy = SandboxNetworkPolicy { + allow_public_traffic: true, base_policy: BaseSandboxNetworkPolicy::Deny, egress: SandboxNetworkEgressPolicy { allowed_cidrs: vec!["8.8.8.8/32".to_string()], @@ -371,6 +408,7 @@ mod tests { #[test] fn build_policy_replacement_flushes_before_installing_rules() { let policy = SandboxNetworkPolicy { + allow_public_traffic: true, base_policy: BaseSandboxNetworkPolicy::Deny, egress: SandboxNetworkEgressPolicy { allowed_cidrs: vec!["8.8.8.8/32".to_string()], From c9f4d9120484ae6ab3e0ebe9894c690232f0380e Mon Sep 17 00:00:00 2001 From: Yingdi Shan <5491399+yingdi-shan@users.noreply.github.com> Date: Tue, 18 Aug 2026 14:02:19 +0000 Subject: [PATCH 7/7] fix: preserve sandbox metrics auth --- Cargo.lock | 1 - deploy/k8s/run.sh | 4 +- docs/src/SUMMARY.md | 2 +- docs/src/configuration/env-vars.md | 2 +- docs/src/deployment/docker-compose.md | 8 +- docs/src/integration/e2b.md | 7 +- .../authentication.md | 2 +- docs/src/security/secure-sandboxes.md | 2 +- scripts/tests/e2e/lib/runtime.sh | 4 +- scripts/tests/e2e/suites/09_e2b_compat.sh | 1 - services/gateway/cmd/main.go | 2 +- services/gateway/internal/server.go | 6 +- src/api/generated/src/models.rs | 2 +- src/api/impls/auth.rs | 15 +- src/api/impls/mod.rs | 5 +- src/api/impls/sandbox.rs | 4 +- src/api/openapi.yml | 2 +- src/api/proxy.rs | 68 ++--- src/api_key.rs | 243 +++++++++++------- src/bin/server.rs | 3 +- src/orchestrator/tests.rs | 11 +- src/sandbox/network/policy.rs | 18 +- storage/overlaybd/src/lsmt/file/helper.rs | 180 ++----------- storage/overlaybd/src/lsmt/file/tests.rs | 54 ---- storage/ublk-daemon/Cargo.toml | 1 - storage/ublk-daemon/src/main.rs | 4 - tests/integration/fc.rs | 4 + 27 files changed, 253 insertions(+), 402 deletions(-) rename docs/src/{configuration => security}/authentication.md (96%) diff --git a/Cargo.lock b/Cargo.lock index 44e627c28..ab0057342 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8312,7 +8312,6 @@ dependencies = [ "serde_json", "storage-util", "tempfile", - "tikv-jemallocator", "tokio", "toml", "tracing", diff --git a/deploy/k8s/run.sh b/deploy/k8s/run.sh index 940384fe4..e891b2f0e 100644 --- a/deploy/k8s/run.sh +++ b/deploy/k8s/run.sh @@ -172,8 +172,8 @@ if [[ "${MODE}" != "delete" ]]; then ensure_namespace || exit 1 bootstrap_api_key || exit 1 fi - if [[ ! "${API_KEY_VALUE}" =~ ^[A-Za-z0-9._~-]{32,4096}$ ]]; then - echo "AENV_API_KEY must contain between 32 and 4096 URL-safe characters" >&2 + if [[ ! "${API_KEY_VALUE}" =~ ^[A-Za-z0-9._~-]{32,256}$ ]]; then + echo "AENV_API_KEY must contain between 32 and 256 URL-safe characters" >&2 exit 1 fi fi diff --git a/docs/src/SUMMARY.md b/docs/src/SUMMARY.md index 74795cb26..a3494dadd 100644 --- a/docs/src/SUMMARY.md +++ b/docs/src/SUMMARY.md @@ -18,12 +18,12 @@ # Configuration -- [Authentication](./configuration/authentication.md) - [Configuration Reference](./configuration/reference.md) - [Environment Variables](./configuration/env-vars.md) # Security +- [Authentication](./security/authentication.md) - [Secure Sandboxes](./security/secure-sandboxes.md) # Core Concepts diff --git a/docs/src/configuration/env-vars.md b/docs/src/configuration/env-vars.md index a5f8c47cb..755f5a087 100644 --- a/docs/src/configuration/env-vars.md +++ b/docs/src/configuration/env-vars.md @@ -71,7 +71,7 @@ export E2B_API_KEY=${AENV_API_KEY} > `E2B_SANDBOX_URL=${E2B_API_URL}`. The explicit `/proxy` prefix > (`${E2B_API_URL}/proxy`) is still accepted for back-compat. -See [Authentication](./authentication.md) for key generation and storage. +See [Authentication](../security/authentication.md) for key generation and storage. ## Gateway and Scheduler diff --git a/docs/src/deployment/docker-compose.md b/docs/src/deployment/docker-compose.md index 4e10bbd97..22648ae97 100644 --- a/docs/src/deployment/docker-compose.md +++ b/docs/src/deployment/docker-compose.md @@ -30,9 +30,6 @@ sudo bash scripts/docker-setup.sh make deploy-up ``` -The Gateway is available at `http://127.0.0.1:8000` and forwards requests to -the backend nodes. - On first startup, the runtime nodes atomically generate one API key and sandbox access-token seed in the shared `agentenv-auth` volume. The gateway mounts that volume read-only and reads the API key; sandbox tokens are validated by the @@ -56,16 +53,13 @@ usually through wildcard DNS for `*.sandbox.example.com`. # Health check via gateway curl http://127.0.0.1:8000/health -# Cluster node snapshots via gateway -curl http://127.0.0.1:8000/nodes - # Authenticated cluster node snapshots via gateway export AENV_API_KEY="$(docker compose -f deploy/docker-compose.yml exec -T agentenv-a \ cat /workspace/env/secrets/api-key)" curl -H "X-API-Key: ${AENV_API_KEY}" http://127.0.0.1:8080/nodes # Direct health check on a backend node -curl http://127.0.0.1:8001/health +curl http://127.0.0.1:8000/health ``` ## Management Commands diff --git a/docs/src/integration/e2b.md b/docs/src/integration/e2b.md index 619bd958e..8650d8595 100644 --- a/docs/src/integration/e2b.md +++ b/docs/src/integration/e2b.md @@ -15,13 +15,12 @@ export E2B_SANDBOX_URL=${E2B_API_URL} export E2B_API_KEY=${AENV_API_KEY} ``` -No `E2B_ACCESS_TOKEN` is needed. AgentENV returns `trafficAccessToken` when -`network.allowPublicTraffic` is false and (for secure sandboxes) +AgentENV returns `trafficAccessToken` when `network.allowPublicTraffic` is false +and (for secure sandboxes) `envdAccessToken` for envd control traffic. These credentials have different headers and trust boundaries: use `e2b-traffic-access-token` for private application routes and `X-Access-Token` only for envd. Public application -routes require neither token. This is transport data, not the deprecated -user-supplied `E2B_ACCESS_TOKEN`. +routes require neither token. ### TypeScript SDK diff --git a/docs/src/configuration/authentication.md b/docs/src/security/authentication.md similarity index 96% rename from docs/src/configuration/authentication.md rename to docs/src/security/authentication.md index c8e0589b2..48dfc9619 100644 --- a/docs/src/configuration/authentication.md +++ b/docs/src/security/authentication.md @@ -38,7 +38,7 @@ If none exists, normal server startup generates and atomically stores a key at the managed path. The gateway checks only the first two sources and never generates a key. -Custom keys must contain 32 to 4096 URL-safe characters. Generated keys use an +Custom keys must contain 32 to 256 URL-safe characters. Generated keys use an E2B-compatible `e2b_` prefix. For example: ```bash diff --git a/docs/src/security/secure-sandboxes.md b/docs/src/security/secure-sandboxes.md index 764a8771a..57807973e 100644 --- a/docs/src/security/secure-sandboxes.md +++ b/docs/src/security/secure-sandboxes.md @@ -5,7 +5,7 @@ Secure sandboxes use an envd access token for control-plane communication. This > [!NOTE] > Secure mode protects envd control-plane operations. Application traffic uses > the sandbox-scoped `trafficAccessToken` described in -> [Authentication](../configuration/authentication.md). +> [Authentication](./authentication.md). Set `secure: true` when creating a sandbox through API or E2B-compatible SDKs to enable secure mode. Or use the CLI: diff --git a/scripts/tests/e2e/lib/runtime.sh b/scripts/tests/e2e/lib/runtime.sh index 3bfa0c759..f1973f118 100644 --- a/scripts/tests/e2e/lib/runtime.sh +++ b/scripts/tests/e2e/lib/runtime.sh @@ -419,7 +419,7 @@ if [[ -z "${E2E_RUNTIME_SH_LOADED:-}" ]]; then AENV_API_KEY="$(_compose_cmd exec -T agentenv-a cat /workspace/env/secrets/api-key)" || die "Failed to read the Compose deployment API key" fi - [[ "${AENV_API_KEY}" =~ ^[A-Za-z0-9._~-]{32,4096}$ ]] || + [[ "${AENV_API_KEY}" =~ ^[A-Za-z0-9._~-]{32,256}$ ]] || die "Compose deployment returned an invalid API key" export AENV_API_KEY @@ -463,7 +463,7 @@ if [[ -z "${E2E_RUNTIME_SH_LOADED:-}" ]]; then AENV_API_KEY="$(_read_k8s_api_key)" || die "Failed to read the Kubernetes deployment API key" fi - [[ "${AENV_API_KEY}" =~ ^[A-Za-z0-9._~-]{32,4096}$ ]] || + [[ "${AENV_API_KEY}" =~ ^[A-Za-z0-9._~-]{32,256}$ ]] || die "Kubernetes deployment returned an invalid API key" export AENV_API_KEY diff --git a/scripts/tests/e2e/suites/09_e2b_compat.sh b/scripts/tests/e2e/suites/09_e2b_compat.sh index b4898edaa..611b09ad6 100755 --- a/scripts/tests/e2e/suites/09_e2b_compat.sh +++ b/scripts/tests/e2e/suites/09_e2b_compat.sh @@ -11,7 +11,6 @@ log "Suite: E2B Compatibility" export E2B_API_URL="${AENV_URL}" export E2B_SANDBOX_URL="${AENV_PROXY_URL}" export E2B_API_KEY="${AENV_API_KEY}" -unset E2B_ACCESS_TOKEN export E2B_COMPAT_USER_IMAGE="${E2B_COMPAT_USER_IMAGE:-${E2E_TEMPLATE_USER_IMAGE:-ghcr.io/linuxserver/baseimage-ubuntu:noble}}" cli_available=0 diff --git a/services/gateway/cmd/main.go b/services/gateway/cmd/main.go index ea6448ea5..e7bbef260 100644 --- a/services/gateway/cmd/main.go +++ b/services/gateway/cmd/main.go @@ -28,7 +28,7 @@ import ( const ( apiKeyEnv = "AENV_API_KEY" defaultAPIKeyPath = "/run/secrets/api-key" - maxAPIKeyLen = 4096 + maxAPIKeyLen = 256 maxAPIKeyFileLen = maxAPIKeyLen + 2 ) diff --git a/services/gateway/internal/server.go b/services/gateway/internal/server.go index d94498c28..775166826 100644 --- a/services/gateway/internal/server.go +++ b/services/gateway/internal/server.go @@ -113,11 +113,7 @@ func (s *Server) Handler() http.Handler { http.Error(w, "target port header required", http.StatusBadRequest) return } - if r.URL.Path == "/metrics" { - http.NotFound(w, r) - return - } - if r.URL.Path == "/health" { + if r.URL.Path == "/health" || r.URL.Path == "/metrics" { hostRoute, hostRouteErr := parseHostRoute(r.Host, s.sandboxProxyDomains) if hostRoute != nil || hostRouteErr != nil { s.handleProxy(w, r) diff --git a/src/api/generated/src/models.rs b/src/api/generated/src/models.rs index def67a2aa..434da3440 100644 --- a/src/api/generated/src/models.rs +++ b/src/api/generated/src/models.rs @@ -5722,7 +5722,7 @@ impl std::convert::TryFrom for header::IntoHeaderValue, diff --git a/src/api/impls/auth.rs b/src/api/impls/auth.rs index 905c8fd8c..518f97e05 100644 --- a/src/api/impls/auth.rs +++ b/src/api/impls/auth.rs @@ -1,3 +1,5 @@ +use super::{ApiImpl, Claims}; +use crate::{api::proxy, types::SandboxId}; use agentenv_http_server::apis; use async_trait::async_trait; use axum::{ @@ -7,10 +9,6 @@ use axum::{ middleware::Next, response::{IntoResponse, Response}, }; -use subtle::ConstantTimeEq; - -use super::{ApiImpl, Claims}; -use crate::{api::proxy, types::SandboxId}; pub(crate) const API_KEY_HEADER: &str = "x-api-key"; pub(crate) const TRAFFIC_ACCESS_TOKEN_HEADER: &str = "e2b-traffic-access-token"; @@ -24,13 +22,8 @@ fn single_header<'a>(headers: &'a HeaderMap, name: &str) -> Option<&'a HeaderVal impl ApiImpl { pub(crate) fn has_valid_api_key(&self, headers: &HeaderMap) -> bool { - single_header(headers, API_KEY_HEADER).is_some_and(|value| { - let candidate = value.as_bytes(); - let expected = self.api_key.as_bytes(); - !expected.is_empty() - && candidate.len() == expected.len() - && bool::from(candidate.ct_eq(expected)) - }) + single_header(headers, API_KEY_HEADER) + .is_some_and(|value| self.api_key.matches(value.as_bytes())) } pub(crate) fn traffic_access_token(&self, sandbox_id: SandboxId) -> String { diff --git a/src/api/impls/mod.rs b/src/api/impls/mod.rs index fb8f2bb71..47787eb5a 100644 --- a/src/api/impls/mod.rs +++ b/src/api/impls/mod.rs @@ -13,6 +13,7 @@ use anyhow::Error as AnyhowError; use async_trait::async_trait; use super::proxy::{build_proxy_client, ProxyClient}; +use crate::api_key::ApiKey; use crate::image::ImageResolver; use crate::observability::ObservabilityService; use crate::orchestrator::Orchestrator; @@ -33,7 +34,7 @@ pub struct ApiImpl { observability: Option>, proxy_client: ProxyClient, sandbox_proxy_domains: Vec, - api_key: String, + api_key: ApiKey, } impl ApiImpl { @@ -44,7 +45,7 @@ impl ApiImpl { image_resolver: Arc, observability: Option>, sandbox_proxy_domains: Vec, - api_key: String, + api_key: ApiKey, ) -> Self { Self { orchestrator, diff --git a/src/api/impls/sandbox.rs b/src/api/impls/sandbox.rs index 73c14311e..dbd417371 100644 --- a/src/api/impls/sandbox.rs +++ b/src/api/impls/sandbox.rs @@ -372,8 +372,7 @@ fn network_policy_from_create( let allow_public_traffic = network .and_then(|network| network.allow_public_traffic) .unwrap_or(true); - let policy = SandboxNetworkPolicy::new(base_policy, egress) - .with_allow_public_traffic(allow_public_traffic); + let policy = SandboxNetworkPolicy::new(allow_public_traffic, base_policy, egress); if policy.has_domain_allow_rules() { anyhow::bail!( "domain entries in allowOut are not supported until TCP egress proxy is enabled" @@ -392,6 +391,7 @@ fn network_policy_from_update( ); } Ok(SandboxNetworkPolicy::new( + true, base_policy_from_allow_internet_access(body.allow_internet_access), policy, )) diff --git a/src/api/openapi.yml b/src/api/openapi.yml index 4c56086fc..32a4c74b7 100644 --- a/src/api/openapi.yml +++ b/src/api/openapi.yml @@ -302,7 +302,7 @@ components: allowPublicTraffic: type: boolean default: true - description: Specify if the sandbox URLs should be accessible without a traffic access token. + description: Specify if the sandbox URLs should be accessible only with authentication. allowOut: type: array description: List of allowed destinations for egress traffic. Each entry can be a CIDR block (e.g. "8.8.8.8/32"), a bare IP address (e.g. "8.8.8.8"), or a domain name (e.g. "example.com", "*.example.com"). Allowed entries always take precedence over denied entries. diff --git a/src/api/proxy.rs b/src/api/proxy.rs index 73661306d..ead177109 100644 --- a/src/api/proxy.rs +++ b/src/api/proxy.rs @@ -1232,6 +1232,7 @@ mod tests { use crate::{ api::server, + api_key::ApiKey, cfg::AppConfig, image::ImageResolver, orchestrator::{FileBackedSandboxPersister, Orchestrator}, @@ -1239,6 +1240,9 @@ mod tests { template::TemplateBuilder, }; + const TEST_API_KEY: &str = + "e2b_0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"; + #[test] fn strip_host_port_handles_dns_and_ipv6_hosts() { for (host, expected) in [ @@ -1622,7 +1626,7 @@ mod tests { } async fn build_api_with_sandbox_proxy_domains(domains: Vec) -> Arc { - build_api_with_auth(domains, "test-key").await + build_api_with_auth(domains, TEST_API_KEY).await } async fn build_api_with_auth(domains: Vec, api_key: &str) -> Arc { @@ -1644,7 +1648,7 @@ mod tests { image_resolver, None, domains, - api_key.to_string(), + ApiKey::new(api_key).unwrap(), )) } @@ -1862,7 +1866,10 @@ mod tests { vec![], vec![(header::AUTHORIZATION.as_str(), "Bearer test-key")], vec![(API_KEY_HEADER, "wrong-key")], - vec![(API_KEY_HEADER, "test-key"), (API_KEY_HEADER, "test-key")], + vec![ + (API_KEY_HEADER, TEST_API_KEY), + (API_KEY_HEADER, TEST_API_KEY), + ], ] { assert_eq!( get_status(&app, "/nonexistent/path", &headers).await, @@ -1873,7 +1880,7 @@ mod tests { for (path, headers, expected) in [ ( "/nonexistent/path", - vec![(API_KEY_HEADER, "test-key")], + vec![(API_KEY_HEADER, TEST_API_KEY)], StatusCode::NOT_FOUND, ), ("/health", vec![], StatusCode::NO_CONTENT), @@ -1885,12 +1892,6 @@ mod tests { StatusCode::UNAUTHORIZED ); - let empty_key_app = server::new(build_api_with_auth(Vec::new(), "").await); - assert_eq!( - get_status(&empty_key_app, "/sandboxes", &[(API_KEY_HEADER, "")]).await, - StatusCode::UNAUTHORIZED - ); - let sandbox_id = SandboxId::new().to_string(); let route = [ (SANDBOX_ID_HEADER, sandbox_id.as_str()), @@ -1939,7 +1940,7 @@ mod tests { for credential in [ None, - Some((API_KEY_HEADER, "test-key")), + Some((API_KEY_HEADER, TEST_API_KEY)), Some((TRAFFIC_ACCESS_TOKEN_HEADER, "incorrect")), Some((ENVD_ACCESS_TOKEN_HEADER, "envd-token")), ] { @@ -1983,10 +1984,13 @@ mod tests { (SANDBOX_ID_HEADER, sandbox_id_text.as_str()), (TARGET_PORT_HEADER, target_port.as_str()), ]; - assert_ne!( - get_status(&app, "/proxy/health", &route).await, - StatusCode::UNAUTHORIZED - ); + let envd_paths = ["/proxy/health", "/proxy/metrics"]; + for path in envd_paths { + assert_ne!( + get_status(&app, path, &route).await, + StatusCode::UNAUTHORIZED + ); + } api.orchestrator() .set_secure_for_test(&sandbox_id, true) @@ -2003,26 +2007,30 @@ mod tests { for credential in [ None, - Some((API_KEY_HEADER, "test-key")), + Some((API_KEY_HEADER, TEST_API_KEY)), Some((TRAFFIC_ACCESS_TOKEN_HEADER, traffic_token.as_str())), Some((ENVD_ACCESS_TOKEN_HEADER, "incorrect")), ] { - let mut headers = route.to_vec(); - if let Some((header_name, value)) = credential { - headers.push((header_name, value)); + for path in envd_paths { + let mut headers = route.to_vec(); + if let Some((header_name, value)) = credential { + headers.push((header_name, value)); + } + assert_eq!( + get_status(&app, path, &headers).await, + StatusCode::UNAUTHORIZED + ); } - assert_eq!( - get_status(&app, "/proxy/health", &headers).await, - StatusCode::UNAUTHORIZED - ); } let mut headers = route.to_vec(); headers.push((ENVD_ACCESS_TOKEN_HEADER, envd_token.expose())); - assert_ne!( - get_status(&app, "/proxy/health", &headers).await, - StatusCode::UNAUTHORIZED - ); + for path in envd_paths { + assert_ne!( + get_status(&app, path, &headers).await, + StatusCode::UNAUTHORIZED + ); + } } #[tokio::test] @@ -2487,7 +2495,7 @@ mod tests { .oneshot( Request::builder() .uri("/nonexistent/path") - .header(API_KEY_HEADER, "test-key") + .header(API_KEY_HEADER, TEST_API_KEY) .body(Body::empty()) .unwrap(), ) @@ -2873,7 +2881,7 @@ mod tests { .unwrap(); request .headers_mut() - .insert("x-api-key", HeaderValue::from_static("test-key")); + .insert("x-api-key", HeaderValue::from_static(TEST_API_KEY)); request.headers_mut().insert( SANDBOX_ID_HEADER, HeaderValue::from_str(&sandbox_id.to_string()).unwrap(), @@ -2948,7 +2956,7 @@ mod tests { .unwrap(); request .headers_mut() - .insert("x-api-key", HeaderValue::from_static("test-key")); + .insert("x-api-key", HeaderValue::from_static(TEST_API_KEY)); request.headers_mut().insert( SANDBOX_ID_HEADER, HeaderValue::from_str(&sandbox_id.to_string()).unwrap(), diff --git a/src/api_key.rs b/src/api_key.rs index 246ee7767..62dd24907 100644 --- a/src/api_key.rs +++ b/src/api_key.rs @@ -1,10 +1,12 @@ use std::ffi::OsStr; +use std::fmt; use std::fs::{File, OpenOptions}; use std::io::{self, Read}; use std::path::Path; use anyhow::{bail, Context, Result}; use rand::{rngs::SysRng, TryRng}; +use subtle::ConstantTimeEq; use tracing::info; use crate::cfg::AppConfig; @@ -13,122 +15,138 @@ use crate::managed_secret::{self, CreateOutcome}; const API_KEY_ENV: &str = "AENV_API_KEY"; const EXTERNAL_API_KEY_PATH: &str = "/run/secrets/api-key"; const MANAGED_API_KEY_RELATIVE_PATH: &str = "secrets/api-key"; -const API_KEY_MAX_LEN: usize = 4096; +const API_KEY_MAX_LEN: usize = 256; const API_KEY_FILE_MAX_LEN: usize = API_KEY_MAX_LEN + 2; const GENERATED_API_KEY_PREFIX: &str = "e2b_"; -pub fn resolve(config: &AppConfig) -> Result { - resolve_from( - std::env::var_os(API_KEY_ENV).as_deref(), - Path::new(EXTERNAL_API_KEY_PATH), - &config.home_path, - ) +#[derive(Clone)] +pub struct ApiKey(String); + +impl fmt::Debug for ApiKey { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str("ApiKey([REDACTED])") + } } -fn resolve_from( - explicit: Option<&OsStr>, - external_path: &Path, - home_path: &Path, -) -> Result { - if let Some(explicit) = explicit { - return validate( - explicit - .to_str() - .context("AENV_API_KEY must contain valid UTF-8")?, +impl ApiKey { + pub fn resolve(config: &AppConfig) -> Result { + Self::resolve_from( + std::env::var_os(API_KEY_ENV).as_deref(), + Path::new(EXTERNAL_API_KEY_PATH), + &config.home_path, ) - .context("invalid AENV_API_KEY"); } - match read_external(external_path) { - Ok(key) => { - info!(path = %external_path.display(), "loaded API key from external secret"); - return Ok(key); + pub fn new(value: impl Into) -> Result { + let value = value.into(); + if !(32..=API_KEY_MAX_LEN).contains(&value.len()) + || !value.bytes().all(|byte| { + byte.is_ascii_alphanumeric() || matches!(byte, b'.' | b'_' | b'~' | b'-') + }) + { + bail!("API key must contain between 32 and {API_KEY_MAX_LEN} URL-safe characters"); } - Err(error) if error.kind() == io::ErrorKind::NotFound => {} - Err(error) => return Err(error).context("load external API key"), + Ok(Self(value)) } - let managed_path = home_path.join(MANAGED_API_KEY_RELATIVE_PATH); - if let Some(value) = - managed_secret::read(&managed_path, API_KEY_FILE_MAX_LEN).context("load managed API key")? - { - return validate_file_contents(&value).context("invalid managed API key"); + pub(crate) fn matches(&self, candidate: &[u8]) -> bool { + candidate.len() == self.0.len() && bool::from(candidate.ct_eq(self.0.as_bytes())) } - create(&managed_path) -} + fn resolve_from( + explicit: Option<&OsStr>, + external_path: &Path, + home_path: &Path, + ) -> Result { + if let Some(explicit) = explicit { + return Self::new( + explicit + .to_str() + .context("AENV_API_KEY must contain valid UTF-8")?, + ) + .context("invalid AENV_API_KEY"); + } -fn read_external(path: &Path) -> Result { - let file = open_external(path)?; - if !file.metadata()?.is_file() { - return Err(io::Error::new( - io::ErrorKind::InvalidData, - "API key secret must be a regular file", - )); - } - let value = read_bounded(file)?; - validate_file_contents(&value).map_err(io::Error::other) -} + match Self::read_external(external_path) { + Ok(key) => { + info!(path = %external_path.display(), "loaded API key from external secret"); + return Ok(key); + } + Err(error) if error.kind() == io::ErrorKind::NotFound => {} + Err(error) => return Err(error).context("load external API key"), + } -fn open_external(path: &Path) -> Result { - let mut options = OpenOptions::new(); - options.read(true); + let managed_path = home_path.join(MANAGED_API_KEY_RELATIVE_PATH); + if let Some(value) = managed_secret::read(&managed_path, API_KEY_FILE_MAX_LEN) + .context("load managed API key")? + { + return Self::from_file_contents(&value).context("invalid managed API key"); + } - #[cfg(unix)] - { - use std::os::unix::fs::OpenOptionsExt; + Self::create(&managed_path) + } - options.custom_flags(libc::O_NONBLOCK); + fn read_external(path: &Path) -> Result { + let file = Self::open_external(path)?; + if !file.metadata()?.is_file() { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "API key secret must be a regular file", + )); + } + let value = Self::read_bounded(file)?; + Self::from_file_contents(&value).map_err(io::Error::other) } - options.open(path) -} + fn open_external(path: &Path) -> Result { + let mut options = OpenOptions::new(); + options.read(true); -fn read_bounded(file: File) -> Result { - let mut value = String::with_capacity(API_KEY_FILE_MAX_LEN); - file.take((API_KEY_FILE_MAX_LEN + 1) as u64) - .read_to_string(&mut value)?; - if value.len() > API_KEY_FILE_MAX_LEN { - return Err(io::Error::new( - io::ErrorKind::InvalidData, - format!("API key file must be at most {API_KEY_FILE_MAX_LEN} bytes"), - )); - } - Ok(value) -} + #[cfg(unix)] + { + use std::os::unix::fs::OpenOptionsExt; -fn validate(value: &str) -> Result { - if !(32..=API_KEY_MAX_LEN).contains(&value.len()) - || !value - .bytes() - .all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'.' | b'_' | b'~' | b'-')) - { - bail!("API key must contain between 32 and {API_KEY_MAX_LEN} URL-safe characters"); - } - Ok(value.to_owned()) -} + options.custom_flags(libc::O_NONBLOCK); + } -fn validate_file_contents(value: &str) -> Result { - let value = value.strip_suffix('\n').unwrap_or(value); - validate(value.strip_suffix('\r').unwrap_or(value)) -} + options.open(path) + } -fn create(path: &Path) -> Result { - let mut random = [0_u8; 32]; - SysRng - .try_fill_bytes(&mut random) - .context("generate managed API key")?; - let key = format!("{GENERATED_API_KEY_PREFIX}{}", hex::encode(random)); - - match managed_secret::create(path, format!("{key}\n").as_bytes())? { - CreateOutcome::Created => { - info!(path = %path.display(), "generated managed API key"); - Ok(key) + fn read_bounded(file: File) -> Result { + let mut value = String::with_capacity(API_KEY_FILE_MAX_LEN); + file.take((API_KEY_FILE_MAX_LEN + 1) as u64) + .read_to_string(&mut value)?; + if value.len() > API_KEY_FILE_MAX_LEN { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + format!("API key file must be at most {API_KEY_FILE_MAX_LEN} bytes"), + )); } - CreateOutcome::Existing(file) => { - let value = managed_secret::read_file(path, file, API_KEY_FILE_MAX_LEN) - .context("load concurrently generated API key")?; - validate_file_contents(&value).context("invalid concurrently generated API key") + Ok(value) + } + + fn from_file_contents(value: &str) -> Result { + let value = value.strip_suffix('\n').unwrap_or(value); + Self::new(value.strip_suffix('\r').unwrap_or(value)) + } + + fn create(path: &Path) -> Result { + let mut random = [0_u8; 32]; + SysRng + .try_fill_bytes(&mut random) + .context("generate managed API key")?; + let key = format!("{GENERATED_API_KEY_PREFIX}{}", hex::encode(random)); + + match managed_secret::create(path, format!("{key}\n").as_bytes())? { + CreateOutcome::Created => { + info!(path = %path.display(), "generated managed API key"); + Self::new(key) + } + CreateOutcome::Existing(file) => { + let value = managed_secret::read_file(path, file, API_KEY_FILE_MAX_LEN) + .context("load concurrently generated API key")?; + Self::from_file_contents(&value).context("invalid concurrently generated API key") + } } } } @@ -149,10 +167,13 @@ mod tests { fs::write(&external_path, format!("{TEST_KEY}\n"))?; assert_eq!( - resolve_from(Some(OsStr::new(TEST_KEY)), &external_path, temp.path())?, - TEST_KEY + ApiKey::resolve_from(Some(OsStr::new(TEST_KEY)), &external_path, temp.path())?.0, + TEST_KEY, + ); + assert_eq!( + ApiKey::resolve_from(None, &external_path, temp.path())?.0, + TEST_KEY, ); - assert_eq!(resolve_from(None, &external_path, temp.path())?, TEST_KEY); assert!(!temp.path().join(MANAGED_API_KEY_RELATIVE_PATH).exists()); Ok(()) } @@ -163,7 +184,7 @@ mod tests { let external_path = temp.path().join("external"); fs::create_dir(&external_path)?; - let error = resolve_from(None, &external_path, temp.path()).unwrap_err(); + let error = ApiKey::resolve_from(None, &external_path, temp.path()).unwrap_err(); assert!(error.to_string().contains("load external API key")); assert!(format!("{error:#}").contains("must be a regular file")); @@ -181,7 +202,10 @@ mod tests { fs::write(&target_path, format!("{TEST_KEY}\n"))?; symlink(&target_path, &external_path)?; - assert_eq!(resolve_from(None, &external_path, temp.path())?, TEST_KEY); + assert_eq!( + ApiKey::resolve_from(None, &external_path, temp.path())?.0, + TEST_KEY, + ); Ok(()) } @@ -189,10 +213,11 @@ mod tests { fn managed_key_is_private_and_stable() -> Result<()> { let temp = TempDir::new()?; let missing_external = temp.path().join("missing"); - let first = resolve_from(None, &missing_external, temp.path())?; + let first = ApiKey::resolve_from(None, &missing_external, temp.path())?; - assert_eq!(resolve_from(None, &missing_external, temp.path())?, first); - assert!(first.starts_with(GENERATED_API_KEY_PREFIX)); + let second = ApiKey::resolve_from(None, &missing_external, temp.path())?; + assert!(second.matches(first.0.as_bytes())); + assert!(first.0.starts_with(GENERATED_API_KEY_PREFIX)); #[cfg(unix)] { use std::os::unix::fs::PermissionsExt; @@ -206,4 +231,22 @@ mod tests { } Ok(()) } + + #[test] + fn validation_enforces_length_and_url_safe_characters() { + assert!(ApiKey::new("a".repeat(32)).is_ok()); + assert!(ApiKey::new("a".repeat(API_KEY_MAX_LEN)).is_ok()); + assert!(ApiKey::new("a".repeat(31)).is_err()); + assert!(ApiKey::new("a".repeat(API_KEY_MAX_LEN + 1)).is_err()); + assert!(ApiKey::new(format!("{}!", "a".repeat(31))).is_err()); + } + + #[test] + fn matches_uses_the_validated_key() -> Result<()> { + let key = ApiKey::new(TEST_KEY)?; + + assert!(key.matches(TEST_KEY.as_bytes())); + assert!(!key.matches(b"wrong-key")); + Ok(()) + } } diff --git a/src/bin/server.rs b/src/bin/server.rs index f9f7d4ed7..0a32d456a 100644 --- a/src/bin/server.rs +++ b/src/bin/server.rs @@ -1,6 +1,7 @@ use std::sync::{Arc, RwLock}; use agentenv::api::{server, ApiImpl}; +use agentenv::api_key::ApiKey; use agentenv::identity::NodeIdentity; use agentenv::image::ImageResolver; use agentenv::observability::{ObservabilityReporter, ObservabilityService}; @@ -79,7 +80,7 @@ async fn main() -> anyhow::Result<()> { agentenv::privileges::require_runtime_capabilities()?; agentenv::privileges::clear_ambient_capabilities()?; - let api_key = agentenv::api_key::resolve(config)?; + let api_key = ApiKey::resolve(config)?; let addr = std::env::var("API_ADDR").unwrap_or_else(|_| "0.0.0.0:8000".to_string()); let identity = NodeIdentity::from_config(&config.node_identity); diff --git a/src/orchestrator/tests.rs b/src/orchestrator/tests.rs index 58e4d0680..0f89f19f6 100644 --- a/src/orchestrator/tests.rs +++ b/src/orchestrator/tests.rs @@ -1252,13 +1252,13 @@ async fn sandbox_network_policy_is_applied_and_persisted() -> Result<()> { make_orchestrator_with_factory(MockBackendFactory::with_behavior(Arc::clone(&behavior))) .await; let initial_policy = SandboxNetworkPolicy::new( + false, BaseSandboxNetworkPolicy::Deny, SandboxNetworkEgressPolicy::new( Some(vec!["8.8.8.8".to_string()]), Some(vec!["203.0.113.0/24".to_string()]), )?, - ) - .with_allow_public_traffic(false); + ); let mut request = create_request(Some(60), &[]); request.network_policy = initial_policy.clone(); @@ -1266,10 +1266,15 @@ async fn sandbox_network_policy_is_applied_and_persisted() -> Result<()> { assert_eq!(created.network_policy, initial_policy); let updated_policy = SandboxNetworkPolicy::new( + true, + BaseSandboxNetworkPolicy::Allow, + SandboxNetworkEgressPolicy::new(None, Some(vec!["198.51.100.0/24".to_string()]))?, + ); + let expected_policy = SandboxNetworkPolicy::new( + false, BaseSandboxNetworkPolicy::Allow, SandboxNetworkEgressPolicy::new(None, Some(vec!["198.51.100.0/24".to_string()]))?, ); - let expected_policy = updated_policy.clone().with_allow_public_traffic(false); orchestrator .replace_sandbox_network_policy(created.id, updated_policy) .await?; diff --git a/src/sandbox/network/policy.rs b/src/sandbox/network/policy.rs index 6c42a0210..c58507cfb 100644 --- a/src/sandbox/network/policy.rs +++ b/src/sandbox/network/policy.rs @@ -96,19 +96,18 @@ impl Default for SandboxNetworkPolicy { } impl SandboxNetworkPolicy { - pub fn new(base_policy: BaseSandboxNetworkPolicy, egress: SandboxNetworkEgressPolicy) -> Self { + pub fn new( + allow_public_traffic: bool, + base_policy: BaseSandboxNetworkPolicy, + egress: SandboxNetworkEgressPolicy, + ) -> Self { Self { - allow_public_traffic: true, + allow_public_traffic, base_policy, egress, } } - pub fn with_allow_public_traffic(mut self, allow_public_traffic: bool) -> Self { - self.allow_public_traffic = allow_public_traffic; - self - } - pub(crate) fn runtime_policy(&self) -> Option { self.has_runtime_egress_rules().then(|| self.clone()) } @@ -351,13 +350,14 @@ mod tests { } #[test] - fn new_sets_base_policy() { + fn new_sets_explicit_policy() { let policy = SandboxNetworkPolicy::new( + false, BaseSandboxNetworkPolicy::Deny, SandboxNetworkEgressPolicy::new(Some(vec!["8.8.8.8/32".to_string()]), None).unwrap(), ); - assert!(policy.allow_public_traffic); + assert!(!policy.allow_public_traffic); assert_eq!(policy.base_policy, BaseSandboxNetworkPolicy::Deny); assert!(policy.egress.denied_cidrs.is_empty()); assert!(policy.has_runtime_egress_rules()); diff --git a/storage/overlaybd/src/lsmt/file/helper.rs b/storage/overlaybd/src/lsmt/file/helper.rs index 2d275c52b..e61eabf24 100644 --- a/storage/overlaybd/src/lsmt/file/helper.rs +++ b/storage/overlaybd/src/lsmt/file/helper.rs @@ -9,7 +9,7 @@ use std::io::{self, ErrorKind}; use std::mem::size_of; use std::os::unix::fs::{FileExt, OpenOptionsExt}; use std::path::{Path, PathBuf}; -use std::sync::{Arc, Mutex as StdMutex, OnceLock, Weak}; +use std::sync::{Arc, OnceLock, Weak}; use std::time::{SystemTime, UNIX_EPOCH}; use tokio::sync::{Mutex, OwnedMutexGuard}; use uuid::Uuid; @@ -472,157 +472,50 @@ pub(super) fn decode_premerged_index_artifact( ))) } -/// Growth in artifact bytes that amortizes one full-dir prune scan. -const PREMERGED_INDEX_PRUNE_SCAN_FRACTION: u64 = 8; - -/// Per-cache-dir growth accounting and scan serialization. -pub(super) struct PremergedIndexPruneState { - account: StdMutex, -} - -struct PremergedIndexScanAccount { - bytes_since_scan: u64, - scan_in_progress: bool, -} - -pub(super) async fn premerged_index_prune_state(cache_dir: &Path) -> Arc { - static STATES: OnceLock>>> = - OnceLock::new(); - STATES - .get_or_init(|| Mutex::new(HashMap::new())) - .lock() - .await - .entry(cache_dir.to_path_buf()) - .or_insert_with(|| { - Arc::new(PremergedIndexPruneState { - account: StdMutex::new(PremergedIndexScanAccount { - bytes_since_scan: 0, - scan_in_progress: false, - }), - }) - }) - .clone() -} - -impl PremergedIndexPruneState { - /// Charge `written_bytes` and elect the single caller that runs the next scan. - pub(super) fn elect_scan(&self, written_bytes: u64, max_dir_bytes: u64) -> bool { - let threshold = (max_dir_bytes / PREMERGED_INDEX_PRUNE_SCAN_FRACTION).max(1); - let mut account = self.account.lock().unwrap_or_else(|err| err.into_inner()); - account.bytes_since_scan = account.bytes_since_scan.saturating_add(written_bytes); - if account.bytes_since_scan >= threshold && !account.scan_in_progress { - account.bytes_since_scan = 0; - account.scan_in_progress = true; - return true; - } - false - } - - /// End the scan; returns true when growth charged while it ran crosses - /// the trigger and a follow-up scan should run. - pub(super) fn scan_finished(&self, max_dir_bytes: u64) -> bool { - let threshold = (max_dir_bytes / PREMERGED_INDEX_PRUNE_SCAN_FRACTION).max(1); - let mut account = self.account.lock().unwrap_or_else(|err| err.into_inner()); - account.scan_in_progress = false; - if account.bytes_since_scan >= threshold { - account.bytes_since_scan = 0; - account.scan_in_progress = true; - return true; - } - false - } - - /// Release a failed scan; concurrent charges are kept so the next write - /// re-elects. - pub(super) fn scan_aborted(&self) { - let mut account = self.account.lock().unwrap_or_else(|err| err.into_inner()); - account.scan_in_progress = false; - } -} - -/// One blocking task for the whole scan: async `tokio::fs` would cost a -/// blocking-pool round-trip per directory entry. async fn prune_premerged_index_dir(dir: &Path, max_dir_bytes: u64) -> Result<()> { - let dir = dir.to_path_buf(); - tokio::task::spawn_blocking(move || prune_premerged_index_dir_blocking(&dir, max_dir_bytes)) - .await - .context("join premerged index cache prune task")? -} - -struct PremergedArtifactEntry { - path: PathBuf, - len: u64, - modified: SystemTime, -} - -/// Delete oldest premerged index artifacts until `dir` fits `max_dir_bytes`. -fn prune_premerged_index_dir_blocking(dir: &Path, max_dir_bytes: u64) -> Result<()> { let mut entries = Vec::new(); let mut total = 0u64; - for entry in std::fs::read_dir(dir) - .with_context(|| format!("read premerged index cache dir {}", dir.display()))? - { - let entry = entry?; + let mut reader = tokio::fs::read_dir(dir).await?; + + while let Some(entry) = reader.next_entry().await? { let path = entry.path(); if path.extension().and_then(|v| v.to_str()) != Some(PREMERGED_INDEX_EXT) { continue; } - let metadata = match entry.metadata() { - Ok(metadata) => metadata, - // Entries deleted concurrently with the scan only shrink it. - Err(err) if err.kind() == ErrorKind::NotFound => continue, - Err(err) => { - return Err(err).with_context(|| format!("stat {}", path.display())); - } - }; + let metadata = entry.metadata().await?; if !metadata.is_file() { continue; } let len = metadata.len(); let modified = metadata.modified().unwrap_or(UNIX_EPOCH); total = total.saturating_add(len); - entries.push(PremergedArtifactEntry { - path, - len, - modified, - }); + entries.push((path, len, modified)); } if total <= max_dir_bytes { return Ok(()); } - entries.sort_by_key(|entry| entry.modified); - for entry in entries { + entries.sort_by_key(|(_, _, modified): &(PathBuf, u64, SystemTime)| *modified); + for (path, len, _) in entries { if total <= max_dir_bytes { break; } - match std::fs::remove_file(&entry.path) { - Ok(()) => total = total.saturating_sub(entry.len), - // Already removed by someone else since the scan: those bytes - // left the dir too, so count them as freed. - Err(err) if err.kind() == ErrorKind::NotFound => { - total = total.saturating_sub(entry.len); - } + match tokio::fs::remove_file(&path).await { + Ok(()) => total = total.saturating_sub(len), Err(err) => { - tracing::warn!( - ?err, - path = %entry.path.display(), - "remove premerged index artifact failed" - ) + tracing::warn!(?err, path = %path.display(), "remove premerged index artifact failed") } } } Ok(()) } -/// Write the artifact atomically (tmp file + rename) and return its size in -/// bytes so callers can account cache growth. async fn write_premerged_index_artifact( cache_dir: &Path, key: &PremergedIndexCacheKey, index: &ReadOnlyIndex, -) -> Result { +) -> Result<()> { let dir = cache_dir.join(PREMERGED_INDEX_DIR); tokio::fs::create_dir_all(&dir) .await @@ -657,7 +550,7 @@ async fn write_premerged_index_artifact( }); } - Ok(artifact.len() as u64) + Ok(()) } pub(super) async fn try_read_premerged_index_artifact( @@ -689,34 +582,6 @@ pub(super) async fn try_read_premerged_index_artifact( } } -/// Charge `written` artifact bytes for `cache_dir` and run prune scans -/// while the growth trigger keeps electing. -async fn prune_premerged_index_cache(cache_dir: &Path, written: u64, max_dir_bytes: u64) { - let state = premerged_index_prune_state(cache_dir).await; - if !state.elect_scan(written, max_dir_bytes) { - return; - } - let dir = cache_dir.join(PREMERGED_INDEX_DIR); - loop { - match prune_premerged_index_dir(&dir, max_dir_bytes).await { - Ok(()) => { - if !state.scan_finished(max_dir_bytes) { - return; - } - } - Err(err) => { - tracing::warn!( - ?err, - path = %dir.display(), - "prune premerged index cache dir failed" - ); - state.scan_aborted(); - return; - } - } - } -} - pub(super) fn spawn_premerged_index_artifact_write( cache_dir: PathBuf, key: PremergedIndexCacheKey, @@ -734,15 +599,18 @@ pub(super) fn spawn_premerged_index_artifact_write( "write premerged index artifact failed" ); } - // Release the merged index and the digest lock before the prune tail: - // the index can be hundreds of MB and the prune scan may pin it for - // the whole scan, while the held lock would block the next writer for - // the same digest. - drop(merged); + let key_digest = key.digest_hex.clone(); drop(guard); - release_premerged_index_lock(&key.digest_hex, &lock).await; - if let Ok(written) = write_result { - prune_premerged_index_cache(&cache_dir, written, max_dir_bytes).await; + release_premerged_index_lock(&key_digest, &lock).await; + if write_result.is_ok() { + let dir = cache_dir.join(PREMERGED_INDEX_DIR); + if let Err(err) = prune_premerged_index_dir(&dir, max_dir_bytes).await { + tracing::warn!( + ?err, + path = %dir.display(), + "prune premerged index cache dir failed" + ); + } } }); } diff --git a/storage/overlaybd/src/lsmt/file/tests.rs b/storage/overlaybd/src/lsmt/file/tests.rs index ea93c6940..907ccd0f5 100644 --- a/storage/overlaybd/src/lsmt/file/tests.rs +++ b/storage/overlaybd/src/lsmt/file/tests.rs @@ -2536,60 +2536,6 @@ async fn test_premerged_index_lock_map_drops_idle_entries() { release_premerged_index_lock(&stale_key, &replacement).await; } -#[tokio::test] -async fn test_premerged_index_prune_state_is_per_dir_and_elects_one_scan() { - let temp_dir = TempDir::new().unwrap(); - let cache_a = temp_dir.path().join("cache-a"); - let cache_b = temp_dir.path().join("cache-b"); - - let a = premerged_index_prune_state(&cache_a).await; - assert!(Arc::ptr_eq( - &a, - &premerged_index_prune_state(&cache_a).await - )); - let b = premerged_index_prune_state(&cache_b).await; - assert!(!Arc::ptr_eq(&a, &b)); - - // Budget 64 triggers one scan per 8 new bytes (`max_dir_bytes` / 8). - let budget = 64; - assert!(!a.elect_scan(7, budget)); - // Growth charged to A leaves B below B's own trigger. - assert!(!b.elect_scan(7, budget)); - // The winner's charge resets: the next scan needs a fresh trigger of - // post-election growth. - assert!(a.elect_scan(1, budget)); - - // Crossings while a scan runs lose the election but keep their charge: - // when they cross the trigger, scan_finished chains one follow-up scan - // instead of waiting for new writes. - assert!(!a.elect_scan(8, budget)); - assert!(a.scan_finished(budget)); - assert!(!a.elect_scan(1, budget)); - // Below the trigger: the chain ends and the gate is released. - assert!(!a.scan_finished(budget)); - assert!(a.elect_scan(8, budget)); - // No growth during the scan: no follow-up. - assert!(!a.scan_finished(budget)); - - // A failed scan releases the gate but keeps concurrent charges. - assert!(a.elect_scan(8, budget)); - assert!(!a.elect_scan(7, budget)); - a.scan_aborted(); - assert!(a.elect_scan(1, budget)); - assert!(!a.scan_finished(budget)); - - // A sub-fraction budget floors the trigger at one byte instead of zero - // (which would elect a scan per write). - assert!(a.elect_scan(1, 0)); - assert!(!a.scan_finished(0)); - - // Saturating accounting: a u64::MAX charge on a near-trigger counter - // must cross the trigger, not wrap 7 + MAX back below it. - assert!(!a.elect_scan(7, budget)); - assert!(a.elect_scan(u64::MAX, budget)); - assert!(!a.scan_finished(budget)); -} - fn premerged_artifact_count(cache_dir: &std::path::Path) -> usize { let dir = cache_dir.join(PREMERGED_INDEX_DIR); match std::fs::read_dir(dir) { diff --git a/storage/ublk-daemon/Cargo.toml b/storage/ublk-daemon/Cargo.toml index 61b53b860..69736265b 100644 --- a/storage/ublk-daemon/Cargo.toml +++ b/storage/ublk-daemon/Cargo.toml @@ -24,7 +24,6 @@ tokio = { version = "1.44.2", features = ["full"] } tracing = "0.1.41" tracing-log = "0.2.0" reqwest = { version = "0.13", default-features = false, features = ["rustls", "json"] } -tikv-jemallocator = { version = "0.6", features = ["background_threads"] } [dev-dependencies] tempfile = "3" diff --git a/storage/ublk-daemon/src/main.rs b/storage/ublk-daemon/src/main.rs index 260b70702..4a9450263 100644 --- a/storage/ublk-daemon/src/main.rs +++ b/storage/ublk-daemon/src/main.rs @@ -15,10 +15,6 @@ use uvm_ublk_daemon::{server::UblkDaemonServer, ResizeToolSpec}; mod metrics_server; -// Mirrors src/bin/server.rs. -#[global_allocator] -static ALLOC: tikv_jemallocator::Jemalloc = tikv_jemallocator::Jemalloc; - #[derive(Debug, Parser)] #[command( name = "uvm-ublk-daemon", diff --git a/tests/integration/fc.rs b/tests/integration/fc.rs index 80cccc2c9..981c6136f 100644 --- a/tests/integration/fc.rs +++ b/tests/integration/fc.rs @@ -537,6 +537,7 @@ async fn microvm_network_policy_controls_egress() -> Result<()> { common::setup().await; let mut sandbox_config = common::default_sandbox_config()?; sandbox_config.common.network_policy = Some(SandboxNetworkPolicy::new( + true, BaseSandboxNetworkPolicy::Deny, SandboxNetworkEgressPolicy::new(Some(vec!["8.8.8.8".to_string()]), None)?, )); @@ -549,6 +550,7 @@ async fn microvm_network_policy_controls_egress() -> Result<()> { sandbox .update_network_policy(Some(SandboxNetworkPolicy::new( + true, BaseSandboxNetworkPolicy::Deny, SandboxNetworkEgressPolicy::new(Some(vec!["1.1.1.1".to_string()]), None)?, ))) @@ -559,6 +561,7 @@ async fn microvm_network_policy_controls_egress() -> Result<()> { sandbox .update_network_policy(Some(SandboxNetworkPolicy::new( + true, BaseSandboxNetworkPolicy::Allow, SandboxNetworkEgressPolicy::new( Some(vec!["8.8.8.8".to_string()]), @@ -578,6 +581,7 @@ async fn microvm_network_policy_controls_egress() -> Result<()> { sandbox .update_network_policy(Some(SandboxNetworkPolicy::new( + true, BaseSandboxNetworkPolicy::Allow, SandboxNetworkEgressPolicy::new(Some(vec!["10.0.0.0/8".to_string()]), None)?, )))