From 917e177d592c929dda44f418cd2ca448677481d2 Mon Sep 17 00:00:00 2001 From: Kevin Date: Tue, 28 Jul 2026 05:41:42 -0500 Subject: [PATCH] feat(agent): share TLS client trust policy --- CHANGELOG.md | 15 +- Cargo.lock | 118 +- Cargo.toml | 17 +- README.md | 74 +- .../container-agent-access-flow-tls.toml | 12 +- examples/docker/gateway-access-flow-tls.toml | 12 +- scripts/test-access-runtime-pin.py | 1 + scripts/validate-access-runtime-pin.py | 7 +- .../run-tls-access-flow-cross-host-smoke.sh | 12 +- .../run-tls-access-flow-stack-smoke.sh | 12 +- src/agent/lifecycle.rs | 9 +- src/agent/relay.rs | 654 ++++- src/agent/relay_transport.rs | 2353 +++++++---------- src/agent_control.rs | 4 + src/config.rs | 9 +- src/config/agent.rs | 139 +- src/config/tests.rs | 98 +- tests/agent_control.rs | 27 +- tests/assets.rs | 4 +- tests/example_configs.rs | 21 +- 20 files changed, 1916 insertions(+), 1682 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 447f0e4..d8d05e1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,14 +2,10 @@ ## [Unreleased] -- Embedded the shared Access Flow relay in `aw-container-agent` behind strict - typed Unix-route configuration, synthetic dependency readiness, checked - process resource budgets, and admission-first bounded shutdown. The - integrated Apple host-proxy profile no longer mounts or supervises a - standalone relay executable or JSON file. - ### Breaking Changes +- Replaced the nested Access Flow TLS `pem_bundle` trust table with required flat + `trust` and conditional `ca_certificate` fields. - Existing generated identity-token files are now validated byte-for-byte as 32-4096 RFC 9110 `tchar` bytes. Files from older releases that end in a newline, or otherwise contain non-bearer bytes, fail container startup. @@ -26,6 +22,13 @@ ### Added +- Added shared `system`, `custom`, `system_plus_custom`, and `insecure` TLS + client trust modes for remote Access Flow routes. +- Embedded the shared Access Flow relay in `aw-container-agent` behind strict + typed Unix-route configuration, synthetic dependency readiness, checked + process resource budgets, and admission-first bounded shutdown. The + integrated Apple host-proxy profile no longer mounts or supervises a + standalone relay executable or JSON file. - Access Flow relay routes now support an exact server-authenticated TLS 1.3 transport with explicit address, independent verification name, bounded PEM trust, immutable Runtime pins, mixed Unix/TLS dispatch, and no fallback, diff --git a/Cargo.lock b/Cargo.lock index 2225c5e..8bcfa7f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5,12 +5,12 @@ version = 4 [[package]] name = "access-async-contracts" version = "0.1.0" -source = "git+https://github.com/kcosr/access-runtime.git?rev=c1031b70ab4dd9f372622d1e3a0e68da49971720#c1031b70ab4dd9f372622d1e3a0e68da49971720" +source = "git+https://github.com/kcosr/access-runtime.git?rev=62fe64934aa8e39390aa3a6902b5dcd747dc41cd#62fe64934aa8e39390aa3a6902b5dcd747dc41cd" [[package]] name = "access-flow" version = "0.1.0" -source = "git+https://github.com/kcosr/access-runtime.git?rev=c1031b70ab4dd9f372622d1e3a0e68da49971720#c1031b70ab4dd9f372622d1e3a0e68da49971720" +source = "git+https://github.com/kcosr/access-runtime.git?rev=62fe64934aa8e39390aa3a6902b5dcd747dc41cd#62fe64934aa8e39390aa3a6902b5dcd747dc41cd" dependencies = [ "access-async-contracts", "access-identity", @@ -21,7 +21,7 @@ dependencies = [ [[package]] name = "access-flow-conformance" version = "0.1.0" -source = "git+https://github.com/kcosr/access-runtime.git?rev=c1031b70ab4dd9f372622d1e3a0e68da49971720#c1031b70ab4dd9f372622d1e3a0e68da49971720" +source = "git+https://github.com/kcosr/access-runtime.git?rev=62fe64934aa8e39390aa3a6902b5dcd747dc41cd#62fe64934aa8e39390aa3a6902b5dcd747dc41cd" dependencies = [ "access-async-contracts", "access-flow", @@ -29,6 +29,7 @@ dependencies = [ "access-flow-tls", "access-flow-unix", "access-identity", + "access-tls-trust", "base64", "serde", "serde_json", @@ -38,7 +39,7 @@ dependencies = [ [[package]] name = "access-flow-relay" version = "0.1.0" -source = "git+https://github.com/kcosr/access-runtime.git?rev=c1031b70ab4dd9f372622d1e3a0e68da49971720#c1031b70ab4dd9f372622d1e3a0e68da49971720" +source = "git+https://github.com/kcosr/access-runtime.git?rev=62fe64934aa8e39390aa3a6902b5dcd747dc41cd#62fe64934aa8e39390aa3a6902b5dcd747dc41cd" dependencies = [ "access-async-contracts", "access-flow", @@ -50,11 +51,12 @@ dependencies = [ [[package]] name = "access-flow-tls" version = "0.1.0" -source = "git+https://github.com/kcosr/access-runtime.git?rev=c1031b70ab4dd9f372622d1e3a0e68da49971720#c1031b70ab4dd9f372622d1e3a0e68da49971720" +source = "git+https://github.com/kcosr/access-runtime.git?rev=62fe64934aa8e39390aa3a6902b5dcd747dc41cd#62fe64934aa8e39390aa3a6902b5dcd747dc41cd" dependencies = [ "access-async-contracts", "access-flow", "access-flow-relay", + "access-tls-trust", "idna", "rustix", "rustls", @@ -69,7 +71,7 @@ dependencies = [ [[package]] name = "access-flow-unix" version = "0.1.0" -source = "git+https://github.com/kcosr/access-runtime.git?rev=c1031b70ab4dd9f372622d1e3a0e68da49971720#c1031b70ab4dd9f372622d1e3a0e68da49971720" +source = "git+https://github.com/kcosr/access-runtime.git?rev=62fe64934aa8e39390aa3a6902b5dcd747dc41cd#62fe64934aa8e39390aa3a6902b5dcd747dc41cd" dependencies = [ "access-async-contracts", "access-flow", @@ -81,7 +83,7 @@ dependencies = [ [[package]] name = "access-identity" version = "0.1.0" -source = "git+https://github.com/kcosr/access-runtime.git?rev=c1031b70ab4dd9f372622d1e3a0e68da49971720#c1031b70ab4dd9f372622d1e3a0e68da49971720" +source = "git+https://github.com/kcosr/access-runtime.git?rev=62fe64934aa8e39390aa3a6902b5dcd747dc41cd#62fe64934aa8e39390aa3a6902b5dcd747dc41cd" dependencies = [ "access-async-contracts", "hmac", @@ -91,6 +93,30 @@ dependencies = [ "zeroize", ] +[[package]] +name = "access-tls-trust" +version = "0.1.0" +source = "git+https://github.com/kcosr/access-runtime.git?rev=62fe64934aa8e39390aa3a6902b5dcd747dc41cd#62fe64934aa8e39390aa3a6902b5dcd747dc41cd" +dependencies = [ + "access-async-contracts", + "access-tls-trust-windows", + "pem-rfc7468 0.7.0", + "rustix", + "rustls", + "rustls-native-certs", + "serde", + "sha2", + "thiserror 1.0.69", +] + +[[package]] +name = "access-tls-trust-windows" +version = "0.1.0" +source = "git+https://github.com/kcosr/access-runtime.git?rev=62fe64934aa8e39390aa3a6902b5dcd747dc41cd#62fe64934aa8e39390aa3a6902b5dcd747dc41cd" +dependencies = [ + "windows-sys 0.61.2", +] + [[package]] name = "aho-corasick" version = "1.1.4" @@ -194,6 +220,7 @@ dependencies = [ "access-flow-tls", "access-flow-unix", "access-identity", + "access-tls-trust", "anyhow", "assert_cmd", "axum", @@ -203,7 +230,7 @@ dependencies = [ "getrandom 0.3.4", "glob", "libc", - "pem-rfc7468", + "pem-rfc7468 1.0.0", "portable-pty", "predicates", "serde", @@ -401,6 +428,22 @@ version = "0.9.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8" +[[package]] +name = "core-foundation" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2a6cd9ae233e7f62ba4e9353e81a88df7fc8a5987b8d445b4d90c879bd156f6" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "core-foundation-sys" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" + [[package]] name = "cpufeatures" version = "0.2.17" @@ -1051,6 +1094,21 @@ version = "1.70.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" +[[package]] +name = "openssl-probe" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c87def4c32ab89d880effc9e097653c8da5d6ef28e6b539d313baaacfbafcbe" + +[[package]] +name = "pem-rfc7468" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88b39c9bfcfc231068454382784bb460aae594343fb030d46e9f50a645418412" +dependencies = [ + "base64ct", +] + [[package]] name = "pem-rfc7468" version = "1.0.0" @@ -1316,6 +1374,18 @@ dependencies = [ "zeroize", ] +[[package]] +name = "rustls-native-certs" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "612460d5f7bea540c490b2b6395d8e34a953e52b491accd6c86c8164c5932a63" +dependencies = [ + "openssl-probe", + "rustls-pki-types", + "schannel", + "security-framework", +] + [[package]] name = "rustls-pki-types" version = "1.15.0" @@ -1342,6 +1412,38 @@ version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" +[[package]] +name = "schannel" +version = "0.1.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91c1b7e4904c873ef0710c1f407dde2e6287de2bebc1bbbf7d430bb7cbffd939" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "security-framework" +version = "3.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7f4bc775c73d9a02cde8bf7b2ec4c9d12743edf609006c7facc23998404cd1d" +dependencies = [ + "bitflags 2.11.1", + "core-foundation", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + +[[package]] +name = "security-framework-sys" +version = "2.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce2691df843ecc5d231c0b14ece2acc3efb62c0a398c7e1d875f3983ce020e3" +dependencies = [ + "core-foundation-sys", + "libc", +] + [[package]] name = "semver" version = "1.0.28" diff --git a/Cargo.toml b/Cargo.toml index 148bc85..37cd432 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -22,11 +22,12 @@ name = "aw-ssh-command-filter" path = "src/bin/aw-ssh-command-filter.rs" [dependencies] -access-async-contracts = { git = "https://github.com/kcosr/access-runtime.git", rev = "c1031b70ab4dd9f372622d1e3a0e68da49971720" } -access-flow-relay = { git = "https://github.com/kcosr/access-runtime.git", rev = "c1031b70ab4dd9f372622d1e3a0e68da49971720" } -access-flow-tls = { git = "https://github.com/kcosr/access-runtime.git", rev = "c1031b70ab4dd9f372622d1e3a0e68da49971720" } -access-flow-unix = { git = "https://github.com/kcosr/access-runtime.git", rev = "c1031b70ab4dd9f372622d1e3a0e68da49971720" } -access-identity = { git = "https://github.com/kcosr/access-runtime.git", rev = "c1031b70ab4dd9f372622d1e3a0e68da49971720" } +access-async-contracts = { git = "https://github.com/kcosr/access-runtime.git", rev = "62fe64934aa8e39390aa3a6902b5dcd747dc41cd" } +access-flow-relay = { git = "https://github.com/kcosr/access-runtime.git", rev = "62fe64934aa8e39390aa3a6902b5dcd747dc41cd" } +access-flow-tls = { git = "https://github.com/kcosr/access-runtime.git", rev = "62fe64934aa8e39390aa3a6902b5dcd747dc41cd" } +access-flow-unix = { git = "https://github.com/kcosr/access-runtime.git", rev = "62fe64934aa8e39390aa3a6902b5dcd747dc41cd" } +access-identity = { git = "https://github.com/kcosr/access-runtime.git", rev = "62fe64934aa8e39390aa3a6902b5dcd747dc41cd" } +access-tls-trust = { git = "https://github.com/kcosr/access-runtime.git", rev = "62fe64934aa8e39390aa3a6902b5dcd747dc41cd" } anyhow = "1.0.103" axum = { version = "0.8.7", features = ["ws"] } clap = { version = "4.5.53", features = ["derive", "env"] } @@ -36,7 +37,6 @@ glob = "0.3.2" getrandom = "0.3.4" libc = "0.2.177" portable-pty = "0.9" -pem-rfc7468 = { version = "1.0.0", features = ["alloc"] } serde = { version = "1.0.228", features = ["derive"] } serde_json = "1.0.145" sha2 = "0.10.9" @@ -51,11 +51,12 @@ zeroize = "1.8.2" [dev-dependencies] assert_cmd = "2.1.1" +pem-rfc7468 = { version = "1.0.0", features = ["alloc"] } predicates = "3.1.3" tempfile = "3.23.0" tokio-tungstenite = "0.29.0" tower = { version = "0.5.2", features = ["util"] } [target.'cfg(target_os = "linux")'.dev-dependencies] -access-flow = { git = "https://github.com/kcosr/access-runtime.git", rev = "c1031b70ab4dd9f372622d1e3a0e68da49971720" } -access-flow-conformance = { git = "https://github.com/kcosr/access-runtime.git", rev = "c1031b70ab4dd9f372622d1e3a0e68da49971720" } +access-flow = { git = "https://github.com/kcosr/access-runtime.git", rev = "62fe64934aa8e39390aa3a6902b5dcd747dc41cd" } +access-flow-conformance = { git = "https://github.com/kcosr/access-runtime.git", rev = "62fe64934aa8e39390aa3a6902b5dcd747dc41cd" } diff --git a/README.md b/README.md index 3f86289..c455c78 100644 --- a/README.md +++ b/README.md @@ -658,10 +658,11 @@ kind = "unix" path = "/run/acl-proxy/transparent-http.sock" ``` -A route may instead use server-authenticated TLS/TCP. Provision its explicit -trust bundle through a read-only mount. Mount the containing directory rather -than the individual file so an atomic host-side replacement remains visible -to a later `SIGHUP`: +A route may instead use TLS/TCP. `system` uses the container process's platform +store and needs no CA mount. For `custom` or `system_plus_custom`, provision the +explicit trust bundle through a read-only mount. Mount the containing directory +rather than the individual file so an atomic host-side replacement remains +visible to a later `SIGHUP`: ```toml [[target_defaults.container_mounts]] @@ -701,10 +702,10 @@ The secure loader evaluates metadata in the container's filesystem view. `/`, every directory component, and the PEM leaf must be owned by root or the agent's effective UID and must not be group- or world-writable. No component may be a symlink. The leaf must be a nonempty regular file with exactly one -hard link and is limited to 1 MiB. For the shipped root-run profile, use -root ownership as shown above. A read-only mount prevents container writes but -does not make unsafe source ownership, modes, links, or host-side mutation -trusted. +hard link, at most 2,113,536 bytes of PEM, at most 128 certificates, and at +most 1 MiB of aggregate DER. For the shipped root-run profile, use root +ownership as shown above. A read-only mount prevents container writes but does +not make unsafe source ownership, modes, links, or host-side mutation trusted. The corresponding TLS route is: @@ -718,39 +719,44 @@ allowed_destination_ports = [80] kind = "tls_tcp" address = "proxy.example.com:7443" server_name = "proxy.example.com" - -[target_defaults.container_agent.access_flow_relay.routes.transport.trust] -kind = "pem_bundle" -path = "/etc/aw-gateway/acl-proxy-trust/roots.pem" +trust = "custom" +ca_certificate = "/etc/aw-gateway/acl-proxy-trust/roots.pem" ``` TLS routes require `bearer_environment`; disabled and anonymous presentation remain available only to Unix-only relays. Unix and TLS routes may coexist in -one bearer-authenticated relay. TLS uses version 1.3, verifies the independently -configured name against the explicit PEM bundle, requires the exact Access +one bearer-authenticated relay. Every TLS route requires an explicit `trust` +mode: `system`, `custom`, `system_plus_custom`, or `insecure`. The +`ca_certificate` field is required only for `custom` and +`system_plus_custom`. TLS uses version 1.3, verifies the independently +configured name in every verified mode, requires the exact Access Flow ALPN, and opens one outer connection for each workload connection. It does not use client certificates, ambient proxy settings, cleartext fallback, endpoint fallback, pooling, or multiplexing. Address, server name, and trust -path are literal and are not rendered from target variables. - -Treat the trust bundle as a direct authority to receive the reusable AWAF -bearer. Any endpoint whose certificate chains to that bundle and satisfies the -configured `server_name` can receive it. Use a dedicated, minimally scoped CA -and name rather than a broad organizational trust bundle. The server private -key remains only on the remote ACL Proxy host; do not mount it into the -container, install it in AW Gateway, or expose it to workloads. Access Flow -TLS performs no online OCSP or CRL retrieval. Revocation therefore requires -short-lived server certificates and explicit trust-generation replacement. - -The agent securely loads a stable regular PEM source with bounded size and -certificate count before reporting relay readiness. No remote reachability -probe is required. `SIGHUP` atomically reloads the complete trust generation -without reloading route configuration or the bearer. Existing flows retain -their established generation and drain. A failed reload keeps them alive but -makes the relay unready and rejects new Unix and TLS flows until a later -successful `SIGHUP`. `SIGTERM` and Ctrl-C retain their ordered shutdown -behavior whether or not the agent control socket is enabled. Foreground -`aw-gateway` signal behavior is unchanged. +source are literal and are not rendered from target variables. + +Every verified trust mode grants its complete authority set permission to +authenticate a server that can receive the reusable AWAF bearer. For custom +trust, use a dedicated, minimally scoped CA and name rather than a broad +organizational bundle. `system` deliberately grants the complete platform +store that authority. The server private key remains only on the remote ACL +Proxy host; do not mount it into the container, install it in AW Gateway, or +expose it to workloads. Access Flow TLS performs no online OCSP or CRL +retrieval. Revocation therefore requires short-lived server certificates and +explicit trust-generation replacement. + +The agent prepares system roots, stable custom PEM sources, or their union +through the shared Access Runtime trust component before reporting relay +readiness. `insecure` still performs encrypted TLS and exact ALPN negotiation +but deliberately skips certificate and server-name authentication. No remote +reachability probe is required. `SIGHUP` atomically reloads the complete trust +generation without reloading route configuration or the bearer. Existing flows +retain their established generation and drain. Candidate construction leaves +the current valid generation ready. Material, custody, or internal failures +then close new admission; transient system-store failures, cancellation, and +generation-budget rejection preserve the ready generation. `SIGTERM` and +Ctrl-C retain their ordered shutdown behavior whether or not the agent control +socket is enabled. Foreground `aw-gateway` signal behavior is unchanged. Agent relay logs expose only fixed event kinds (`Prepared`, `Ready`, `ConnectionOpened`, `ConnectionRejected`, `AdmissionClosed`, diff --git a/examples/docker/container-agent-access-flow-tls.toml b/examples/docker/container-agent-access-flow-tls.toml index e60a8b2..6121387 100644 --- a/examples/docker/container-agent-access-flow-tls.toml +++ b/examples/docker/container-agent-access-flow-tls.toml @@ -37,10 +37,8 @@ allowed_destination_ports = [80] kind = "tls_tcp" address = "proxy.example.com:7443" server_name = "proxy.example.com" - -[container_agent.access_flow_relay.routes.transport.trust] -kind = "pem_bundle" -path = "/etc/aw-gateway/acl-proxy-trust/roots.pem" +trust = "custom" +ca_certificate = "/etc/aw-gateway/acl-proxy-trust/roots.pem" [[container_agent.access_flow_relay.routes]] name = "https" @@ -51,7 +49,5 @@ allowed_destination_ports = [443] kind = "tls_tcp" address = "proxy.example.com:7444" server_name = "proxy.example.com" - -[container_agent.access_flow_relay.routes.transport.trust] -kind = "pem_bundle" -path = "/etc/aw-gateway/acl-proxy-trust/roots.pem" +trust = "custom" +ca_certificate = "/etc/aw-gateway/acl-proxy-trust/roots.pem" diff --git a/examples/docker/gateway-access-flow-tls.toml b/examples/docker/gateway-access-flow-tls.toml index 38fb9a2..a8f9595 100644 --- a/examples/docker/gateway-access-flow-tls.toml +++ b/examples/docker/gateway-access-flow-tls.toml @@ -154,10 +154,8 @@ allowed_destination_ports = [80] kind = "tls_tcp" address = "proxy.example.com:7443" server_name = "proxy.example.com" - -[target_defaults.container_agent.access_flow_relay.routes.transport.trust] -kind = "pem_bundle" -path = "/etc/aw-gateway/acl-proxy-trust/roots.pem" +trust = "custom" +ca_certificate = "/etc/aw-gateway/acl-proxy-trust/roots.pem" [[target_defaults.container_agent.access_flow_relay.routes]] name = "https" @@ -168,10 +166,8 @@ allowed_destination_ports = [443] kind = "tls_tcp" address = "proxy.example.com:7444" server_name = "proxy.example.com" - -[target_defaults.container_agent.access_flow_relay.routes.transport.trust] -kind = "pem_bundle" -path = "/etc/aw-gateway/acl-proxy-trust/roots.pem" +trust = "custom" +ca_certificate = "/etc/aw-gateway/acl-proxy-trust/roots.pem" [[target_defaults.container_agent.services]] name = "container-sshd" diff --git a/scripts/test-access-runtime-pin.py b/scripts/test-access-runtime-pin.py index 4610c37..5e982c4 100644 --- a/scripts/test-access-runtime-pin.py +++ b/scripts/test-access-runtime-pin.py @@ -17,6 +17,7 @@ "access-flow-tls", "access-flow-unix", "access-identity", + "access-tls-trust", ) LINUX_TEST_NAMES = ("access-flow", "access-flow-conformance") diff --git a/scripts/validate-access-runtime-pin.py b/scripts/validate-access-runtime-pin.py index 6fb330b..4f34542 100644 --- a/scripts/validate-access-runtime-pin.py +++ b/scripts/validate-access-runtime-pin.py @@ -12,11 +12,13 @@ "access-flow-tls", "access-flow-unix", "access-identity", + "access-tls-trust", } LINUX_TEST_NAMES = { "access-flow", "access-flow-conformance", } +TRANSITIVE_NAMES = {"access-tls-trust-windows"} LINUX_TARGET = 'cfg(target_os = "linux")' @@ -113,6 +115,7 @@ def main() -> None: revision = revisions.pop() names = PRODUCTION_NAMES | LINUX_TEST_NAMES + lock_names = names | TRANSITIVE_NAMES packages = lock.get("package") if not isinstance(packages, list): raise SystemExit("Cargo.lock package inventory is missing") @@ -123,12 +126,12 @@ def main() -> None: if ( isinstance(source, str) and source.startswith(runtime_source_prefix) - and name not in names + and name not in lock_names ): raise SystemExit(f"unexpected Access Runtime lock package: {name!r}") expected_source = f"git+{expected_url}?rev={revision}#{revision}" - for name in names: + for name in lock_names: matching = [package for package in packages if package.get("name") == name] if len(matching) != 1: raise SystemExit( diff --git a/smoke/scripts/run-tls-access-flow-cross-host-smoke.sh b/smoke/scripts/run-tls-access-flow-cross-host-smoke.sh index 08549a1..5d1c1b0 100755 --- a/smoke/scripts/run-tls-access-flow-cross-host-smoke.sh +++ b/smoke/scripts/run-tls-access-flow-cross-host-smoke.sh @@ -1763,10 +1763,8 @@ allowed_destination_ports = [80] kind = "tls_tcp" address = "$REMOTE_ADDRESS:$TLS_HTTP_PORT" server_name = "proxy.access-flow.test" - -[container_agent.access_flow_relay.routes.transport.trust] -kind = "pem_bundle" -path = "/run/aw-gateway/trust/access-flow-root.pem" +trust = "custom" +ca_certificate = "/run/aw-gateway/trust/access-flow-root.pem" [[container_agent.access_flow_relay.routes]] name = "https" @@ -1777,10 +1775,8 @@ allowed_destination_ports = [443] kind = "tls_tcp" address = "$REMOTE_ADDRESS:$TLS_HTTPS_PORT" server_name = "proxy.access-flow.test" - -[container_agent.access_flow_relay.routes.transport.trust] -kind = "pem_bundle" -path = "/run/aw-gateway/trust/access-flow-root.pem" +trust = "custom" +ca_certificate = "/run/aw-gateway/trust/access-flow-root.pem" EOF chmod 0600 "$TMP_DIR/container-agent.toml" diff --git a/smoke/scripts/run-tls-access-flow-stack-smoke.sh b/smoke/scripts/run-tls-access-flow-stack-smoke.sh index 28a1131..36fc3d9 100755 --- a/smoke/scripts/run-tls-access-flow-stack-smoke.sh +++ b/smoke/scripts/run-tls-access-flow-stack-smoke.sh @@ -1245,20 +1245,16 @@ AGENT_HTTP_TRANSPORT=$(cat <) -> bool { async fn perform_shutdown(state: Arc) { tracing::info!("container agent shutdown starting"); state.accepting_bridge.store(false, Ordering::SeqCst); + let relay_close_deadline = state.access_flow_relay.as_ref().map(|relay| { + std::time::Instant::now() + .checked_add(relay.drain_timeout()) + .unwrap_or_else(std::time::Instant::now) + }); for phase in shutdown_phases(state.access_flow_relay.is_some()) { match phase { ShutdownPhase::CloseRelayAdmission => { @@ -78,7 +83,9 @@ async fn perform_shutdown(state: Arc) { .access_flow_relay .as_ref() .expect("relay shutdown phase requires configured relay") - .close_admission() + .close_admission_by( + relay_close_deadline.expect("relay shutdown deadline must be available"), + ) .await; } ShutdownPhase::StopRelayDependents => stop_relay_dependents(&state).await, diff --git a/src/agent/relay.rs b/src/agent/relay.rs index 581481f..d0baca7 100644 --- a/src/agent/relay.rs +++ b/src/agent/relay.rs @@ -22,7 +22,9 @@ use crate::config::{ AccessFlowRelayConfig, CompiledAccessFlowRelayConfig, CompiledAccessFlowRelayEndpoint, }; -use super::relay_transport::{PendingRelayTransportReload, RelayTransportRuntime}; +use super::relay_transport::{ + PendingRelayTransportReload, RelayTransportRuntime, RelayTransportTrustBudget, +}; use super::service::ManagedService; use super::state::AgentState; @@ -47,7 +49,10 @@ pub(super) enum RelayFatalKind { enum RelayCommand { ReloadSecurity(PendingRelayTransportReload), - CloseAdmission(oneshot::Sender<()>), + CloseAdmission { + deadline: Instant, + completed: oneshot::Sender<()>, + }, Shutdown { deadline: Instant, completed: oneshot::Sender<()>, @@ -62,10 +67,11 @@ enum RelayCommand { #[derive(Debug)] pub(super) struct RelayControl { state: Mutex, - route_names: Box<[String]>, + routes: Box<[(String, Option)]>, active_flows: Arc, accepting: AtomicBool, security_healthy: Arc, + security_failure: StdMutex>, reload_in_progress: AtomicBool, phase: AtomicU8, lifecycle: StdMutex<()>, @@ -94,14 +100,23 @@ impl RelayControl { ( Arc::new(Self { state: Mutex::new(AccessFlowRelayStateName::Preparing), - route_names: config + routes: config .routes .iter() - .map(|route| route.name.clone()) + .map(|route| { + let trust_mode = match route.transport { + crate::config::AccessFlowRelayTransport::Unix { .. } => None, + crate::config::AccessFlowRelayTransport::TlsTcp { trust, .. } => { + Some(trust) + } + }; + (route.name.clone(), trust_mode) + }) .collect(), active_flows: Arc::new(AtomicUsize::new(0)), accepting: AtomicBool::new(false), security_healthy: Arc::new(AtomicBool::new(false)), + security_failure: StdMutex::new(None), reload_in_progress: AtomicBool::new(false), phase: AtomicU8::new(RELAY_PHASE_PREPARING), lifecycle: StdMutex::new(()), @@ -134,13 +149,26 @@ impl RelayControl { ready: state == AccessFlowRelayStateName::Accepting && accepting, active_flows: self.active_flows.load(Ordering::Acquire), routes: self - .route_names + .routes .iter() - .map(|name| AccessFlowRelayRouteStatus { + .map(|(name, trust_mode)| AccessFlowRelayRouteStatus { name: name.clone(), accepting, + trust_mode: *trust_mode, }) .collect(), + trust_failure: if self + .transport + .get() + .is_some_and(RelayTransportRuntime::reload_blocked) + { + Some("trust_reload_blocked".to_string()) + } else { + self.security_failure + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .map(|error| error.status_code().to_string()) + }, } } @@ -158,7 +186,15 @@ impl RelayControl { self.drain_timeout } + #[cfg(test)] pub(super) async fn close_admission(&self) { + let deadline = Instant::now() + .checked_add(self.drain_timeout) + .unwrap_or_else(Instant::now); + self.close_admission_by(deadline).await; + } + + pub(super) async fn close_admission_by(&self, deadline: Instant) { let phase = { let _lifecycle = self .lifecycle @@ -177,10 +213,16 @@ impl RelayControl { if phase == RELAY_PHASE_PREPARING { self.startup_cancellation.cancel(); } + if let Some(transport) = self.transport.get() { + transport.close(); + } let (completed, wait) = oneshot::channel(); if self .command_tx - .send(RelayCommand::CloseAdmission(completed)) + .send(RelayCommand::CloseAdmission { + deadline, + completed, + }) .await .is_ok() { @@ -254,8 +296,9 @@ impl RelayControl { finish_security_reload(self, ReloadFinish::Completed, Ok(())); return Ok(()); } - Err(_) => { + Err(error) => { drop(command); + self.set_security_failure(Some(error)); finish_security_reload(self, ReloadFinish::Rejected, Err(())); return Err(()); } @@ -277,6 +320,9 @@ impl RelayControl { self.startup_cancellation.cancel(); } } + if let Some(transport) = self.transport.get() { + transport.close(); + } let (completed, wait) = oneshot::channel(); if self .command_tx @@ -308,6 +354,13 @@ impl RelayControl { *self.state.lock().await = state; } + fn set_security_failure(&self, error: Option) { + *self + .security_failure + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) = error; + } + #[cfg(test)] async fn pause_before_activation(&self) -> (oneshot::Receiver<()>, oneshot::Sender<()>) { let (reached, wait_for_reached) = oneshot::channel(); @@ -539,10 +592,26 @@ async fn run_relay( presentation, ) .context("compile access flow relay configuration")?; + for route in &config.routes { + if matches!( + route.transport, + crate::config::AccessFlowRelayTransport::TlsTcp { + trust: access_tls_trust::TlsClientTrustMode::Insecure, + .. + } + ) { + tracing::warn!( + category = "security_material", + route = %route.name, + "Access Flow TLS route does not authenticate the remote server" + ); + } + } probe_unix_endpoints(&compiled)?; let transport_reserve = RelayTransportRuntime::resource_reserve(&compiled.plan) .context("project access flow relay transport reload resources")?; - let budget = relay_resource_budget( + let agent_budget = relay_resource_budget(state.bridge_enabled, services.len(), 0, 0)?; + let relay_budget = relay_resource_budget( state.bridge_enabled, services.len(), transport_reserve.descriptors, @@ -561,9 +630,14 @@ async fn run_relay( Arc::new(RelayObserver { active_flows: Arc::clone(&control.active_flows), }), - budget, + relay_budget, ) .context("access flow relay resource preflight")?; + let relay_projection = relay.resource_projection(); + let trust_budget = relay_transport_trust_budget(agent_budget, relay_projection)?; + transport + .configure_trust_budget(trust_budget) + .context("configure Access Flow TLS trust residual")?; let projected_descriptors = relay .resource_projection() .total_descriptors @@ -579,10 +653,19 @@ async fn run_relay( memory_bytes = projected_memory_bytes, "access flow relay resource preflight passed" ); - transport - .activate_prepared() - .await - .context("activate access flow relay transport")?; + match transport.activate_prepared().await { + Ok(()) => {} + Err(error) if ordered_startup_cancelled(state, control) => { + tracing::debug!( + error = %error, + "access flow relay transport activation cancelled by shutdown" + ); + return finish_cancelled_start(control, commands).await; + } + Err(error) => { + return Err(error).context("activate access flow relay transport"); + } + } let prepared = match relay .prepare(Arc::new(control.startup_cancellation.clone())) .await @@ -680,7 +763,7 @@ async fn handle_start_command( finish_security_reload(control, ReloadFinish::Rejected, Err(())); None } - Some(RelayCommand::CloseAdmission(completed)) => { + Some(RelayCommand::CloseAdmission { completed, .. }) => { control.startup_cancellation.cancel(); control.accepting.store(false, Ordering::Release); control.set_state(AccessFlowRelayStateName::Draining).await; @@ -721,7 +804,7 @@ async fn finish_cancelled_start( let _ = pending.complete().await; finish_security_reload(control, ReloadFinish::Rejected, Err(())); } - RelayCommand::CloseAdmission(completed) => { + RelayCommand::CloseAdmission { completed, .. } => { let _ = completed.send(()); } RelayCommand::Shutdown { completed, .. } => { @@ -752,10 +835,14 @@ async fn run_active_relay( tokio::select! { failure = running.wait_for_failure() => { transport.close(); + let reload_deadline = Instant::now() + .checked_add(configured_drain_timeout) + .unwrap_or_else(Instant::now); finish_pending_reload( control, &mut pending_reload, ReloadFinish::Shutdown, + reload_deadline, ).await; return begin_failed_relay_shutdown( running, @@ -791,7 +878,10 @@ async fn run_active_relay( ); } } - Some(RelayCommand::CloseAdmission(completed)) => { + Some(RelayCommand::CloseAdmission { + deadline, + completed, + }) => { transport.close(); #[cfg(test)] control.publish_close_before_reload_join().await; @@ -799,6 +889,7 @@ async fn run_active_relay( control, &mut pending_reload, ReloadFinish::Shutdown, + deadline, ).await; #[cfg(test)] control.publish_close_after_reload_join().await; @@ -813,6 +904,7 @@ async fn run_active_relay( control, &mut pending_reload, ReloadFinish::Shutdown, + deadline, ).await; let configured_deadline = Instant::now() .checked_add(configured_drain_timeout) @@ -832,6 +924,7 @@ async fn run_active_relay( control, &mut pending_reload, ReloadFinish::Shutdown, + Instant::now(), ).await; control.accepting.store(false, Ordering::Release); let _ = running.shutdown(Instant::now()).await; @@ -840,10 +933,14 @@ async fn run_active_relay( #[cfg(test)] Some(RelayCommand::InjectFailure { failure, observed }) => { transport.close(); + let reload_deadline = Instant::now() + .checked_add(configured_drain_timeout) + .unwrap_or_else(Instant::now); finish_pending_reload( control, &mut pending_reload, ReloadFinish::Shutdown, + reload_deadline, ).await; return begin_failed_relay_shutdown( running, @@ -859,6 +956,7 @@ async fn run_active_relay( } } result = wait_for_reload(&mut pending_reload), if pending_reload.is_some() => { + control.set_security_failure(result.as_ref().err().copied()); let pending = pending_reload .take() .expect("completed reload remains owned by the relay loop"); @@ -887,9 +985,10 @@ async fn finish_pending_reload( control: &RelayControl, pending: &mut Option, finish: ReloadFinish, + deadline: Instant, ) { if let Some(pending) = pending.take() { - let result = pending.complete().await.map_err(|_| ()); + let result = pending.complete_by(deadline.into()).await.map_err(|_| ()); finish_security_reload(control, finish, result); } } @@ -985,7 +1084,7 @@ async fn await_failed_relay_shutdown( let _ = pending.complete().await; finish_security_reload(control, ReloadFinish::Rejected, Err(())); } - RelayCommand::CloseAdmission(completed) => { + RelayCommand::CloseAdmission { completed, .. } => { running.close_admission().await; let _ = completed.send(()); } @@ -1051,6 +1150,22 @@ fn relay_resource_budget( ) } +fn relay_transport_trust_budget( + agent_budget: AccessFlowRelayResourceBudget, + relay_projection: access_flow_relay::AccessFlowRelayResourceProjection, +) -> anyhow::Result { + Ok(RelayTransportTrustBudget { + descriptors: agent_budget + .descriptors + .checked_sub(relay_projection.total_descriptors) + .context("Access Flow TLS trust descriptor residual is exhausted")?, + memory_bytes: agent_budget + .memory_bytes + .checked_sub(relay_projection.total_memory_bytes) + .context("Access Flow TLS trust memory residual is exhausted")?, + }) +} + fn relay_resource_budget_with_nofile( bridge_enabled: bool, service_count: usize, @@ -1121,8 +1236,8 @@ fn soft_nofile_limit() -> anyhow::Result { mod tests { use super::*; use crate::config::{ - AccessFlowRelayPresentation, AccessFlowRelayRoute, AccessFlowRelayTransport, - AccessFlowRelayTrust, LoggingConfig, RestartPolicy, ServiceConfig, + AccessFlowRelayPresentation, AccessFlowRelayRoute, AccessFlowRelayTransport, LoggingConfig, + RestartPolicy, ServiceConfig, }; #[cfg(target_os = "linux")] use access_flow::{ @@ -1139,6 +1254,7 @@ mod tests { TlsAccessFlowServerChannel, TlsAccessFlowServerIdentity, TlsAccessFlowServerLimits, TlsAccessFlowTcpListener, }; + use access_tls_trust::TlsClientTrustMode; use std::collections::BTreeMap; #[cfg(target_os = "linux")] use std::convert::Infallible; @@ -1365,6 +1481,280 @@ MC4CAQAwBQYDK2VwBCIEIGRXBokZ2/yO2kASVZKtUVGnOwIM7kZJKJgugoeMCRxd assert!(relay_resource_budget(false, 0, 0, u64::MAX).is_err()); } + #[test] + fn system_trust_exact_peak_fits_both_product_residuals_and_oversubscription_rejects() { + let config = AccessFlowRelayConfig { + setup_timeout: "2s".into(), + drain_timeout: "1s".into(), + max_connections: 64, + copy_buffer_bytes_per_direction: 16 * 1024, + start_after_services: Vec::new(), + presentation: AccessFlowRelayPresentation::BearerEnvironment { + variable: "AW_ACCESS_FLOW_TEST_TOKEN".into(), + }, + routes: vec![AccessFlowRelayRoute { + name: "https".into(), + listen: "127.0.0.1:3129".into(), + allowed_destination_ports: vec![443], + transport: AccessFlowRelayTransport::TlsTcp { + address: "127.0.0.1:7443".into(), + server_name: "access-flow.test".into(), + trust: TlsClientTrustMode::System, + ca_certificate: None, + }, + }], + }; + let compile = || { + config + .compile_with_presentation( + crate::config::AccessFlowRelayValidationMode::Agent, + IdentityPresentation::Bearer( + access_identity::SensitiveBearer::new(TEST_ACCESS_FLOW_BEARER).unwrap(), + ), + ) + .unwrap() + }; + let compiled = compile(); + let reserve = RelayTransportRuntime::resource_reserve(&compiled.plan).unwrap(); + assert_eq!(reserve.memory_bytes, 58_855_424); + + for bridge_enabled in [false, true] { + let compiled = compile(); + let agent_budget = + relay_resource_budget_with_nofile(bridge_enabled, 0, 0, 0, 4096).unwrap(); + let relay_budget = relay_resource_budget_with_nofile( + bridge_enabled, + 0, + reserve.descriptors, + reserve.memory_bytes, + 4096, + ) + .unwrap(); + let transport = + RelayTransportRuntime::prepare(&compiled.plan, Arc::new(AtomicBool::new(false))) + .unwrap(); + let relay = AccessFlowRelay::new( + compiled.plan, + transport.connector(), + Arc::new(RelayObserver { + active_flows: Arc::new(AtomicUsize::new(0)), + }), + relay_budget, + ) + .unwrap(); + let residual = + relay_transport_trust_budget(agent_budget, relay.resource_projection()).unwrap(); + assert!(residual.memory_bytes >= 58_855_424); + transport.configure_trust_budget(residual).unwrap(); + + let rejected = compile(); + let oversubscribed = + RelayTransportRuntime::prepare(&rejected.plan, Arc::new(AtomicBool::new(false))) + .unwrap(); + assert!( + oversubscribed + .configure_trust_budget(RelayTransportTrustBudget { + descriptors: residual.descriptors, + memory_bytes: reserve.memory_bytes - 1, + }) + .is_err() + ); + } + } + + #[tokio::test] + async fn blocked_trust_operation_preserves_readiness_excludes_reload_and_recovers() { + let source = std::net::TcpListener::bind((std::net::Ipv4Addr::LOCALHOST, 0)).unwrap(); + let listen = source.local_addr().unwrap(); + drop(source); + let config = AccessFlowRelayConfig { + setup_timeout: "2s".into(), + drain_timeout: "1s".into(), + max_connections: 4, + copy_buffer_bytes_per_direction: 4096, + start_after_services: Vec::new(), + presentation: AccessFlowRelayPresentation::BearerEnvironment { + variable: "AW_ACCESS_FLOW_TEST_TOKEN".into(), + }, + routes: vec![AccessFlowRelayRoute { + name: "https".into(), + listen: listen.to_string(), + allowed_destination_ports: vec![443], + transport: AccessFlowRelayTransport::TlsTcp { + address: "127.0.0.1:7443".into(), + server_name: "access-flow.test".into(), + trust: TlsClientTrustMode::Insecure, + ca_certificate: None, + }, + }], + }; + let (control, commands) = RelayControl::configured(&config); + let state = Arc::new(AgentState::new( + PathBuf::from("/tmp/aw-gateway-blocked-trust-test"), + None, + false, + None, + None, + Some(control.clone()), + )); + let supervisor = tokio::spawn(run_relay_supervisor( + config, + IdentityPresentation::Bearer( + access_identity::SensitiveBearer::new(TEST_ACCESS_FLOW_BEARER).unwrap(), + ), + Vec::new(), + state, + control.clone(), + commands, + )); + tokio::time::timeout(Duration::from_secs(2), async { + while !control.is_ready() { + sleep(Duration::from_millis(10)).await; + } + }) + .await + .unwrap(); + + let (release, wait_for_release) = oneshot::channel::<()>(); + control + .transport + .get() + .unwrap() + .install_test_blocked_operation(tokio::spawn(async move { + let _ = wait_for_release.await; + })); + assert!(control.is_ready()); + let blocked = control.status().await; + assert!(blocked.ready); + assert_eq!( + blocked.trust_failure.as_deref(), + Some("trust_reload_blocked") + ); + assert!(control.initiate_security_reload().is_err()); + assert!(control.is_ready()); + + release.send(()).unwrap(); + tokio::time::timeout(Duration::from_secs(1), async { + while control + .transport + .get() + .is_some_and(RelayTransportRuntime::reload_blocked) + { + tokio::task::yield_now().await; + } + }) + .await + .unwrap(); + control.initiate_security_reload().unwrap(); + tokio::time::timeout(Duration::from_secs(1), async { + while control.reload_in_progress.load(Ordering::Acquire) { + tokio::task::yield_now().await; + } + }) + .await + .unwrap(); + assert!(control.is_ready()); + assert_eq!(control.status().await.trust_failure, None); + + control + .shutdown(Instant::now() + Duration::from_secs(1)) + .await; + supervisor.await.unwrap().unwrap(); + } + + #[tokio::test] + async fn shutdown_deadline_detaches_and_accounts_noninterruptible_reload() { + let source = std::net::TcpListener::bind((std::net::Ipv4Addr::LOCALHOST, 0)).unwrap(); + let listen = source.local_addr().unwrap(); + drop(source); + let config = AccessFlowRelayConfig { + setup_timeout: "2s".into(), + drain_timeout: "50ms".into(), + max_connections: 4, + copy_buffer_bytes_per_direction: 4096, + start_after_services: Vec::new(), + presentation: AccessFlowRelayPresentation::BearerEnvironment { + variable: "AW_ACCESS_FLOW_TEST_TOKEN".into(), + }, + routes: vec![AccessFlowRelayRoute { + name: "https".into(), + listen: listen.to_string(), + allowed_destination_ports: vec![443], + transport: AccessFlowRelayTransport::TlsTcp { + address: "127.0.0.1:7443".into(), + server_name: "access-flow.test".into(), + trust: TlsClientTrustMode::Insecure, + ca_certificate: None, + }, + }], + }; + let (control, commands) = RelayControl::configured(&config); + let (reload_reached, resume_reload) = control.pause_blocking_reload(); + let state = Arc::new(AgentState::new( + PathBuf::from("/tmp/aw-gateway-reload-deadline-test"), + None, + false, + None, + None, + Some(control.clone()), + )); + let supervisor = tokio::spawn(run_relay_supervisor( + config, + IdentityPresentation::Bearer( + access_identity::SensitiveBearer::new(TEST_ACCESS_FLOW_BEARER).unwrap(), + ), + Vec::new(), + state, + control.clone(), + commands, + )); + tokio::time::timeout(Duration::from_secs(2), async { + while !control.is_ready() { + sleep(Duration::from_millis(10)).await; + } + }) + .await + .unwrap(); + control.initiate_security_reload().unwrap(); + tokio::time::timeout( + Duration::from_secs(1), + tokio::task::spawn_blocking(move || reload_reached.recv()), + ) + .await + .unwrap() + .unwrap() + .unwrap(); + + let deadline = Instant::now() + Duration::from_millis(50); + tokio::time::timeout( + Duration::from_millis(250), + control.close_admission_by(deadline), + ) + .await + .expect("admission close exceeded the requested deadline"); + let transport = control.transport.get().unwrap(); + assert!(transport.reload_blocked()); + assert_eq!( + control.status().await.trust_failure.as_deref(), + Some("trust_reload_blocked") + ); + control.shutdown(deadline).await; + tokio::time::timeout(Duration::from_millis(250), supervisor) + .await + .expect("relay shutdown exceeded the original requested deadline") + .unwrap() + .unwrap(); + + resume_reload.send(()).unwrap(); + tokio::time::timeout(Duration::from_secs(1), async { + while transport.reload_blocked() { + tokio::task::yield_now().await; + } + }) + .await + .unwrap(); + } + #[cfg(unix)] #[tokio::test] async fn shipped_tls_agent_plan_fits_supported_product_topologies() { @@ -1391,11 +1781,11 @@ MC4CAQAwBQYDK2VwBCIEIGRXBokZ2/yO2kASVZKtUVGnOwIM7kZJKJgugoeMCRxd std::fs::write(&trust_path, TEST_ACCESS_FLOW_ROOT_PEM).unwrap(); std::fs::set_permissions(&trust_path, std::fs::Permissions::from_mode(0o644)).unwrap(); for route in &mut relay_config.routes { - let AccessFlowRelayTransport::TlsTcp { trust, .. } = &mut route.transport else { + let AccessFlowRelayTransport::TlsTcp { ca_certificate, .. } = &mut route.transport + else { panic!("shipped TLS relay contains a non-TLS route"); }; - let AccessFlowRelayTrust::PemBundle { path } = trust; - *path = trust_path.display().to_string(); + *ca_certificate = Some(trust_path.display().to_string()); } let relay_config = relay_config.clone(); let bearer = vec![b'B'; access_identity::MAX_BEARER_LEN]; @@ -1411,6 +1801,9 @@ MC4CAQAwBQYDK2VwBCIEIGRXBokZ2/yO2kASVZKtUVGnOwIM7kZJKJgugoeMCRxd .unwrap(); let transport_reserve = RelayTransportRuntime::resource_reserve(&compiled.plan).unwrap(); + let agent_budget = + relay_resource_budget_with_nofile(bridge_enabled, service_count, 0, 0, 4096) + .unwrap(); let budget = relay_resource_budget_with_nofile( bridge_enabled, service_count, @@ -1431,6 +1824,22 @@ MC4CAQAwBQYDK2VwBCIEIGRXBokZ2/yO2kASVZKtUVGnOwIM7kZJKJgugoeMCRxd budget, ) .unwrap(); + let trust_budget = + relay_transport_trust_budget(agent_budget, relay.resource_projection()).unwrap(); + let non_relay_memory = if bridge_enabled { + BRIDGE_NON_RELAY_MEMORY_BYTES + } else { + NO_BRIDGE_NON_RELAY_MEMORY_BYTES + }; + assert_eq!( + trust_budget.memory_bytes, + TOTAL_AGENT_MEMORY_PREFLIGHT_BYTES + - non_relay_memory + - PER_SERVICE_NON_RELAY_MEMORY_BYTES * service_count as u64 + - relay.resource_projection().total_memory_bytes + ); + assert!(trust_budget.memory_bytes >= transport_reserve.memory_bytes); + transport.configure_trust_budget(trust_budget).unwrap(); transport.activate_prepared().await.unwrap(); let projection = relay.resource_projection(); assert!(projection.total_descriptors <= budget.descriptors); @@ -1471,11 +1880,11 @@ MC4CAQAwBQYDK2VwBCIEIGRXBokZ2/yO2kASVZKtUVGnOwIM7kZJKJgugoeMCRxd std::fs::write(&trust_path, TEST_ACCESS_FLOW_ROOT_PEM).unwrap(); std::fs::set_permissions(&trust_path, std::fs::Permissions::from_mode(0o644)).unwrap(); for route in &mut relay_config.routes { - let AccessFlowRelayTransport::TlsTcp { trust, .. } = &mut route.transport else { + let AccessFlowRelayTransport::TlsTcp { ca_certificate, .. } = &mut route.transport + else { panic!("RT06 boundary relay contains a non-TLS route"); }; - let AccessFlowRelayTrust::PemBundle { path } = trust; - *path = trust_path.display().to_string(); + *ca_certificate = Some(trust_path.display().to_string()); } let bearer = vec![b'B'; access_identity::MAX_BEARER_LEN]; @@ -1488,6 +1897,7 @@ MC4CAQAwBQYDK2VwBCIEIGRXBokZ2/yO2kASVZKtUVGnOwIM7kZJKJgugoeMCRxd ) .unwrap(); let transport_reserve = RelayTransportRuntime::resource_reserve(&compiled.plan).unwrap(); + let agent_budget = relay_resource_budget_with_nofile(false, 1, 0, 0, 4096).unwrap(); let budget = relay_resource_budget_with_nofile( false, 1, @@ -1508,6 +1918,11 @@ MC4CAQAwBQYDK2VwBCIEIGRXBokZ2/yO2kASVZKtUVGnOwIM7kZJKJgugoeMCRxd budget, ) .expect("phase-aware RT06 resource projection must fit the product budget"); + transport + .configure_trust_budget( + relay_transport_trust_budget(agent_budget, relay.resource_projection()).unwrap(), + ) + .unwrap(); let projection = relay.resource_projection(); assert!(projection.total_descriptors <= budget.descriptors); assert!(projection.total_memory_bytes <= budget.memory_bytes); @@ -1573,8 +1988,8 @@ MC4CAQAwBQYDK2VwBCIEIGRXBokZ2/yO2kASVZKtUVGnOwIM7kZJKJgugoeMCRxd .checked_add(projection.setup_bytes.min(projection.copy_buffer_bytes)) .unwrap(); assert!( - obsolete_summed_phase_bytes > budget.memory_bytes, - "the RT06 boundary must detect summing mutually exclusive connection phases" + obsolete_summed_phase_bytes > projection.total_memory_bytes, + "the RT06 projection must not sum mutually exclusive connection phases" ); } @@ -1658,9 +2073,8 @@ MC4CAQAwBQYDK2VwBCIEIGRXBokZ2/yO2kASVZKtUVGnOwIM7kZJKJgugoeMCRxd transport: AccessFlowRelayTransport::TlsTcp { address: tls_address.to_string(), server_name, - trust: AccessFlowRelayTrust::PemBundle { - path: trust_path.display().to_string(), - }, + trust: TlsClientTrustMode::Custom, + ca_certificate: Some(trust_path.display().to_string()), }, }], }; @@ -1813,9 +2227,8 @@ MC4CAQAwBQYDK2VwBCIEIGRXBokZ2/yO2kASVZKtUVGnOwIM7kZJKJgugoeMCRxd transport: AccessFlowRelayTransport::TlsTcp { address: tls_address.to_string(), server_name, - trust: AccessFlowRelayTrust::PemBundle { - path: trust_path.display().to_string(), - }, + trust: TlsClientTrustMode::Custom, + ca_certificate: Some(trust_path.display().to_string()), }, }], }; @@ -1928,9 +2341,8 @@ MC4CAQAwBQYDK2VwBCIEIGRXBokZ2/yO2kASVZKtUVGnOwIM7kZJKJgugoeMCRxd transport: AccessFlowRelayTransport::TlsTcp { address: tls_address.to_string(), server_name: "access-flow.test".into(), - trust: AccessFlowRelayTrust::PemBundle { - path: trust_path.display().to_string(), - }, + trust: TlsClientTrustMode::Custom, + ca_certificate: Some(trust_path.display().to_string()), }, }], }; @@ -2223,9 +2635,8 @@ MC4CAQAwBQYDK2VwBCIEIGRXBokZ2/yO2kASVZKtUVGnOwIM7kZJKJgugoeMCRxd transport: AccessFlowRelayTransport::TlsTcp { address: tls_address.to_string(), server_name: "access-flow.test".into(), - trust: AccessFlowRelayTrust::PemBundle { - path: trust_path.display().to_string(), - }, + trust: TlsClientTrustMode::Custom, + ca_certificate: Some(trust_path.display().to_string()), }, }, ], @@ -2268,36 +2679,9 @@ MC4CAQAwBQYDK2VwBCIEIGRXBokZ2/yO2kASVZKtUVGnOwIM7kZJKJgugoeMCRxd .unwrap() .unwrap(); std::fs::write(&trust_path, b"stable invalid trust material").unwrap(); - - let mut rejected_unix = tokio::net::TcpStream::connect(http_listen).await.unwrap(); - let mut eof = [0_u8; 1]; - assert_eq!( - tokio::time::timeout(Duration::from_millis(250), rejected_unix.read(&mut eof)) - .await - .expect("Unix route did not reject during trust reload") - .unwrap(), - 0 - ); assert!( - tokio::time::timeout(Duration::from_millis(50), unix_listener.accept()) - .await - .is_err(), - "Unix connector ran while trust reload gated the relay" - ); - - let mut rejected_tls = tokio::net::TcpStream::connect(https_listen).await.unwrap(); - assert_eq!( - tokio::time::timeout(Duration::from_millis(250), rejected_tls.read(&mut eof)) - .await - .expect("TLS route did not reject during trust reload") - .unwrap(), - 0 - ); - tls_tcp.set_nonblocking(true).unwrap(); - assert_eq!( - tls_tcp.accept().unwrap_err().kind(), - std::io::ErrorKind::WouldBlock, - "TLS connector ran while trust reload gated the relay" + control.is_ready(), + "staged reload must preserve the current generation until failure is known" ); resume_reload.send(()).unwrap(); @@ -2417,9 +2801,8 @@ MC4CAQAwBQYDK2VwBCIEIGRXBokZ2/yO2kASVZKtUVGnOwIM7kZJKJgugoeMCRxd transport: AccessFlowRelayTransport::TlsTcp { address: remote_address.to_string(), server_name: fixture.server_name().host().dns_name().unwrap().to_string(), - trust: AccessFlowRelayTrust::PemBundle { - path: trust_path.display().to_string(), - }, + trust: TlsClientTrustMode::Custom, + ca_certificate: Some(trust_path.display().to_string()), }, }], }; @@ -2468,25 +2851,9 @@ MC4CAQAwBQYDK2VwBCIEIGRXBokZ2/yO2kASVZKtUVGnOwIM7kZJKJgugoeMCRxd .unwrap() .unwrap(); assert!( - !control.is_ready(), - "reload initiation did not synchronously close admission" + control.is_ready(), + "staged trust reload must preserve current-generation readiness" ); - let mut rejected = tokio::net::TcpStream::connect(listen).await.unwrap(); - let mut byte = [0_u8; 1]; - let rejection = tokio::time::timeout(Duration::from_secs(1), rejected.read(&mut byte)) - .await - .expect("reload-gated route did not close promptly"); - assert!( - matches!(rejection, Ok(0) | Err(_)), - "reload-gated route admitted application bytes" - ); - remote.set_nonblocking(true).unwrap(); - assert_eq!( - remote.accept().unwrap_err().kind(), - std::io::ErrorKind::WouldBlock, - "reload-gated route opened a stale-trust upstream connection" - ); - assert!(!control.is_ready()); assert!( control.initiate_security_reload().is_err(), "concurrent reload was not coalesced" @@ -2717,6 +3084,109 @@ MC4CAQAwBQYDK2VwBCIEIGRXBokZ2/yO2kASVZKtUVGnOwIM7kZJKJgugoeMCRxd ); } + #[tokio::test] + async fn slow_relay_dependent_stop_does_not_consume_the_flow_drain_window() { + let dir = tempfile::tempdir().unwrap(); + let endpoint = dir.path().join("access-flow.sock"); + let endpoint_listener = std::os::unix::net::UnixListener::bind(&endpoint).unwrap(); + endpoint_listener.set_nonblocking(true).unwrap(); + let endpoint_listener = tokio::net::UnixListener::from_std(endpoint_listener).unwrap(); + let source = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + let listen = source.local_addr().unwrap(); + drop(source); + let mut config = test_config(listen.to_string(), endpoint.display().to_string()); + config.drain_timeout = "200ms".into(); + let (control, commands) = RelayControl::configured(&config); + let dependent = Arc::new(ManagedService::new( + ServiceConfig { + name: "relay-dependent".into(), + required: true, + user: "root".into(), + command: vec!["sleep".into(), "infinity".into()], + cwd: None, + restart: RestartPolicy::Never, + restart_backoff: None, + restart_backoff_max: None, + startup_timeout: None, + shutdown_timeout: Some("1s".into()), + depends_on: vec![crate::config::ACCESS_FLOW_RELAY_NODE.into()], + env: BTreeMap::new(), + health_check: None, + }, + dir.path().to_path_buf(), + LoggingConfig::default(), + )); + let state = Arc::new(AgentState::new( + dir.path().join("state"), + None, + false, + None, + None, + Some(control.clone()), + )); + *state.services.lock().await = vec![Arc::clone(&dependent)]; + let supervisor = tokio::spawn(run_relay_supervisor( + config, + IdentityPresentation::Disabled, + Vec::new(), + Arc::clone(&state), + control.clone(), + commands, + )); + tokio::time::timeout(Duration::from_secs(2), async { + while !control.is_ready() { + sleep(Duration::from_millis(10)).await; + } + }) + .await + .unwrap(); + let mut client = tokio::net::TcpStream::connect(listen).await.unwrap(); + let (mut channel, _) = endpoint_listener.accept().await.unwrap(); + let mut preface = [0_u8; 16]; + channel.read_exact(&mut preface).await.unwrap(); + tokio::time::timeout(Duration::from_secs(1), async { + while control.active_flows() != 1 { + tokio::task::yield_now().await; + } + }) + .await + .unwrap(); + + let child_guard = dependent.child.lock().await; + let shutdown_state = Arc::clone(&state); + let shutdown = + tokio::spawn( + async move { super::super::lifecycle::shutdown_agent(shutdown_state).await }, + ); + tokio::time::timeout(Duration::from_secs(1), async { + while !dependent.stopping.load(Ordering::Acquire) { + tokio::task::yield_now().await; + } + }) + .await + .unwrap(); + sleep(Duration::from_millis(150)).await; + assert!(!shutdown.is_finished()); + drop(child_guard); + + sleep(Duration::from_millis(75)).await; + assert!( + !shutdown.is_finished(), + "relay-dependent service stop consumed the flow drain window" + ); + client.write_all(b"fresh-window").await.unwrap(); + let mut received = [0_u8; 12]; + channel.read_exact(&mut received).await.unwrap(); + assert_eq!(&received, b"fresh-window"); + + tokio::time::timeout(Duration::from_secs(1), shutdown) + .await + .unwrap() + .unwrap(); + supervisor.await.unwrap().unwrap(); + assert_eq!(control.active_flows(), 0); + } + #[tokio::test] async fn fatal_listener_path_retains_live_flows_until_root_ordered_shutdown() { let dir = tempfile::tempdir().unwrap(); diff --git a/src/agent/relay_transport.rs b/src/agent/relay_transport.rs index 9c40e1d..b97466f 100644 --- a/src/agent/relay_transport.rs +++ b/src/agent/relay_transport.rs @@ -5,58 +5,106 @@ use access_flow_relay::{ AccessFlowConnector, AccessFlowRelayPlan, }; use access_flow_tls::{ - ACTIVE_CHANNEL_BYTES, GENERATION_CONTROL_BYTES, MAX_DER_CERTIFICATE_BYTES, - MAX_TRUST_ANCHOR_BYTES, MAX_TRUST_ANCHORS, TlsAccessFlowClientEndpoint, - TlsAccessFlowClientLimits, TlsAccessFlowConnector, TlsAccessFlowGeneration, - TlsAccessFlowPreparedClientEndpoint, TlsAccessFlowStream, TlsAccessFlowTrust, - project_tls_client_resources, + ACTIVE_CHANNEL_BYTES, CLIENT_DNS_ADDRESS_BYTES, CLIENT_PEER_CHAIN_BYTES, HANDSHAKE_BYTES, + TlsAccessFlowClientEndpoint, TlsAccessFlowClientLimits, TlsAccessFlowConnector, + TlsAccessFlowPreparedClientEndpoint, TlsAccessFlowStream, }; use access_flow_unix::{UnixAccessFlowConnector, UnixAccessFlowStream}; +use access_tls_trust::{ + CUSTOM_COMPONENT_RETAINED_CEILING, EFFECTIVE_PLAN_CONTROL_BYTES, GENERATION_CONTROL_BYTES, + MAX_CUSTOM_DER_BYTES, MAX_CUSTOM_PEM_BYTES, MAX_SYSTEM_DER_BYTES, PreparedTlsTrustCandidate, + SYSTEM_COMPONENT_RETAINED_CEILING, TRUST_SOURCE_WORKSPACE_CONTROL_BYTES, TlsClientTrustMode, + TlsClientTrustPlan, TlsTrustFileSource, TlsTrustGeneration, TlsTrustLoadError, + TlsTrustLoadLimits, TlsTrustLoadWorkspace, TlsTrustRevalidationProgress, + WORKSPACE_SCRATCH_BYTES, +}; use std::collections::{BTreeMap, BTreeSet}; use std::error::Error; use std::fmt; use std::io; -use std::path::{Component, Path, PathBuf}; +use std::path::PathBuf; use std::pin::Pin; use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; -use std::sync::{Arc, Mutex}; +use std::sync::{Arc, Mutex, OnceLock, Weak}; use std::task::{Context, Poll}; +use std::time::Duration; use tokio::io::{AsyncRead, AsyncWrite, ReadBuf}; use tokio::sync::watch; +use tokio::time::Instant; +use tokio_util::sync::CancellationToken; -const CERTIFICATE_BEGIN: &[u8] = b"-----BEGIN CERTIFICATE-----"; -const CERTIFICATE_END: &[u8] = b"-----END CERTIFICATE-----"; -const MAX_TRUST_SOURCE_BYTES: usize = 1024 * 1024; -const MAX_TRUST_READ_BUFFER_BYTES: usize = MAX_TRUST_SOURCE_BYTES + 1; -const TRUST_LOAD_DESCRIPTOR_ENVELOPE: u64 = 2; -// Frozen envelopes for Vec/Box/BTree/CString/Arc metadata after exact -// address, name, path, certificate, and policy bytes are counted. -const TLS_ROUTE_CONTROL_BYTES: u64 = 4 * 1024; -const TLS_UNIQUE_SOURCE_CONTROL_BYTES: u64 = 4 * 1024; +const MAX_PUBLISHED_TRUST_GENERATIONS: usize = 8; +#[cfg(test)] +const TEST_TLS_TRUST_BUDGET_BYTES: u64 = 256 * 1024 * 1024; +#[cfg(test)] +const TEST_TLS_TRUST_DESCRIPTOR_BUDGET: u64 = 32; +const TLS_ENDPOINT_CONTROL_BYTES: u64 = 8 * 1024; +const TRUST_OPERATION_TIMEOUT: Duration = Duration::from_secs(30); +const TRUST_CANCELLATION_JOIN_GRACE: Duration = Duration::from_millis(10); +const TRUST_CANDIDATE_BASE_TIMEOUT: Duration = Duration::from_secs(60); +const TRUST_CANDIDATE_MAX_TIMEOUT: Duration = Duration::from_secs(180); /// Fixed, path-free product transport failure. #[derive(Clone, Copy, Debug, Eq, PartialEq)] pub(super) enum RelayTransportError { - Unavailable, + SystemStoreEmpty, + SystemStoreUnavailable, UntrustedSource, - ResourceLimit, InvalidMaterial, + SourceResourceLimit, InvalidPlan, + Cancelled, + Internal, + TrustGenerationLimit, ReloadInProgress, - Coordinator, + Unavailable, ShuttingDown, } +impl RelayTransportError { + fn keeps_active_generation_ready(self) -> bool { + matches!( + self, + Self::SystemStoreEmpty + | Self::SystemStoreUnavailable + | Self::Cancelled + | Self::TrustGenerationLimit + | Self::ReloadInProgress + ) + } + + pub(super) const fn status_code(self) -> &'static str { + match self { + Self::SystemStoreEmpty => "system_store_empty", + Self::SystemStoreUnavailable => "system_store_unavailable", + Self::UntrustedSource => "untrusted_source", + Self::InvalidMaterial => "invalid_material", + Self::SourceResourceLimit => "trust_source_resource_limit", + Self::InvalidPlan => "invalid_plan", + Self::Cancelled => "cancelled", + Self::Internal => "internal", + Self::TrustGenerationLimit => "trust_generation_limit", + Self::ReloadInProgress => "trust_reload_blocked", + Self::Unavailable => "unavailable", + Self::ShuttingDown => "shutting_down", + } + } +} + impl fmt::Display for RelayTransportError { fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { formatter.write_str(match self { - Self::Unavailable => "Access Flow trust source is unavailable", + Self::SystemStoreEmpty => "Access Flow system trust store is empty", + Self::SystemStoreUnavailable => "Access Flow system trust store is unavailable", Self::UntrustedSource => "Access Flow trust source is not trusted", - Self::ResourceLimit => "Access Flow trust source exceeds a resource bound", Self::InvalidMaterial => "Access Flow trust material is invalid", + Self::SourceResourceLimit => "Access Flow trust source exceeds a resource bound", Self::InvalidPlan => "Access Flow transport plan is invalid", + Self::Cancelled => "Access Flow trust reload was cancelled", + Self::Internal => "Access Flow transport coordinator failed", + Self::TrustGenerationLimit => "Access Flow trust generation limit was reached", Self::ReloadInProgress => "Access Flow transport reload is already in progress", - Self::Coordinator => "Access Flow transport coordinator failed", + Self::Unavailable => "Access Flow transport is unavailable", Self::ShuttingDown => "Access Flow transport is shutting down", }) } @@ -64,16 +112,43 @@ impl fmt::Display for RelayTransportError { impl Error for RelayTransportError {} +impl From for RelayTransportError { + fn from(error: TlsTrustLoadError) -> Self { + match error { + TlsTrustLoadError::SystemStoreEmpty => Self::SystemStoreEmpty, + TlsTrustLoadError::SystemStoreUnavailable => Self::SystemStoreUnavailable, + TlsTrustLoadError::UntrustedSource => Self::UntrustedSource, + TlsTrustLoadError::InvalidMaterial => Self::InvalidMaterial, + TlsTrustLoadError::ResourceLimit => Self::SourceResourceLimit, + TlsTrustLoadError::InvalidPlan => Self::InvalidPlan, + TlsTrustLoadError::Cancelled => Self::Cancelled, + TlsTrustLoadError::Internal => Self::Internal, + } + } +} + #[derive(Clone)] struct TlsRouteSource { address: access_flow_tls::TlsAccessFlowAddress, server_name: access_flow_tls::TlsAccessFlowServerName, - trust_path: PathBuf, + mode: TlsClientTrustMode, + plan: TlsClientTrustPlan, + trust_path: Option, } -struct RelayTransportGeneration { - id: TlsAccessFlowGeneration, +pub(super) struct RelayTransportGeneration { + id: TlsTrustGeneration, tls_endpoints: Box<[TlsAccessFlowPreparedClientEndpoint]>, + trust_candidate: Option, +} + +impl RelayTransportGeneration { + fn retained_bytes(&self) -> u64 { + self.trust_candidate + .as_ref() + .map(|candidate| candidate.resource_projection().retained_bytes()) + .unwrap_or(0) + } } #[derive(Clone)] @@ -90,8 +165,12 @@ struct RelayTransportInner { readiness: Arc, next_generation: AtomicU64, reload_in_progress: AtomicBool, + blocked_operation: AtomicBool, + trust_budget: OnceLock, publication: Mutex<()>, + retained_generations: Mutex>>, shutdown_started: AtomicBool, + shutdown_cancellation: CancellationToken, unix: UnixAccessFlowConnector, tls: TlsAccessFlowConnector, } @@ -108,6 +187,12 @@ pub(super) struct RelayTransportResourceReserve { pub(super) memory_bytes: u64, } +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(super) struct RelayTransportTrustBudget { + pub(super) descriptors: u64, + pub(super) memory_bytes: u64, +} + impl RelayTransportRuntime { pub(super) fn prepare( plan: &AccessFlowRelayPlan, @@ -124,8 +209,12 @@ impl RelayTransportRuntime { readiness, next_generation: AtomicU64::new(2), reload_in_progress: AtomicBool::new(false), + blocked_operation: AtomicBool::new(false), + trust_budget: OnceLock::new(), publication: Mutex::new(()), + retained_generations: Mutex::new(Vec::new()), shutdown_started: AtomicBool::new(false), + shutdown_cancellation: CancellationToken::new(), unix: UnixAccessFlowConnector::new(), tls: TlsAccessFlowConnector::with_system_resolver( TlsAccessFlowClientLimits::default(), @@ -140,12 +229,53 @@ impl RelayTransportRuntime { readiness: Arc, ) -> Result { let runtime = Self::prepare(plan, readiness)?; + runtime.configure_trust_budget(RelayTransportTrustBudget { + descriptors: TEST_TLS_TRUST_DESCRIPTOR_BUDGET, + memory_bytes: TEST_TLS_TRUST_BUDGET_BYTES, + })?; runtime.activate_prepared().await?; Ok(runtime) } + pub(super) fn configure_trust_budget( + &self, + budget: RelayTransportTrustBudget, + ) -> Result<(), RelayTransportError> { + let candidate_memory = project_candidate_loader_peak(&self.inner.sources)?; + let candidate_descriptors = projected_candidate_descriptors(&self.inner.sources)?; + if !self.inner.sources.is_empty() + && (budget.descriptors == 0 + || budget.memory_bytes == 0 + || candidate_memory > budget.memory_bytes + || candidate_descriptors > budget.descriptors) + { + return Err(RelayTransportError::SourceResourceLimit); + } + self.inner + .trust_budget + .set(budget) + .map_err(|_| RelayTransportError::InvalidPlan) + } + pub(super) async fn activate_prepared(&self) -> Result<(), RelayTransportError> { - let initial = load_generation(self.inner.sources.to_vec(), 1).await?; + if self.inner.blocked_operation.load(Ordering::Acquire) { + return Err(RelayTransportError::ReloadInProgress); + } + let budget = self.inner.trust_budget()?; + let initial = match load_generation( + self.inner.sources.to_vec(), + 1, + budget, + self.inner.shutdown_cancellation.clone(), + ) + .await + { + GenerationLoadOutcome::Finished(result) => result?, + GenerationLoadOutcome::TimedOut(watcher) => { + self.inner.install_blocked_operation(watcher); + return Err(RelayTransportError::Cancelled); + } + }; let _publication = self .inner .publication @@ -155,9 +285,10 @@ impl RelayTransportRuntime { return Err(RelayTransportError::ShuttingDown); } if !matches!(&*self.inner.state.borrow(), RelayTransportState::Pending) { - return Err(RelayTransportError::Coordinator); + return Err(RelayTransportError::Internal); } - self.inner.publish_generation_as_healthy(Arc::new(initial)); + self.inner + .publish_generation_as_healthy(Arc::new(initial))?; Ok(()) } @@ -167,16 +298,21 @@ impl RelayTransportRuntime { } } + pub(super) fn reload_blocked(&self) -> bool { + self.inner.blocked_operation.load(Ordering::Acquire) + } + + #[cfg(test)] + pub(super) fn install_test_blocked_operation(&self, watcher: tokio::task::JoinHandle<()>) { + self.inner.install_blocked_operation(watcher); + } + pub(super) fn resource_reserve( plan: &AccessFlowRelayPlan, ) -> Result { let sources = collect_tls_sources(plan)?; Ok(RelayTransportResourceReserve { - descriptors: if sources.is_empty() { - 0 - } else { - TRUST_LOAD_DESCRIPTOR_ENVELOPE - }, + descriptors: projected_candidate_descriptors(&sources)?, memory_bytes: project_candidate_loader_peak(&sources)?, }) } @@ -210,6 +346,9 @@ impl RelayTransportRuntime { if matches!(&*self.inner.state.borrow(), RelayTransportState::Pending) { return Err(RelayTransportError::Unavailable); } + if self.inner.blocked_operation.load(Ordering::Acquire) { + return Err(RelayTransportError::ReloadInProgress); + } if self.inner.sources.is_empty() { return Ok(None); } @@ -221,7 +360,21 @@ impl RelayTransportRuntime { { return Err(RelayTransportError::ReloadInProgress); } - self.inner.preserve_generation_as_unhealthy(); + let retained_bytes = self.inner.retained_generation_bytes()?; + let candidate_peak = project_candidate_loader_peak(&self.inner.sources)?; + let budget = self.inner.trust_budget()?; + let Some(working_limit) = budget.memory_bytes.checked_sub(retained_bytes) else { + self.inner + .reload_in_progress + .store(false, Ordering::Release); + return Err(RelayTransportError::TrustGenerationLimit); + }; + if candidate_peak > working_limit { + self.inner + .reload_in_progress + .store(false, Ordering::Release); + return Err(RelayTransportError::TrustGenerationLimit); + } let generation = match self.inner.next_generation.fetch_update( Ordering::AcqRel, Ordering::Acquire, @@ -232,13 +385,26 @@ impl RelayTransportRuntime { self.inner .reload_in_progress .store(false, Ordering::Release); - return Err(RelayTransportError::ResourceLimit); + return Err(RelayTransportError::TrustGenerationLimit); } }; let sources = self.inner.sources.to_vec(); - let worker = tokio::task::spawn_blocking(move || { - worker_started(); - load_generation_blocking(&sources, generation) + let shutdown_cancellation = self.inner.shutdown_cancellation.clone(); + let worker = tokio::spawn(async move { + let hook = tokio::task::spawn_blocking(worker_started); + if hook.await.is_err() { + return GenerationLoadOutcome::Finished(Err(RelayTransportError::Internal)); + } + load_generation( + sources, + generation, + RelayTransportTrustBudget { + descriptors: budget.descriptors, + memory_bytes: working_limit, + }, + shutdown_cancellation, + ) + .await }); Ok(Some(PendingRelayTransportReload { inner: Arc::clone(&self.inner), @@ -246,12 +412,8 @@ impl RelayTransportRuntime { })) } - #[cfg(test)] - fn healthy(&self) -> bool { - self.inner.readiness.load(Ordering::Acquire) - } - pub(super) fn close(&self) { + self.inner.shutdown_cancellation.cancel(); let _publication = self .inner .publication @@ -273,30 +435,62 @@ impl fmt::Debug for RelayTransportRuntime { #[must_use = "pending transport reload workers must be joined"] pub(super) struct PendingRelayTransportReload { inner: Arc, - worker: Option>>, + worker: Option>, } impl PendingRelayTransportReload { - /// Joins and publishes this reload. Cancelling the wait leaves the worker owned here. pub(super) async fn wait(&mut self) -> Result<(), RelayTransportError> { - let loaded = { - let worker = self - .worker - .as_mut() - .ok_or(RelayTransportError::Coordinator)?; - worker - .await - .unwrap_or(Err(RelayTransportError::Coordinator)) + let outcome = { + let worker = self.worker.as_mut().ok_or(RelayTransportError::Internal)?; + worker.await.unwrap_or(GenerationLoadOutcome::Finished(Err( + RelayTransportError::Internal, + ))) }; self.worker = None; - self.publish(loaded) + match outcome { + GenerationLoadOutcome::Finished(loaded) => self.publish(loaded), + GenerationLoadOutcome::TimedOut(watcher) => { + self.inner.install_blocked_operation(watcher); + Err(RelayTransportError::Cancelled) + } + } } - /// Final join used by terminal shutdown after the runtime is closed. pub(super) async fn complete(mut self) -> Result<(), RelayTransportError> { self.wait().await } + pub(super) async fn complete_by( + mut self, + deadline: Instant, + ) -> Result<(), RelayTransportError> { + let mut worker = self.worker.take().ok_or(RelayTransportError::Internal)?; + match tokio::time::timeout_at(deadline, &mut worker).await { + Ok(outcome) => match outcome.unwrap_or(GenerationLoadOutcome::Finished(Err( + RelayTransportError::Internal, + ))) { + GenerationLoadOutcome::Finished(loaded) => self.publish(loaded), + GenerationLoadOutcome::TimedOut(watcher) => { + self.inner.install_blocked_operation(watcher); + Err(RelayTransportError::Cancelled) + } + }, + Err(_) => { + self.inner.shutdown_cancellation.cancel(); + tokio::task::yield_now().await; + if worker.is_finished() { + let _ = worker.await; + self.inner + .reload_in_progress + .store(false, Ordering::Release); + } else { + self.inner.install_blocked_candidate(worker); + } + Err(RelayTransportError::Cancelled) + } + } + } + fn publish( &self, loaded: Result, @@ -310,12 +504,15 @@ impl PendingRelayTransportReload { Err(RelayTransportError::ShuttingDown) } else { match loaded { - Ok(generation) => { - self.inner - .publish_generation_as_healthy(Arc::new(generation)); - Ok(()) + Ok(generation) => self + .inner + .publish_generation_as_healthy(Arc::new(generation)), + Err(error) => { + if !error.keeps_active_generation_ready() { + self.inner.preserve_generation_as_unhealthy(); + } + Err(error) } - Err(error) => Err(error), } }; self.inner @@ -332,44 +529,100 @@ impl fmt::Debug for PendingRelayTransportReload { } impl RelayTransportInner { - fn preserve_generation_as_unhealthy(&self) { - self.preserve_generation_as_unhealthy_with(|| {}); + fn trust_budget(&self) -> Result { + self.trust_budget + .get() + .copied() + .ok_or(RelayTransportError::InvalidPlan) + } + + fn install_blocked_operation(self: &Arc, watcher: tokio::task::JoinHandle<()>) { + self.blocked_operation.store(true, Ordering::Release); + let inner = Arc::clone(self); + tokio::spawn(async move { + let _ = watcher.await; + inner.blocked_operation.store(false, Ordering::Release); + inner.reload_in_progress.store(false, Ordering::Release); + }); + } + + fn install_blocked_candidate( + self: &Arc, + watcher: tokio::task::JoinHandle, + ) { + self.blocked_operation.store(true, Ordering::Release); + let inner = Arc::clone(self); + tokio::spawn(async move { + if let Ok(GenerationLoadOutcome::TimedOut(operation)) = watcher.await { + let _ = operation.await; + } + inner.blocked_operation.store(false, Ordering::Release); + inner.reload_in_progress.store(false, Ordering::Release); + }); + } + fn retained_generation_bytes(&self) -> Result { + let mut retained = self + .retained_generations + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + retained.retain(|generation| generation.strong_count() > 0); + retained + .iter() + .filter_map(Weak::upgrade) + .try_fold(0_u64, |total, generation| { + total.checked_add(generation.retained_bytes()) + }) + .ok_or(RelayTransportError::TrustGenerationLimit) } - fn preserve_generation_as_unhealthy_with(&self, after_readiness: impl FnOnce()) { + fn preserve_generation_as_unhealthy(&self) { let current = match self.state.borrow().clone() { RelayTransportState::Healthy(generation) | RelayTransportState::Unhealthy(generation) => generation, RelayTransportState::Pending | RelayTransportState::Closed => return, }; self.readiness.store(false, Ordering::Release); - after_readiness(); self.state .send_replace(RelayTransportState::Unhealthy(current)); } - fn publish_generation_as_healthy(&self, generation: Arc) { - self.publish_generation_as_healthy_with(generation, || {}); - } - - fn publish_generation_as_healthy_with( + fn publish_generation_as_healthy( &self, generation: Arc, - after_state: impl FnOnce(), - ) { + ) -> Result<(), RelayTransportError> { + let mut retained = self + .retained_generations + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + retained.retain(|generation| generation.strong_count() > 0); + let retained_bytes = retained + .iter() + .filter_map(Weak::upgrade) + .try_fold(0_u64, |total, generation| { + total.checked_add(generation.retained_bytes()) + }) + .ok_or(RelayTransportError::TrustGenerationLimit)?; + if retained.len() >= MAX_PUBLISHED_TRUST_GENERATIONS + || retained_bytes + .checked_add(generation.retained_bytes()) + .is_none_or(|bytes| { + self.trust_budget + .get() + .is_none_or(|budget| bytes > budget.memory_bytes) + }) + { + return Err(RelayTransportError::TrustGenerationLimit); + } + retained.push(Arc::downgrade(&generation)); + drop(retained); self.state .send_replace(RelayTransportState::Healthy(generation)); - after_state(); self.readiness.store(true, Ordering::Release); + Ok(()) } fn publish_closed(&self) { - self.publish_closed_with(|| {}); - } - - fn publish_closed_with(&self, after_readiness: impl FnOnce()) { self.readiness.store(false, Ordering::Release); - after_readiness(); self.state.send_replace(RelayTransportState::Closed); } @@ -382,16 +635,7 @@ impl RelayTransportInner { } } - #[cfg(test)] - fn retained_generation(&self) -> Option> { - match self.state.borrow().clone() { - RelayTransportState::Healthy(generation) - | RelayTransportState::Unhealthy(generation) => Some(generation), - RelayTransportState::Pending | RelayTransportState::Closed => None, - } - } - - fn generation_is_current(&self, expected: TlsAccessFlowGeneration) -> bool { + fn generation_is_current(&self, expected: TlsTrustGeneration) -> bool { matches!( &*self.state.borrow(), RelayTransportState::Healthy(generation) if generation.id == expected @@ -399,7 +643,6 @@ impl RelayTransportInner { } } -/// Exact product connector over the mutually exclusive Unix and TLS adapters. #[derive(Clone)] pub(super) struct RelayTransportConnector { inner: Arc, @@ -411,17 +654,19 @@ impl fmt::Debug for RelayTransportConnector { } } -/// Product stream sum that does not expose either physical adapter upstream. pub(super) enum RelayTransportStream { Unix(UnixAccessFlowStream), - Tls(TlsAccessFlowStream), + Tls { + stream: TlsAccessFlowStream, + _generation: Arc, + }, } impl fmt::Debug for RelayTransportStream { fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { formatter.write_str(match self { Self::Unix(_) => "RelayTransportStream::Unix", - Self::Tls(_) => "RelayTransportStream::Tls", + Self::Tls { .. } => "RelayTransportStream::Tls", }) } } @@ -434,7 +679,7 @@ impl AsyncRead for RelayTransportStream { ) -> Poll> { match self.get_mut() { Self::Unix(stream) => Pin::new(stream).poll_read(context, buffer), - Self::Tls(stream) => Pin::new(stream).poll_read(context, buffer), + Self::Tls { stream, .. } => Pin::new(stream).poll_read(context, buffer), } } } @@ -447,21 +692,21 @@ impl AsyncWrite for RelayTransportStream { ) -> Poll> { match self.get_mut() { Self::Unix(stream) => Pin::new(stream).poll_write(context, buffer), - Self::Tls(stream) => Pin::new(stream).poll_write(context, buffer), + Self::Tls { stream, .. } => Pin::new(stream).poll_write(context, buffer), } } fn poll_flush(self: Pin<&mut Self>, context: &mut Context<'_>) -> Poll> { match self.get_mut() { Self::Unix(stream) => Pin::new(stream).poll_flush(context), - Self::Tls(stream) => Pin::new(stream).poll_flush(context), + Self::Tls { stream, .. } => Pin::new(stream).poll_flush(context), } } fn poll_shutdown(self: Pin<&mut Self>, context: &mut Context<'_>) -> Poll> { match self.get_mut() { Self::Unix(stream) => Pin::new(stream).poll_shutdown(context), - Self::Tls(stream) => Pin::new(stream).poll_shutdown(context), + Self::Tls { stream, .. } => Pin::new(stream).poll_shutdown(context), } } } @@ -502,7 +747,10 @@ impl AccessFlowConnector for RelayTransportConnector { .tls .connect(endpoint, gated_context) .await - .map(RelayTransportStream::Tls) + .map(|stream| RelayTransportStream::Tls { + stream, + _generation: Arc::clone(&generation), + }) } }?; if self.inner.generation_is_current(generation.id) { @@ -522,18 +770,29 @@ impl AccessFlowConnector for RelayTransportConnector { CompiledAccessFlowRelayEndpoint::Unix(endpoint) => { self.inner.unix.resource_projection(endpoint) } - CompiledAccessFlowRelayEndpoint::TlsTcp { - address, - server_name, - trust_path, - .. - } => maximum_tls_channel_cost(address, server_name, trust_path), + CompiledAccessFlowRelayEndpoint::TlsTcp { .. } => { + let connecting_bytes = ACTIVE_CHANNEL_BYTES + .checked_add(HANDSHAKE_BYTES) + .and_then(|bytes| bytes.checked_add(CLIENT_PEER_CHAIN_BYTES)) + .and_then(|bytes| bytes.checked_add(CLIENT_DNS_ADDRESS_BYTES)) + .ok_or(AccessFlowChannelFailure::ResourceExhausted)?; + let active_bytes = ACTIVE_CHANNEL_BYTES + .checked_add(CLIENT_PEER_CHAIN_BYTES) + .ok_or(AccessFlowChannelFailure::ResourceExhausted)?; + AccessFlowChannelResourceCost::new( + TLS_ENDPOINT_CONTROL_BYTES, + 1, + 1, + connecting_bytes, + active_bytes, + ) + } } } } struct GenerationCancellation<'a> { - expected: TlsAccessFlowGeneration, + expected: TlsTrustGeneration, state: watch::Receiver, caller: &'a dyn AccessCancellation, } @@ -579,82 +838,6 @@ impl AccessCancellation for GenerationCancellation<'_> { } } -fn tls_endpoint_metadata_bytes( - address: &access_flow_tls::TlsAccessFlowAddress, - server_name: &access_flow_tls::TlsAccessFlowServerName, -) -> Result { - let marker_generation = - TlsAccessFlowGeneration::new(1).map_err(|_| RelayTransportError::ResourceLimit)?; - let marker_trust = TlsAccessFlowTrust::new(marker_generation, vec![vec![0_u8]]) - .map_err(map_trust_contract_error)?; - let marker = - TlsAccessFlowClientEndpoint::new(address.clone(), server_name.clone(), marker_trust); - project_tls_client_resources(&marker, 1, 1) - .map(|projection| projection.endpoint_bytes()) - .map_err(map_trust_contract_error) -} - -fn maximum_tls_channel_cost( - address: &access_flow_tls::TlsAccessFlowAddress, - server_name: &access_flow_tls::TlsAccessFlowServerName, - trust_path: &Path, -) -> Result { - let endpoint_metadata = tls_endpoint_metadata_bytes(address, server_name) - .map_err(|_| AccessFlowChannelFailure::ResourceExhausted)?; - let trust_path_bytes = projected_trust_path_bytes(trust_path) - .map_err(|_| AccessFlowChannelFailure::InvalidEndpoint)?; - let maximum_generation = (MAX_TRUST_ANCHOR_BYTES as u64) - .checked_add(GENERATION_CONTROL_BYTES) - .ok_or(AccessFlowChannelFailure::ResourceExhausted)?; - let retained_endpoint_bytes = endpoint_metadata - .checked_mul(2) - .and_then(|bytes| bytes.checked_add(trust_path_bytes)) - .and_then(|bytes| bytes.checked_add(maximum_generation)) - .ok_or(AccessFlowChannelFailure::ResourceExhausted)?; - let connecting_bytes = access_flow_tls::HANDSHAKE_BYTES - .checked_add(access_flow_tls::CLIENT_DNS_ADDRESS_BYTES) - .ok_or(AccessFlowChannelFailure::ResourceExhausted)?; - AccessFlowChannelResourceCost::new( - retained_endpoint_bytes, - 1, - 1, - connecting_bytes, - // Established streams can retain policies from distinct prior reload - // generations, so the maximum policy remains per active flow. - ACTIVE_CHANNEL_BYTES - .checked_add(access_flow_tls::CLIENT_PEER_CHAIN_BYTES) - .and_then(|bytes| bytes.checked_add(maximum_generation)) - .ok_or(AccessFlowChannelFailure::ResourceExhausted)?, - ) -} - -fn projected_trust_path_bytes(path: &Path) -> Result { - let value = path.to_str().ok_or(RelayTransportError::InvalidPlan)?; - if value.len() > crate::config::MAX_ACCESS_FLOW_TRUST_PATH_BYTES { - return Err(RelayTransportError::ResourceLimit); - } - let mut components = path.components(); - if components.next() != Some(Component::RootDir) { - return Err(RelayTransportError::InvalidPlan); - } - let mut count = 0_usize; - for component in components { - if !matches!(component, Component::Normal(_)) { - return Err(RelayTransportError::InvalidPlan); - } - count = count - .checked_add(1) - .ok_or(RelayTransportError::ResourceLimit)?; - if count > crate::config::MAX_ACCESS_FLOW_TRUST_PATH_COMPONENTS { - return Err(RelayTransportError::ResourceLimit); - } - } - if count == 0 { - return Err(RelayTransportError::InvalidPlan); - } - u64::try_from(value.len()).map_err(|_| RelayTransportError::ResourceLimit) -} - fn collect_tls_sources( plan: &AccessFlowRelayPlan, ) -> Result, RelayTransportError> { @@ -664,15 +847,18 @@ fn collect_tls_sources( tls_index, address, server_name, + trust_mode, + trust_plan, trust_path, } = route.endpoint() else { continue; }; - projected_trust_path_bytes(trust_path)?; let source = TlsRouteSource { address: address.clone(), server_name: server_name.clone(), + mode: *trust_mode, + plan: *trust_plan, trust_path: trust_path.clone(), }; if indexed.insert(*tls_index, source).is_some() { @@ -690,107 +876,245 @@ fn collect_tls_sources( Ok(sources) } -fn project_candidate_loader_peak(sources: &[TlsRouteSource]) -> Result { +fn projected_candidate_descriptors(sources: &[TlsRouteSource]) -> Result { if sources.is_empty() { return Ok(0); } - let routes = u64::try_from(sources.len()).map_err(|_| RelayTransportError::ResourceLimit)?; - let unique_paths = sources + let custom_sources = sources .iter() - .map(|source| &source.trust_path) - .collect::>(); - let unique_source_count = - u64::try_from(unique_paths.len()).map_err(|_| RelayTransportError::ResourceLimit)?; - let maximum_generation = (MAX_TRUST_ANCHOR_BYTES as u64) - .checked_add(GENERATION_CONTROL_BYTES) - .ok_or(RelayTransportError::ResourceLimit)?; - let mut route_source_bytes = 0_u64; - let mut candidate_endpoint_bytes = 0_u64; - for source in sources { - let endpoint_metadata = tls_endpoint_metadata_bytes(&source.address, &source.server_name)?; - let path_bytes = projected_trust_path_bytes(&source.trust_path)?; - route_source_bytes = route_source_bytes - .checked_add(endpoint_metadata) - .and_then(|bytes| bytes.checked_add(path_bytes)) - .ok_or(RelayTransportError::ResourceLimit)?; - candidate_endpoint_bytes = candidate_endpoint_bytes - .checked_add(endpoint_metadata) - .and_then(|bytes| bytes.checked_add(maximum_generation)) - .ok_or(RelayTransportError::ResourceLimit)?; - } - let unique_path_bytes = unique_paths.iter().try_fold(0_u64, |total, path| { - projected_trust_path_bytes(path) - .ok() - .and_then(|length| total.checked_add(length)) - .ok_or(RelayTransportError::ResourceLimit) - })?; - let loaded_anchor_bytes = unique_source_count - .checked_mul(MAX_TRUST_ANCHOR_BYTES as u64) - .ok_or(RelayTransportError::ResourceLimit)?; - let route_control_bytes = routes - .checked_mul(TLS_ROUTE_CONTROL_BYTES) - .ok_or(RelayTransportError::ResourceLimit)?; - let source_control_bytes = unique_source_count - .checked_mul(TLS_UNIQUE_SOURCE_CONTROL_BYTES) - .ok_or(RelayTransportError::ResourceLimit)?; - let activation_scratch_bytes = routes - .checked_mul(MAX_TRUST_ANCHOR_BYTES as u64) - .ok_or(RelayTransportError::ResourceLimit)?; - route_source_bytes - .checked_mul(2) - .and_then(|bytes| bytes.checked_add(unique_path_bytes)) - .and_then(|bytes| bytes.checked_add(loaded_anchor_bytes)) - .and_then(|bytes| bytes.checked_add(MAX_TRUST_READ_BUFFER_BYTES as u64)) - .and_then(|bytes| bytes.checked_add(crate::config::MAX_ACCESS_FLOW_TRUST_PATH_BYTES as u64)) - .and_then(|bytes| bytes.checked_add(candidate_endpoint_bytes)) - .and_then(|bytes| bytes.checked_add(activation_scratch_bytes)) - .and_then(|bytes| bytes.checked_add(route_control_bytes)) - .and_then(|bytes| bytes.checked_add(source_control_bytes)) - .ok_or(RelayTransportError::ResourceLimit) + .filter_map(|source| source.plan.custom_source()) + .collect::>() + .len(); + u64::try_from(custom_sources) + .ok() + .and_then(|count| count.checked_add(8)) + .ok_or(RelayTransportError::SourceResourceLimit) +} + +fn project_candidate_loader_peak(sources: &[TlsRouteSource]) -> Result { + if sources.is_empty() { + return Ok(0); + } + let plan_count = u64::try_from( + sources + .iter() + .map(|source| source.plan) + .collect::>() + .len(), + ) + .map_err(|_| RelayTransportError::SourceResourceLimit)?; + let custom_count = u64::try_from( + sources + .iter() + .filter_map(|source| source.plan.custom_source()) + .collect::>() + .len(), + ) + .map_err(|_| RelayTransportError::SourceResourceLimit)?; + let system_required = sources.iter().any(|source| { + matches!( + source.mode, + TlsClientTrustMode::System | TlsClientTrustMode::SystemPlusCustom + ) + }); + let retained_base = GENERATION_CONTROL_BYTES + .checked_add( + plan_count + .checked_mul(EFFECTIVE_PLAN_CONTROL_BYTES) + .ok_or(RelayTransportError::SourceResourceLimit)?, + ) + .ok_or(RelayTransportError::SourceResourceLimit)?; + let workspace_base = WORKSPACE_SCRATCH_BYTES + .checked_add( + custom_count + .checked_mul(TRUST_SOURCE_WORKSPACE_CONTROL_BYTES) + .ok_or(RelayTransportError::SourceResourceLimit)?, + ) + .and_then(|bytes| { + bytes.checked_add(if custom_count > 0 { + MAX_CUSTOM_PEM_BYTES as u64 + } else { + 0 + }) + }) + .ok_or(RelayTransportError::SourceResourceLimit)?; + let retained_before_custom = retained_base + .checked_add(if system_required { + SYSTEM_COMPONENT_RETAINED_CEILING + } else { + 0 + }) + .ok_or(RelayTransportError::SourceResourceLimit)?; + let retained_complete = retained_before_custom + .checked_add( + custom_count + .checked_mul(CUSTOM_COMPONENT_RETAINED_CEILING) + .ok_or(RelayTransportError::SourceResourceLimit)?, + ) + .ok_or(RelayTransportError::SourceResourceLimit)?; + let system_step = if system_required { + retained_base + .checked_add(workspace_base) + .and_then(|bytes| bytes.checked_add(SYSTEM_COMPONENT_RETAINED_CEILING)) + .and_then(|bytes| bytes.checked_add(MAX_SYSTEM_DER_BYTES as u64)) + .ok_or(RelayTransportError::SourceResourceLimit)? + } else { + 0 + }; + let custom_step = if custom_count > 0 { + retained_complete + .checked_add(workspace_base) + .and_then(|bytes| bytes.checked_add(MAX_CUSTOM_DER_BYTES as u64)) + .ok_or(RelayTransportError::SourceResourceLimit)? + } else { + 0 + }; + retained_complete + .checked_add(workspace_base) + .map(|complete| complete.max(system_step).max(custom_step)) + .ok_or(RelayTransportError::SourceResourceLimit) } async fn load_generation( sources: Vec, generation: u64, -) -> Result { - tokio::task::spawn_blocking(move || load_generation_blocking(&sources, generation)) - .await - .map_err(|_| RelayTransportError::Coordinator)? + budget: RelayTransportTrustBudget, + shutdown_cancellation: CancellationToken, +) -> GenerationLoadOutcome { + match load_generation_inner(&sources, generation, budget, shutdown_cancellation).await { + Ok(generation) => GenerationLoadOutcome::Finished(Ok(generation)), + Err(GenerationLoadFailure::Error(error)) => GenerationLoadOutcome::Finished(Err(error)), + Err(GenerationLoadFailure::TimedOut(watcher)) => GenerationLoadOutcome::TimedOut(watcher), + } } -fn load_generation_blocking( - sources: &[TlsRouteSource], - generation: u64, -) -> Result { - load_generation_blocking_with_hook(sources, generation, |_| {}) +enum GenerationLoadOutcome { + Finished(Result), + TimedOut(tokio::task::JoinHandle<()>), +} + +enum GenerationLoadFailure { + Error(RelayTransportError), + TimedOut(tokio::task::JoinHandle<()>), +} + +impl From for GenerationLoadFailure { + fn from(error: RelayTransportError) -> Self { + Self::Error(error) + } +} + +impl From for GenerationLoadFailure { + fn from(error: TlsTrustLoadError) -> Self { + Self::Error(error.into()) + } +} + +enum WorkspaceOperation { + LoadSystem, + LoadCustom(TlsTrustFileSource), + Revalidate, + #[cfg(test)] + Block(Arc), + #[cfg(test)] + Mark(Arc), + #[cfg(test)] + WaitForCancellation { + started: Arc, + observed: Arc, + }, } -fn load_generation_blocking_with_hook( +async fn load_generation_inner( sources: &[TlsRouteSource], generation: u64, - mut after_source: impl FnMut(usize), -) -> Result { - let id = - TlsAccessFlowGeneration::new(generation).map_err(|_| RelayTransportError::ResourceLimit)?; - let mut loaded_sources = BTreeMap::::new(); + budget: RelayTransportTrustBudget, + shutdown_cancellation: CancellationToken, +) -> Result { + if sources.is_empty() { + return Ok(RelayTransportGeneration { + id: TlsTrustGeneration::new(generation)?, + tls_endpoints: Box::new([]), + trust_candidate: None, + }); + } + let candidate_started = Instant::now(); + let custom_count = sources + .iter() + .filter_map(|source| source.plan.custom_source()) + .collect::>() + .len(); + let candidate_timeout = candidate_timeout(custom_count)?; + let candidate_deadline = candidate_started + candidate_timeout; + let generation = TlsTrustGeneration::new(generation)?; + let plans = sources + .iter() + .map(|source| source.plan) + .collect::>(); + let limits = TlsTrustLoadLimits::new(budget.memory_bytes, budget.descriptors)?; + let mut workspace = TlsTrustLoadWorkspace::new(generation, plans, limits)?; + if sources.iter().any(|source| { + matches!( + source.mode, + TlsClientTrustMode::System | TlsClientTrustMode::SystemPlusCustom + ) + }) { + let (next, _) = supervise_workspace_operation( + workspace, + WorkspaceOperation::LoadSystem, + candidate_deadline, + &shutdown_cancellation, + ) + .await?; + workspace = next; + } + let mut custom_sources = BTreeMap::new(); for source in sources { - if !loaded_sources.contains_key(&source.trust_path) { - let loaded = load_trust_source(&source.trust_path)?; - loaded_sources.insert(source.trust_path.clone(), loaded); - after_source(loaded_sources.len() - 1); + let Some(path) = &source.trust_path else { + continue; + }; + custom_sources + .entry( + source + .plan + .custom_source() + .ok_or(RelayTransportError::InvalidPlan)?, + ) + .or_insert_with(|| path.clone()); + } + for path in custom_sources.into_values() { + let source = TlsTrustFileSource::new(path)?; + let (next, _) = supervise_workspace_operation( + workspace, + WorkspaceOperation::LoadCustom(source), + candidate_deadline, + &shutdown_cancellation, + ) + .await?; + workspace = next; + } + loop { + let (next, progress) = supervise_workspace_operation( + workspace, + WorkspaceOperation::Revalidate, + candidate_deadline, + &shutdown_cancellation, + ) + .await?; + workspace = next; + if progress == Some(TlsTrustRevalidationProgress::Complete) { + break; } } - for (path, source) in &loaded_sources { - source.revalidate(path)?; + if shutdown_cancellation.is_cancelled() || Instant::now() >= candidate_deadline { + return Err(RelayTransportError::Cancelled.into()); } + let candidate = workspace.finalize()?; let mut endpoints = Vec::with_capacity(sources.len()); for source in sources { - let anchors = loaded_sources - .get(&source.trust_path) - .ok_or(RelayTransportError::Coordinator)? - .anchors - .clone(); - let trust = TlsAccessFlowTrust::new(id, anchors).map_err(map_trust_contract_error)?; + let trust = candidate + .prepared(&source.plan) + .ok_or(RelayTransportError::Internal)?; let endpoint = TlsAccessFlowClientEndpoint::new( source.address.clone(), source.server_name.clone(), @@ -801,1214 +1125,447 @@ fn load_generation_blocking_with_hook( endpoints.push(endpoint); } Ok(RelayTransportGeneration { - id, + id: generation, tls_endpoints: endpoints.into_boxed_slice(), + trust_candidate: Some(candidate), }) } -fn map_trust_contract_error( - error: access_flow_tls::TlsAccessFlowContractError, -) -> RelayTransportError { - use access_flow_tls::TlsAccessFlowContractError; - match error { - TlsAccessFlowContractError::CertificateCountExceeded - | TlsAccessFlowContractError::CertificateSizeExceeded - | TlsAccessFlowContractError::CertificateAggregateExceeded - | TlsAccessFlowContractError::ResourceOverflow => RelayTransportError::ResourceLimit, - _ => RelayTransportError::InvalidMaterial, - } -} - -#[cfg(unix)] -struct LoadedTrustSource { - anchors: Vec>, - snapshot: StableMetadata, -} - -#[cfg(unix)] -impl LoadedTrustSource { - fn revalidate(&self, path: &Path) -> Result<(), RelayTransportError> { - let ((), current) = with_secure_trust_file(path, |_| Ok(()))?; - if current != self.snapshot { - return Err(RelayTransportError::UntrustedSource); +async fn supervise_workspace_operation( + workspace: TlsTrustLoadWorkspace, + operation: WorkspaceOperation, + candidate_deadline: Instant, + shutdown_cancellation: &CancellationToken, +) -> Result<(TlsTrustLoadWorkspace, Option), GenerationLoadFailure> { + if shutdown_cancellation.is_cancelled() || Instant::now() >= candidate_deadline { + return Err(RelayTransportError::Cancelled.into()); + } + let operation_deadline = (Instant::now() + TRUST_OPERATION_TIMEOUT).min(candidate_deadline); + let cancellation = OperationCancellation { + operation: CancellationToken::new(), + shutdown: shutdown_cancellation.clone(), + }; + let worker_cancellation = cancellation.clone(); + let mut worker = tokio::task::spawn_blocking(move || { + let mut workspace = workspace; + let result = match operation { + WorkspaceOperation::LoadSystem => { + workspace.load_system(&worker_cancellation).map(|()| None) + } + WorkspaceOperation::LoadCustom(source) => workspace + .load_custom(&source, &worker_cancellation) + .map(|()| None), + WorkspaceOperation::Revalidate => { + workspace.revalidate_next(&worker_cancellation).map(Some) + } + #[cfg(test)] + WorkspaceOperation::Block(barrier) => { + barrier.wait(); + Ok(None) + } + #[cfg(test)] + WorkspaceOperation::Mark(started) => { + started.store(true, Ordering::Release); + Ok(None) + } + #[cfg(test)] + WorkspaceOperation::WaitForCancellation { started, observed } => { + started.store(true, Ordering::Release); + while !worker_cancellation.is_cancelled() { + std::thread::yield_now(); + } + observed.store(true, Ordering::Release); + Err(TlsTrustLoadError::Cancelled) + } + }; + (workspace, result) + }); + let completion = tokio::select! { + result = tokio::time::timeout_at(operation_deadline, &mut worker) => Some(result), + () = shutdown_cancellation.cancelled() => None, + }; + match completion { + Some(Ok(Ok((workspace, result)))) => Ok((workspace, result?)), + Some(Ok(Err(_))) => Err(RelayTransportError::Internal.into()), + Some(Err(_)) | None => { + cancellation.operation.cancel(); + if tokio::time::timeout(TRUST_CANCELLATION_JOIN_GRACE, &mut worker) + .await + .is_ok() + { + return Err(RelayTransportError::Cancelled.into()); + } + let watcher = tokio::spawn(async move { + let _ = worker.await; + }); + Err(GenerationLoadFailure::TimedOut(watcher)) } - Ok(()) - } -} - -#[cfg(not(unix))] -struct LoadedTrustSource { - anchors: Vec>, -} - -#[cfg(not(unix))] -impl LoadedTrustSource { - fn revalidate(&self, _path: &Path) -> Result<(), RelayTransportError> { - Err(RelayTransportError::Unavailable) } } -#[cfg(test)] -fn load_trust_anchors(path: &Path) -> Result>, RelayTransportError> { - let loaded = load_trust_source(path)?; - loaded.revalidate(path)?; - Ok(loaded.anchors) -} - -#[cfg(all(test, unix))] -fn load_trust_anchors_with_hook( - path: &Path, - after_read: impl FnOnce(), -) -> Result>, RelayTransportError> { - let loaded = load_trust_source_with_hook(path, after_read)?; - loaded.revalidate(path)?; - Ok(loaded.anchors) +fn candidate_timeout(custom_count: usize) -> Result { + let custom_allowance = Duration::from_millis( + u64::try_from(custom_count) + .map_err(|_| RelayTransportError::SourceResourceLimit)? + .checked_mul(250) + .ok_or(RelayTransportError::SourceResourceLimit)?, + ); + Ok(TRUST_CANDIDATE_BASE_TIMEOUT + .checked_add(custom_allowance) + .unwrap_or(TRUST_CANDIDATE_MAX_TIMEOUT) + .min(TRUST_CANDIDATE_MAX_TIMEOUT)) } -fn load_trust_source(path: &Path) -> Result { - load_trust_source_with_hook(path, || {}) +#[derive(Clone)] +struct OperationCancellation { + operation: CancellationToken, + shutdown: CancellationToken, } -#[cfg(unix)] -fn load_trust_source_with_hook( - path: &Path, - after_read: impl FnOnce(), -) -> Result { - use std::io::Read as _; - - let (bytes, snapshot) = with_secure_trust_file(path, move |file| { - let opened_size = usize::try_from( - file.metadata() - .map_err(|_| RelayTransportError::Unavailable)? - .len(), - ) - .map_err(|_| RelayTransportError::ResourceLimit)?; - if opened_size > MAX_TRUST_SOURCE_BYTES { - return Err(RelayTransportError::ResourceLimit); - } - let mut bytes = vec![0_u8; MAX_TRUST_READ_BUFFER_BYTES]; - let mut length = 0_usize; - while length < bytes.len() { - match file.read(&mut bytes[length..]) { - Ok(0) => break, - Ok(read) => { - length = length - .checked_add(read) - .ok_or(RelayTransportError::ResourceLimit)?; - } - Err(error) if error.kind() == io::ErrorKind::Interrupted => {} - Err(_) => return Err(RelayTransportError::Unavailable), - } - } - bytes.truncate(length); - if bytes.len() > MAX_TRUST_SOURCE_BYTES { - return Err(RelayTransportError::ResourceLimit); - } - after_read(); - if bytes.len() != opened_size { - return Err(RelayTransportError::UntrustedSource); - } - Ok(bytes) - })?; - Ok(LoadedTrustSource { - anchors: parse_trust_pem(&bytes)?, - snapshot, - }) -} +impl AccessCancellation for OperationCancellation { + fn is_cancelled(&self) -> bool { + self.operation.is_cancelled() || self.shutdown.is_cancelled() + } -#[cfg(unix)] -fn with_secure_trust_file( - path: &Path, - operation: impl FnOnce(&mut std::fs::File) -> Result, -) -> Result<(T, StableMetadata), RelayTransportError> { - use std::ffi::CString; - use std::os::fd::{AsRawFd as _, OwnedFd}; - use std::os::unix::ffi::OsStrExt as _; - - if path.as_os_str().as_bytes().len() > crate::config::MAX_ACCESS_FLOW_TRUST_PATH_BYTES { - return Err(RelayTransportError::ResourceLimit); - } - let mut components = path.components(); - if components.next() != Some(Component::RootDir) { - return Err(RelayTransportError::UntrustedSource); - } - let names = components - .map(|component| match component { - Component::Normal(name) => { - CString::new(name.as_bytes()).map_err(|_| RelayTransportError::UntrustedSource) + fn cancelled(&self) -> BoxAccessFuture<'_, ()> { + Box::pin(async move { + tokio::select! { + () = self.operation.cancelled() => {} + () = self.shutdown.cancelled() => {} } - Component::Prefix(_) - | Component::RootDir - | Component::CurDir - | Component::ParentDir => Err(RelayTransportError::UntrustedSource), }) - .collect::, _>>()?; - if names.len() > crate::config::MAX_ACCESS_FLOW_TRUST_PATH_COMPONENTS { - return Err(RelayTransportError::ResourceLimit); - } - let (leaf, ancestors) = names - .split_last() - .ok_or(RelayTransportError::UntrustedSource)?; - let effective_uid = unsafe { libc::geteuid() }; - let root = CString::new("/").expect("filesystem root contains no NUL"); - let root_fd = unsafe { - libc::open( - root.as_ptr(), - libc::O_RDONLY | libc::O_DIRECTORY | libc::O_CLOEXEC | libc::O_NOFOLLOW, - ) - }; - let mut directory = owned_fd(root_fd)?; - verify_trusted_directory(directory.as_raw_fd(), effective_uid)?; - for ancestor in ancestors { - let descriptor = unsafe { - libc::openat( - directory.as_raw_fd(), - ancestor.as_ptr(), - libc::O_RDONLY | libc::O_DIRECTORY | libc::O_CLOEXEC | libc::O_NOFOLLOW, - ) - }; - let next = owned_fd(descriptor)?; - verify_trusted_directory(next.as_raw_fd(), effective_uid)?; - directory = next; - } - - let probed = probe_leaf(directory.as_raw_fd(), leaf)?; - verify_trust_file(&probed, effective_uid)?; - let descriptor = unsafe { - libc::openat( - directory.as_raw_fd(), - leaf.as_ptr(), - libc::O_RDONLY | libc::O_CLOEXEC | libc::O_NOFOLLOW | libc::O_NONBLOCK, - ) - }; - let descriptor: OwnedFd = owned_fd(descriptor)?; - let opened = descriptor_metadata(descriptor.as_raw_fd())?; - verify_trust_file(&opened, effective_uid)?; - if object_metadata(&opened) != object_metadata(&probed) { - return Err(RelayTransportError::UntrustedSource); - } - let mut file = std::fs::File::from(descriptor); - let opened_stable = stable_metadata(&file)?; - let result = operation(&mut file)?; - let post = descriptor_metadata(file.as_raw_fd())?; - verify_trust_file(&post, effective_uid)?; - let final_probe = probe_leaf(directory.as_raw_fd(), leaf)?; - if stable_metadata(&file)? != opened_stable - || object_metadata(&post) != object_metadata(&opened) - || object_metadata(&final_probe) != object_metadata(&opened) - { - return Err(RelayTransportError::UntrustedSource); - } - Ok((result, opened_stable)) -} - -#[cfg(not(unix))] -fn load_trust_source_with_hook( - _path: &Path, - _after_read: impl FnOnce(), -) -> Result { - Err(RelayTransportError::Unavailable) + } } -#[cfg(unix)] -fn owned_fd(descriptor: libc::c_int) -> Result { - use std::os::fd::FromRawFd as _; - if descriptor < 0 { - return Err(RelayTransportError::Unavailable); +#[cfg(test)] +mod tests { + use super::*; + use crate::config::{ + AccessFlowRelayConfig, AccessFlowRelayPresentation, AccessFlowRelayRoute, + AccessFlowRelayTransport, + }; + #[cfg(unix)] + use std::os::unix::fs::PermissionsExt; + + #[cfg(unix)] + const TEST_ROOT_PEM: &str = r#"-----BEGIN CERTIFICATE----- +MIIBeDCCASqgAwIBAgIUXLfYBhGLaC2YWMIvB0aPnyCXmZMwBQYDK2VwMCgxJjAk +BgNVBAMMHUFXIEFjY2VzcyBGbG93IFJUMDEgVGVzdCBSb290MB4XDTI2MDcyNjE5 +NDMwN1oXDTM2MDcyMzE5NDMwN1owKDEmMCQGA1UEAwwdQVcgQWNjZXNzIEZsb3cg +UlQwMSBUZXN0IFJvb3QwKjAFBgMrZXADIQBpAdFVn/HrfItwIx/XktXtNOZRrLFE +bRD4FW2ahSmyWaNmMGQwHwYDVR0jBBgwFoAUEXrimwcSAhT4Ae6XbVXVkbSfUUgw +EgYDVR0TAQH/BAgwBgEB/wIBADAOBgNVHQ8BAf8EBAMCAQYwHQYDVR0OBBYEFBF6 +4psHEgIU+AHul21V1ZG0n1FIMAUGAytlcANBAFpX6ZvogOz9Sd4QpaxfhacxJKGu +O6IBKa79z07RBsJ3vyWrw6+ytc5B2vUiZTDhocxsDzNCyZPnHB1Iq7iIFwQ= +-----END CERTIFICATE----- +"#; + + fn insecure_workspace() -> TlsTrustLoadWorkspace { + let generation = TlsTrustGeneration::new(1).unwrap(); + let plan = TlsClientTrustPlan::new(TlsClientTrustMode::Insecure, None).unwrap(); + let limits = TlsTrustLoadLimits::new(TEST_TLS_TRUST_BUDGET_BYTES, 8).unwrap(); + TlsTrustLoadWorkspace::new(generation, [plan], limits).unwrap() } - Ok(unsafe { std::os::fd::OwnedFd::from_raw_fd(descriptor) }) -} -#[cfg(unix)] -fn descriptor_metadata(descriptor: std::os::fd::RawFd) -> Result { - let mut metadata = std::mem::MaybeUninit::::uninit(); - if unsafe { libc::fstat(descriptor, metadata.as_mut_ptr()) } != 0 { - return Err(RelayTransportError::Unavailable); + #[test] + fn candidate_deadline_is_bounded_and_scales_with_custom_sources() { + assert_eq!(candidate_timeout(0).unwrap(), Duration::from_secs(60)); + assert_eq!(candidate_timeout(16).unwrap(), Duration::from_secs(64)); + assert_eq!(candidate_timeout(512).unwrap(), Duration::from_secs(180)); } - Ok(unsafe { metadata.assume_init() }) -} -#[cfg(unix)] -fn probe_leaf( - directory: std::os::fd::RawFd, - leaf: &std::ffi::CStr, -) -> Result { - let mut metadata = std::mem::MaybeUninit::::uninit(); - if unsafe { - libc::fstatat( - directory, - leaf.as_ptr(), - metadata.as_mut_ptr(), - libc::AT_SYMLINK_NOFOLLOW, + #[tokio::test] + async fn expired_candidate_does_not_start_a_blocking_operation() { + let started = Arc::new(AtomicBool::new(false)); + let shutdown = CancellationToken::new(); + let failure = match supervise_workspace_operation( + insecure_workspace(), + WorkspaceOperation::Mark(Arc::clone(&started)), + Instant::now(), + &shutdown, ) - } != 0 - { - return Err(RelayTransportError::Unavailable); + .await + { + Ok(_) => panic!("expired candidate deadline was accepted"), + Err(failure) => failure, + }; + let GenerationLoadFailure::Error(RelayTransportError::Cancelled) = failure else { + panic!("expired candidate returned the wrong failure"); + }; + assert!(!started.load(Ordering::Acquire)); } - Ok(unsafe { metadata.assume_init() }) -} - -#[cfg(unix)] -fn verify_trusted_directory( - descriptor: std::os::fd::RawFd, - effective_uid: libc::uid_t, -) -> Result<(), RelayTransportError> { - let metadata = descriptor_metadata(descriptor)?; - if metadata.st_mode & libc::S_IFMT != libc::S_IFDIR - || !trusted_owner(metadata.st_uid, effective_uid) - || metadata.st_mode & 0o022 != 0 - { - return Err(RelayTransportError::UntrustedSource); - } - Ok(()) -} - -#[cfg(unix)] -fn verify_trust_file( - metadata: &libc::stat, - effective_uid: libc::uid_t, -) -> Result<(), RelayTransportError> { - if metadata.st_mode & libc::S_IFMT != libc::S_IFREG - || !trusted_owner(metadata.st_uid, effective_uid) - || metadata.st_mode & 0o022 != 0 - || metadata.st_nlink != 1 - { - return Err(RelayTransportError::UntrustedSource); - } - let size = usize::try_from(metadata.st_size).map_err(|_| RelayTransportError::ResourceLimit)?; - if size == 0 { - return Err(RelayTransportError::InvalidMaterial); - } - if size > MAX_TRUST_SOURCE_BYTES { - return Err(RelayTransportError::ResourceLimit); - } - Ok(()) -} - -#[cfg(unix)] -const fn trusted_owner(owner: libc::uid_t, effective_uid: libc::uid_t) -> bool { - owner == 0 || owner == effective_uid -} - -#[cfg(unix)] -#[derive(Clone, Copy, Eq, PartialEq)] -struct ObjectMetadata { - device: libc::dev_t, - inode: libc::ino_t, - mode: libc::mode_t, - owner: libc::uid_t, - links: libc::nlink_t, - size: libc::off_t, -} -#[cfg(unix)] -fn object_metadata(metadata: &libc::stat) -> ObjectMetadata { - ObjectMetadata { - device: metadata.st_dev, - inode: metadata.st_ino, - mode: metadata.st_mode, - owner: metadata.st_uid, - links: metadata.st_nlink, - size: metadata.st_size, + #[tokio::test] + async fn timed_out_noninterruptible_operation_returns_an_owned_watcher() { + let barrier = Arc::new(std::sync::Barrier::new(2)); + let shutdown = CancellationToken::new(); + let failure = match supervise_workspace_operation( + insecure_workspace(), + WorkspaceOperation::Block(Arc::clone(&barrier)), + Instant::now() + Duration::from_millis(20), + &shutdown, + ) + .await + { + Ok(_) => panic!("noninterruptible operation did not time out"), + Err(failure) => failure, + }; + let GenerationLoadFailure::TimedOut(watcher) = failure else { + panic!("noninterruptible operation returned the wrong failure"); + }; + barrier.wait(); + watcher.await.unwrap(); } -} - -#[cfg(unix)] -#[derive(Clone, Copy, Eq, PartialEq)] -struct StableMetadata { - device: u64, - inode: u64, - mode: u32, - owner: u32, - links: u64, - size: u64, - modified_seconds: i64, - modified_nanoseconds: i64, - changed_seconds: i64, - changed_nanoseconds: i64, -} -#[cfg(unix)] -fn stable_metadata(file: &std::fs::File) -> Result { - use std::os::unix::fs::MetadataExt as _; - let metadata = file - .metadata() - .map_err(|_| RelayTransportError::Unavailable)?; - Ok(StableMetadata { - device: metadata.dev(), - inode: metadata.ino(), - mode: metadata.mode(), - owner: metadata.uid(), - links: metadata.nlink(), - size: metadata.size(), - modified_seconds: metadata.mtime(), - modified_nanoseconds: metadata.mtime_nsec(), - changed_seconds: metadata.ctime(), - changed_nanoseconds: metadata.ctime_nsec(), - }) -} - -fn parse_trust_pem(input: &[u8]) -> Result>, RelayTransportError> { - let mut remaining = input; - let mut anchors = Vec::new(); - while !remaining.trim_ascii().is_empty() { - let (block, rest) = take_exact_pem_block(remaining)?; - let (label, der) = - pem_rfc7468::decode_vec(block).map_err(|_| RelayTransportError::InvalidMaterial)?; - if label != "CERTIFICATE" { - return Err(RelayTransportError::InvalidMaterial); - } - if der.is_empty() { - return Err(RelayTransportError::InvalidMaterial); - } - if der.len() > MAX_DER_CERTIFICATE_BYTES { - return Err(RelayTransportError::ResourceLimit); - } - anchors.push(der); - if anchors.len() > MAX_TRUST_ANCHORS { - return Err(RelayTransportError::ResourceLimit); - } - remaining = rest; - } - if anchors.is_empty() { - return Err(RelayTransportError::InvalidMaterial); - } - Ok(anchors) -} - -fn take_exact_pem_block(input: &[u8]) -> Result<(&[u8], &[u8]), RelayTransportError> { - let trimmed = input.trim_ascii_start(); - if !trimmed.starts_with(CERTIFICATE_BEGIN) { - return Err(RelayTransportError::InvalidMaterial); - } - let end_offset = trimmed - .windows(CERTIFICATE_END.len()) - .position(|window| window == CERTIFICATE_END) - .ok_or(RelayTransportError::InvalidMaterial)?; - let block_end = end_offset - .checked_add(CERTIFICATE_END.len()) - .ok_or(RelayTransportError::ResourceLimit)?; - Ok((&trimmed[..block_end], &trimmed[block_end..])) -} - -#[cfg(all(test, unix))] -mod tests { - use super::*; - use access_flow_relay::{AccessFlowRoute, AccessFlowRouteName}; - use access_flow_tls::{TlsAccessFlowAddress, TlsAccessFlowServerName}; - use access_flow_unix::{NormalizedUnixSocketPath, UnixAccessFlowEndpoint}; - use access_identity::{IdentityPresentation, SensitiveBearer}; - use std::net::{Ipv4Addr, SocketAddrV4}; - use std::num::{NonZeroU16, NonZeroUsize}; - use std::os::unix::fs::{MetadataExt as _, PermissionsExt as _, symlink}; - use std::time::{Duration, Instant}; - - const TEST_ROOT_PEM: &str = "\ ------BEGIN CERTIFICATE-----\n\ -MIIBZDCCAQqgAwIBAgIUVeNPvjXF+esAR5JSWSDHPhHi8Z8wCgYIKoZIzj0EAwIw\n\ -FzEVMBMGA1UEAwwMYWNsLXByb3h5IENBMCAXDTc1MDEwMTAwMDAwMFoYDzQwOTYw\n\ -MTAxMDAwMDAwWjAXMRUwEwYDVQQDDAxhY2wtcHJveHkgQ0EwWTATBgcqhkjOPQIB\n\ -BggqhkjOPQMBBwNCAASSq7ztpOLW2yTnbT6B7tdXn2E37SCt7/WeOajZV3mUDpvH\n\ -lpLGD6uz16wTm75vtZ6aoLpTq7iE4pzTO9jOwftTozIwMDAdBgNVHQ4EFgQUKOgN\n\ -bJ2u/7pEok6/UT9IFfajHPYwDwYDVR0TAQH/BAUwAwEB/zAKBggqhkjOPQQDAgNI\n\ -ADBFAiB005pgAL7CLsHpHJFXEEgDG/fmG91oI1vRO/ZFSVufDQIhAOYNaysbiwJR\n\ -c+E0ChYtUrWyHuDFX+/4kDlyJh3LeI70\n\ ------END CERTIFICATE-----\n"; - - struct NeverCancelled; - - impl AccessCancellation for NeverCancelled { - fn is_cancelled(&self) -> bool { - false - } - - fn cancelled(&self) -> BoxAccessFuture<'_, ()> { - Box::pin(std::future::pending()) + #[tokio::test] + async fn close_cancellation_is_observed_and_cooperative_worker_is_joined() { + let started = Arc::new(AtomicBool::new(false)); + let observed = Arc::new(AtomicBool::new(false)); + let shutdown = CancellationToken::new(); + let operation = supervise_workspace_operation( + insecure_workspace(), + WorkspaceOperation::WaitForCancellation { + started: Arc::clone(&started), + observed: Arc::clone(&observed), + }, + Instant::now() + Duration::from_secs(1), + &shutdown, + ); + tokio::pin!(operation); + tokio::select! { + _ = &mut operation => panic!("cooperative worker ended before cancellation"), + () = async { + while !started.load(Ordering::Acquire) { + tokio::task::yield_now().await; + } + } => {} } - } - - fn trusted_tempdir() -> tempfile::TempDir { - let home = std::env::var_os("HOME").expect("test user home directory"); - let directory = tempfile::Builder::new() - .prefix(".relay-transport-test-") - .tempdir_in(home) - .expect("trusted temporary directory"); - std::fs::set_permissions(directory.path(), std::fs::Permissions::from_mode(0o700)) - .expect("private temporary directory"); - directory - } - - fn write_trust(directory: &Path, contents: &[u8]) -> PathBuf { - let path = directory.join("trust.pem"); - std::fs::write(&path, contents).expect("write trust"); - std::fs::set_permissions(&path, std::fs::Permissions::from_mode(0o644)) - .expect("set trust mode"); - path - } - - fn presentation() -> IdentityPresentation { - IdentityPresentation::Bearer( - SensitiveBearer::new(b"abcdefghijklmnopqrstuvwxyzABCDEF").expect("test bearer"), - ) + shutdown.cancel(); + let failure = operation.await.unwrap_err(); + assert!(matches!( + failure, + GenerationLoadFailure::Error(RelayTransportError::Cancelled) + )); + assert!(observed.load(Ordering::Acquire)); + } + + fn insecure_plan() -> AccessFlowRelayPlan { + let config = AccessFlowRelayConfig { + setup_timeout: "2s".into(), + drain_timeout: "2s".into(), + max_connections: 8, + copy_buffer_bytes_per_direction: 4096, + start_after_services: Vec::new(), + presentation: AccessFlowRelayPresentation::BearerEnvironment { + variable: "AW_IDENTITY_TOKEN".into(), + }, + routes: vec![AccessFlowRelayRoute { + name: "https".into(), + listen: "127.0.0.1:3129".into(), + allowed_destination_ports: vec![443], + transport: AccessFlowRelayTransport::TlsTcp { + address: "proxy.example.test:7443".into(), + server_name: "proxy.example.test".into(), + trust: TlsClientTrustMode::Insecure, + ca_certificate: None, + }, + }], + }; + config + .compile_with_presentation( + crate::config::AccessFlowRelayValidationMode::Agent, + access_identity::IdentityPresentation::Bearer( + access_identity::SensitiveBearer::new(b"abcdefghijklmnopqrstuvwxyzABCDEF") + .unwrap(), + ), + ) + .unwrap() + .plan } fn tls_plan( - trust_path: PathBuf, - address: &str, + modes: &[(TlsClientTrustMode, Option)], ) -> AccessFlowRelayPlan { - let endpoint = CompiledAccessFlowRelayEndpoint::TlsTcp { - tls_index: 0, - address: TlsAccessFlowAddress::parse(address).expect("test TLS address"), - server_name: TlsAccessFlowServerName::parse("localhost").expect("test server name"), - trust_path, - }; - AccessFlowRelayPlan::new( - vec![ - AccessFlowRoute::new( - AccessFlowRouteName::new("tls").expect("route name"), - SocketAddrV4::new(Ipv4Addr::LOCALHOST, 18080), - vec![NonZeroU16::new(80).expect("port")], - endpoint, - ) - .expect("TLS route"), - ], - presentation(), - Duration::from_secs(5), - NonZeroUsize::new(8).expect("connections"), - NonZeroUsize::new(4096).expect("buffer"), - ) - .expect("TLS plan") - } - - fn tls_sources(first: PathBuf, second: PathBuf) -> Vec { - [first, second] - .into_iter() + let routes = modes + .iter() .enumerate() - .map(|(index, trust_path)| TlsRouteSource { - address: TlsAccessFlowAddress::parse(&format!("127.0.0.1:{}", 7443 + index)) - .expect("test TLS address"), - server_name: TlsAccessFlowServerName::parse("localhost").expect("test server name"), - trust_path, + .map(|(index, (trust, ca_certificate))| AccessFlowRelayRoute { + name: format!("tls-{index}"), + listen: format!("127.0.0.1:{}", 32000 + index), + allowed_destination_ports: vec![443], + transport: AccessFlowRelayTransport::TlsTcp { + address: "127.0.0.1:7443".into(), + server_name: "proxy.example.test".into(), + trust: *trust, + ca_certificate: ca_certificate.clone(), + }, }) - .collect() - } - - fn unix_endpoint(path: &Path) -> CompiledAccessFlowRelayEndpoint { - CompiledAccessFlowRelayEndpoint::Unix(UnixAccessFlowEndpoint::new( - NormalizedUnixSocketPath::new(path.to_str().expect("UTF-8 path")) - .expect("normalized Unix path"), - )) - } - - async fn complete_reload(runtime: &RelayTransportRuntime) -> Result<(), RelayTransportError> { - match runtime.begin_reload()? { - Some(pending) => pending.complete().await, - None => Ok(()), - } - } - - #[test] - fn strict_pem_accepts_certificates_and_rejects_junk_or_other_labels() { - let anchors = parse_trust_pem(TEST_ROOT_PEM.as_bytes()).expect("valid root"); - assert_eq!(anchors.len(), 1); - let invalid_inputs = [ - b"junk\n".to_vec(), - b"-----BEGIN PRIVATE KEY-----\nAA==\n-----END PRIVATE KEY-----\n".to_vec(), - format!("{TEST_ROOT_PEM}junk").into_bytes(), - b"-----BEGIN CERTIFICATE-----\n!!!!\n-----END CERTIFICATE-----\n".to_vec(), - ]; - for invalid in invalid_inputs { - assert_eq!( - parse_trust_pem(&invalid), - Err(RelayTransportError::InvalidMaterial) - ); + .collect(); + AccessFlowRelayConfig { + setup_timeout: "2s".into(), + drain_timeout: "2s".into(), + max_connections: 8, + copy_buffer_bytes_per_direction: 4096, + start_after_services: Vec::new(), + presentation: AccessFlowRelayPresentation::BearerEnvironment { + variable: "AW_IDENTITY_TOKEN".into(), + }, + routes, } - } - - #[test] - fn strict_pem_enforces_certificate_count_and_size() { - let oversized = pem_rfc7468::encode_string( - "CERTIFICATE", - pem_rfc7468::LineEnding::LF, - &vec![7_u8; MAX_DER_CERTIFICATE_BYTES + 1], + .compile_with_presentation( + crate::config::AccessFlowRelayValidationMode::Agent, + access_identity::IdentityPresentation::Bearer( + access_identity::SensitiveBearer::new(b"abcdefghijklmnopqrstuvwxyzABCDEF").unwrap(), + ), ) - .expect("encode oversized certificate"); - assert_eq!( - parse_trust_pem(oversized.as_bytes()), - Err(RelayTransportError::ResourceLimit) - ); - - let mut over_count = String::new(); - for index in 0..=MAX_TRUST_ANCHORS { - over_count.push_str( - &pem_rfc7468::encode_string( - "CERTIFICATE", - pem_rfc7468::LineEnding::LF, - &[u8::try_from(index).expect("bounded index")], - ) - .expect("encode certificate"), - ); - } - assert_eq!( - parse_trust_pem(over_count.as_bytes()), - Err(RelayTransportError::ResourceLimit) - ); - } - - #[test] - fn secure_loader_accepts_stable_single_link_public_trust() { - let directory = trusted_tempdir(); - let path = write_trust(directory.path(), TEST_ROOT_PEM.as_bytes()); - let anchors = load_trust_anchors(&path).expect("stable trust source"); - assert_eq!(anchors.len(), 1); + .unwrap() + .plan } #[test] - fn secure_loader_rejects_leaf_symlink_hardlink_and_writable_mode() { - let directory = trusted_tempdir(); - let path = write_trust(directory.path(), TEST_ROOT_PEM.as_bytes()); - let link = directory.path().join("link.pem"); - symlink(&path, &link).expect("trust symlink"); - assert!(matches!( - load_trust_anchors(&link), - Err(RelayTransportError::Unavailable | RelayTransportError::UntrustedSource) - )); - - let hardlink = directory.path().join("hardlink.pem"); - std::fs::hard_link(&path, &hardlink).expect("trust hard link"); - assert_eq!( - load_trust_anchors(&path), - Err(RelayTransportError::UntrustedSource) - ); - std::fs::remove_file(&hardlink).expect("remove hard link"); - - std::fs::set_permissions(&path, std::fs::Permissions::from_mode(0o666)) - .expect("unsafe trust mode"); - assert_eq!( - load_trust_anchors(&path), - Err(RelayTransportError::UntrustedSource) - ); - } - - #[test] - fn secure_loader_rejects_symlinked_or_writable_ancestor() { - let directory = trusted_tempdir(); - let real = directory.path().join("real"); - std::fs::create_dir(&real).expect("real directory"); - std::fs::set_permissions(&real, std::fs::Permissions::from_mode(0o700)) - .expect("private directory"); - let trust = write_trust(&real, TEST_ROOT_PEM.as_bytes()); - let link = directory.path().join("linked"); - symlink(&real, &link).expect("ancestor symlink"); - assert!(matches!( - load_trust_anchors(&link.join("trust.pem")), - Err(RelayTransportError::Unavailable | RelayTransportError::UntrustedSource) - )); - - std::fs::set_permissions(&real, std::fs::Permissions::from_mode(0o777)) - .expect("unsafe ancestor mode"); - assert_eq!( - load_trust_anchors(&trust), - Err(RelayTransportError::UntrustedSource) - ); - } - - #[test] - fn secure_loader_detects_same_descriptor_mutation() { - let directory = trusted_tempdir(); - let path = write_trust(directory.path(), TEST_ROOT_PEM.as_bytes()); - let mutation_path = path.clone(); + fn system_only_one_plan_has_exact_design_peak() { + let plan = tls_plan(&[(TlsClientTrustMode::System, None)]); + let reserve = RelayTransportRuntime::resource_reserve(&plan).unwrap(); + assert_eq!(reserve.memory_bytes, 58_855_424); + let accepted = + RelayTransportRuntime::prepare(&plan, Arc::new(AtomicBool::new(false))).unwrap(); + accepted + .configure_trust_budget(RelayTransportTrustBudget { + descriptors: reserve.descriptors, + memory_bytes: reserve.memory_bytes, + }) + .unwrap(); + let rejected = + RelayTransportRuntime::prepare(&plan, Arc::new(AtomicBool::new(false))).unwrap(); assert_eq!( - load_trust_anchors_with_hook(&path, move || { - let mut bytes = TEST_ROOT_PEM.as_bytes().to_vec(); - bytes[40] ^= 1; - std::fs::write(mutation_path, bytes).expect("mutate trust"); + rejected.configure_trust_budget(RelayTransportTrustBudget { + descriptors: reserve.descriptors, + memory_bytes: reserve.memory_bytes - 1, }), - Err(RelayTransportError::UntrustedSource) - ); - } - - #[test] - fn secure_loader_enforces_raw_source_bound_and_regular_type() { - let directory = trusted_tempdir(); - let oversized = write_trust(directory.path(), &vec![b' '; MAX_TRUST_SOURCE_BYTES + 1]); - assert_eq!( - load_trust_anchors(&oversized), - Err(RelayTransportError::ResourceLimit) - ); - assert_eq!( - load_trust_anchors(directory.path()), - Err(RelayTransportError::UntrustedSource) - ); - } - - #[test] - fn secure_loader_enforces_path_memory_and_component_bounds() { - let oversized = PathBuf::from(format!( - "/{}", - "a".repeat(crate::config::MAX_ACCESS_FLOW_TRUST_PATH_BYTES) - )); - assert_eq!( - load_trust_anchors(&oversized), - Err(RelayTransportError::ResourceLimit) - ); - - let too_deep = PathBuf::from(format!( - "/{}", - std::iter::repeat_n( - "a", - crate::config::MAX_ACCESS_FLOW_TRUST_PATH_COMPONENTS + 1, - ) - .collect::>() - .join("/") - )); - assert_eq!( - load_trust_anchors(&too_deep), - Err(RelayTransportError::ResourceLimit) + Err(RelayTransportError::SourceResourceLimit) ); } - #[test] - fn generation_revalidates_every_source_after_all_sources_are_read() { - let directory = trusted_tempdir(); - let first = write_trust(directory.path(), TEST_ROOT_PEM.as_bytes()); - let second = directory.path().join("second.pem"); - std::fs::write(&second, TEST_ROOT_PEM).expect("write second trust"); - std::fs::set_permissions(&second, std::fs::Permissions::from_mode(0o644)) - .expect("set second trust mode"); - let mutation_path = first.clone(); - let sources = tls_sources(first, second); - - let result = load_generation_blocking_with_hook(&sources, 2, move |index| { - if index == 1 { - let mut still_valid = TEST_ROOT_PEM.as_bytes().to_vec(); - still_valid.push(b'\n'); - std::fs::write(&mutation_path, still_valid) - .expect("mutate first source after second read"); - } - }); - - assert!(matches!(result, Err(RelayTransportError::UntrustedSource))); - } - + #[cfg(unix)] #[tokio::test] - async fn failed_reload_is_atomic_unhealthy_and_later_reload_recovers() { - let directory = trusted_tempdir(); - let trust = write_trust(directory.path(), TEST_ROOT_PEM.as_bytes()); - let plan = tls_plan(trust.clone(), "127.0.0.1:7443"); - let readiness = Arc::new(AtomicBool::new(false)); - let runtime = RelayTransportRuntime::activate(&plan, Arc::clone(&readiness)) - .await - .expect("initial activation"); - assert!(runtime.healthy()); - assert!(readiness.load(Ordering::Acquire)); - let first = runtime - .inner - .healthy_generation() - .expect("initial generation") - .id; - - std::fs::write(&trust, b"invalid").expect("invalidate trust"); - assert_eq!( - complete_reload(&runtime).await, - Err(RelayTransportError::InvalidMaterial) - ); - assert!(!runtime.healthy()); - assert!(!readiness.load(Ordering::Acquire)); - let retained = runtime - .inner - .retained_generation() - .expect("retained generation"); - assert_eq!(retained.id, first); - - std::fs::write(&trust, TEST_ROOT_PEM).expect("restore trust"); - complete_reload(&runtime).await.expect("recover trust"); - assert!(runtime.healthy()); - assert!(readiness.load(Ordering::Acquire)); - assert_ne!( - runtime - .inner - .healthy_generation() - .expect("recovered generation") - .id, - first - ); + async fn system_and_composite_modes_activate_reload_and_mix_atomically() { + let dir = tempfile::Builder::new() + .prefix(".relay-shared-trust-modes-") + .tempdir_in(std::env::var_os("HOME").unwrap()) + .unwrap(); + std::fs::set_permissions(dir.path(), std::fs::Permissions::from_mode(0o700)).unwrap(); + let trust_path = dir.path().join("roots.pem"); + std::fs::write(&trust_path, TEST_ROOT_PEM).unwrap(); + std::fs::set_permissions(&trust_path, std::fs::Permissions::from_mode(0o644)).unwrap(); + let trust_path = trust_path.display().to_string(); + + for modes in [ + vec![(TlsClientTrustMode::System, None)], + vec![( + TlsClientTrustMode::SystemPlusCustom, + Some(trust_path.clone()), + )], + vec![ + (TlsClientTrustMode::System, None), + (TlsClientTrustMode::Custom, Some(trust_path.clone())), + ( + TlsClientTrustMode::SystemPlusCustom, + Some(trust_path.clone()), + ), + (TlsClientTrustMode::Insecure, None), + ], + ] { + let readiness = Arc::new(AtomicBool::new(false)); + let runtime = + RelayTransportRuntime::activate(&tls_plan(&modes), Arc::clone(&readiness)) + .await + .unwrap(); + let initial = runtime.inner.healthy_generation().unwrap(); + reload(&runtime).await.unwrap(); + let reloaded = runtime.inner.healthy_generation().unwrap(); + assert_ne!(initial.id, reloaded.id); + assert!(readiness.load(Ordering::Acquire)); + assert_eq!(reloaded.tls_endpoints.len(), modes.len()); + } } - #[tokio::test] - async fn unhealthy_state_rejects_unix_and_tls_without_fallback() { - let directory = trusted_tempdir(); - let trust = write_trust(directory.path(), TEST_ROOT_PEM.as_bytes()); - let plan = tls_plan(trust.clone(), "127.0.0.1:7443"); - let runtime = RelayTransportRuntime::activate(&plan, Arc::new(AtomicBool::new(false))) + async fn reload(runtime: &RelayTransportRuntime) -> Result<(), RelayTransportError> { + runtime + .begin_reload()? + .expect("TLS trust reload") + .complete() .await - .expect("initial activation"); - std::fs::write(&trust, b"invalid").expect("invalidate trust"); - assert!(complete_reload(&runtime).await.is_err()); - let connector = runtime.connector(); - let cancellation = NeverCancelled; - let context = - AccessFlowConnectContext::new(Instant::now() + Duration::from_secs(1), &cancellation); - assert!(matches!( - connector - .connect( - &unix_endpoint(&directory.path().join("missing.sock")), - context - ) - .await, - Err(AccessFlowChannelFailure::Unavailable) - )); - let context = - AccessFlowConnectContext::new(Instant::now() + Duration::from_secs(1), &cancellation); - assert!(matches!( - connector - .connect(plan.routes()[0].endpoint(), context) - .await, - Err(AccessFlowChannelFailure::Unavailable) - )); } #[tokio::test] - async fn synchronous_reload_gate_is_fail_closed_and_clone_can_recover() { - let directory = trusted_tempdir(); - let trust = write_trust(directory.path(), TEST_ROOT_PEM.as_bytes()); - let plan = tls_plan(trust, "127.0.0.1:7443"); - let expected_memory = - project_candidate_loader_peak(&collect_tls_sources(&plan).unwrap()).unwrap(); - assert_eq!( - RelayTransportRuntime::resource_reserve(&plan), - Ok(RelayTransportResourceReserve { - descriptors: TRUST_LOAD_DESCRIPTOR_ENVELOPE, - memory_bytes: expected_memory, - }) - ); + async fn insecure_only_reload_stays_ready_and_publishes_a_new_generation() { let readiness = Arc::new(AtomicBool::new(false)); - let runtime = RelayTransportRuntime::activate(&plan, Arc::clone(&readiness)) + let runtime = RelayTransportRuntime::activate(&insecure_plan(), Arc::clone(&readiness)) .await - .expect("initial activation"); - let initial = runtime - .inner - .healthy_generation() - .expect("initial generation") - .id; - - let pending = runtime - .begin_reload() - .expect("begin reload") - .expect("TLS reload worker"); - assert!(!readiness.load(Ordering::Acquire)); - assert!(!runtime.healthy()); - assert_eq!( - runtime - .inner - .retained_generation() - .expect("retained generation") - .id, - initial - ); - - let owned = runtime.clone(); - tokio::spawn(async move { pending.complete().await }) - .await - .expect("reload task") - .expect("reload recovery"); - drop(owned); + .unwrap(); + let initial = runtime.inner.healthy_generation().unwrap(); + let mut pending = runtime.begin_reload().unwrap().unwrap(); assert!(readiness.load(Ordering::Acquire)); - assert!(runtime.healthy()); - } - - #[tokio::test] - async fn prepared_transport_projects_before_any_trust_source_read() { - let directory = trusted_tempdir(); - let missing = directory.path().join("missing.pem"); - let plan = tls_plan(missing, "127.0.0.1:7443"); - let readiness = Arc::new(AtomicBool::new(true)); - let runtime = RelayTransportRuntime::prepare(&plan, Arc::clone(&readiness)) - .expect("bounded transport preparation does not read trust"); - assert!(!readiness.load(Ordering::Acquire)); - assert!( - runtime - .connector() - .resource_projection(plan.routes()[0].endpoint()) - .is_ok() - ); - assert_eq!( - runtime.activate_prepared().await, - Err(RelayTransportError::Unavailable) - ); - assert!(!readiness.load(Ordering::Acquire)); + pending.wait().await.unwrap(); + assert!(readiness.load(Ordering::Acquire)); + let current = runtime.inner.healthy_generation().unwrap(); + assert_ne!(initial.id, current.id); } #[tokio::test] - async fn pending_reload_owns_started_worker_until_terminal_join() { - let directory = trusted_tempdir(); - let trust = write_trust(directory.path(), TEST_ROOT_PEM.as_bytes()); - let plan = tls_plan(trust, "127.0.0.1:7443"); + async fn eight_retained_generations_reject_then_recover_after_retirement() { let readiness = Arc::new(AtomicBool::new(false)); - let runtime = RelayTransportRuntime::activate(&plan, Arc::clone(&readiness)) + let runtime = RelayTransportRuntime::activate(&insecure_plan(), Arc::clone(&readiness)) .await - .expect("initial activation"); - let (started_tx, started_rx) = std::sync::mpsc::sync_channel(0); - let (release_tx, release_rx) = std::sync::mpsc::sync_channel(0); - let mut pending = runtime - .begin_reload_with_hook(move || { - started_tx.send(()).expect("report worker start"); - release_rx.recv().expect("release blocking worker"); - }) - .expect("begin reload") - .expect("TLS reload worker"); - - started_rx - .recv_timeout(Duration::from_secs(1)) - .expect("blocking worker actually started"); - assert!(!readiness.load(Ordering::Acquire)); - assert_eq!( - runtime.begin_reload().unwrap_err(), - RelayTransportError::ReloadInProgress - ); - assert!( - tokio::time::timeout(Duration::from_millis(20), pending.wait()) - .await - .is_err(), - "select-style wait remains pending while the worker is paused" - ); - - runtime.close(); - release_tx.send(()).expect("release blocking worker"); + .unwrap(); + let mut leases = vec![runtime.inner.healthy_generation().unwrap()]; + for _ in 1..MAX_PUBLISHED_TRUST_GENERATIONS { + reload(&runtime).await.unwrap(); + leases.push(runtime.inner.healthy_generation().unwrap()); + } assert_eq!( - pending.complete().await, - Err(RelayTransportError::ShuttingDown) + reload(&runtime).await, + Err(RelayTransportError::TrustGenerationLimit) ); - assert!(!readiness.load(Ordering::Acquire)); - assert!(matches!( - &*runtime.inner.state.borrow(), - RelayTransportState::Closed - )); + assert!(readiness.load(Ordering::Acquire)); + leases.remove(0); + reload(&runtime).await.unwrap(); + assert!(readiness.load(Ordering::Acquire)); } #[tokio::test] - async fn readiness_publication_orders_each_state_transition() { - let directory = trusted_tempdir(); - let trust = write_trust(directory.path(), TEST_ROOT_PEM.as_bytes()); - let plan = tls_plan(trust, "127.0.0.1:7443"); + async fn startup_blocked_operation_keeps_not_ready_and_excludes_activation() { let readiness = Arc::new(AtomicBool::new(false)); - let runtime = RelayTransportRuntime::activate(&plan, Arc::clone(&readiness)) - .await - .expect("initial activation"); - let generation = runtime - .inner - .healthy_generation() - .expect("initial generation"); - - runtime.inner.preserve_generation_as_unhealthy_with(|| { - assert!(!readiness.load(Ordering::Acquire)); - assert!(matches!( - &*runtime.inner.state.borrow(), - RelayTransportState::Healthy(_) - )); - }); - assert!(matches!( - &*runtime.inner.state.borrow(), - RelayTransportState::Unhealthy(_) - )); - + let plan = insecure_plan(); + let runtime = RelayTransportRuntime::prepare(&plan, Arc::clone(&readiness)).unwrap(); runtime - .inner - .publish_generation_as_healthy_with(generation, || { - assert!(matches!( - &*runtime.inner.state.borrow(), - RelayTransportState::Healthy(_) - )); - assert!(!readiness.load(Ordering::Acquire)); - }); - assert!(readiness.load(Ordering::Acquire)); - - runtime.inner.publish_closed_with(|| { - assert!(!readiness.load(Ordering::Acquire)); - assert!(matches!( - &*runtime.inner.state.borrow(), - RelayTransportState::Healthy(_) - )); - }); - assert!(matches!( - &*runtime.inner.state.borrow(), - RelayTransportState::Closed - )); - } - - #[tokio::test] - async fn failed_reload_cancels_tls_connect_already_in_progress() { - let directory = trusted_tempdir(); - let trust = write_trust(directory.path(), TEST_ROOT_PEM.as_bytes()); - let listener = tokio::net::TcpListener::bind((Ipv4Addr::LOCALHOST, 0)) - .await - .expect("TCP listener"); - let address = listener.local_addr().expect("listener address"); - let plan = tls_plan(trust.clone(), &address.to_string()); - let runtime = RelayTransportRuntime::activate(&plan, Arc::new(AtomicBool::new(false))) - .await - .expect("initial activation"); - let connector = runtime.connector(); - let cancellation = NeverCancelled; - let context = - AccessFlowConnectContext::new(Instant::now() + Duration::from_secs(10), &cancellation); - let connect = connector.connect(plan.routes()[0].endpoint(), context); - tokio::pin!(connect); - let accept = listener.accept(); - tokio::pin!(accept); - let accepted_stream = tokio::select! { - accepted = &mut accept => { - accepted.expect("accepted TLS TCP").0 - } - result = &mut connect => panic!("TLS setup finished before reload: {result:?}"), - }; - - std::fs::write(&trust, b"invalid").expect("invalidate trust"); - let pending = runtime - .begin_reload() - .expect("begin reload") - .expect("TLS reload worker"); - assert!(matches!( - tokio::time::timeout(Duration::from_secs(1), &mut connect) - .await - .expect("generation cancellation"), - Err(AccessFlowChannelFailure::Cancelled) - )); - assert_eq!( - pending.complete().await, - Err(RelayTransportError::InvalidMaterial) - ); - drop(accepted_stream); - } - - #[tokio::test] - async fn tls_projection_and_transport_reserve_have_distinct_ownership() { - let directory = trusted_tempdir(); - let trust = write_trust(directory.path(), TEST_ROOT_PEM.as_bytes()); - let plan = tls_plan(trust, "127.0.0.1:7443"); - let runtime = RelayTransportRuntime::activate(&plan, Arc::new(AtomicBool::new(false))) - .await - .expect("initial activation"); - let connector = runtime.connector(); - let endpoint = plan.routes()[0].endpoint(); - let projected = connector - .resource_projection(endpoint) - .expect("product projection"); - let generation_bytes = MAX_TRUST_ANCHOR_BYTES as u64 + GENERATION_CONTROL_BYTES; - let CompiledAccessFlowRelayEndpoint::TlsTcp { - tls_index, - address, - server_name, - trust_path, - } = endpoint - else { - panic!("TLS endpoint expected"); - }; - let endpoint_metadata = tls_endpoint_metadata_bytes(address, server_name).unwrap(); - let trust_path_bytes = trust_path.as_os_str().as_encoded_bytes().len() as u64; - let trust_der_bytes = parse_trust_pem(TEST_ROOT_PEM.as_bytes()) - .unwrap() - .into_iter() - .map(|anchor| anchor.len() as u64) - .sum::(); - let generation = runtime - .inner - .retained_generation() - .expect("retained generation"); - let base = runtime - .inner - .tls - .resource_projection(&generation.tls_endpoints[*tls_index]) - .expect("base projection"); - assert_eq!( - projected.retained_endpoint_bytes, - base.retained_endpoint_bytes - + endpoint_metadata - + trust_path_bytes - + (MAX_TRUST_ANCHOR_BYTES as u64 - trust_der_bytes) - ); - assert_eq!(projected.active_bytes, base.active_bytes + generation_bytes); - let candidate_bytes = - project_candidate_loader_peak(&collect_tls_sources(&plan).unwrap()).unwrap(); - assert_eq!( - RelayTransportRuntime::resource_reserve(&plan), - Ok(RelayTransportResourceReserve { - descriptors: TRUST_LOAD_DESCRIPTOR_ENVELOPE, - memory_bytes: candidate_bytes, - }) - ); - } - - #[tokio::test] - async fn candidate_loader_peak_is_one_plan_reserve_across_shared_routes() { - let directory = trusted_tempdir(); - let trust = write_trust(directory.path(), TEST_ROOT_PEM.as_bytes()); - let routes = (0_u16..2) - .map(|index| { - let endpoint = CompiledAccessFlowRelayEndpoint::TlsTcp { - tls_index: usize::from(index), - address: TlsAccessFlowAddress::parse(&format!("127.0.0.1:{}", 7443 + index)) - .expect("test TLS address"), - server_name: TlsAccessFlowServerName::parse("localhost") - .expect("test server name"), - trust_path: trust.clone(), - }; - AccessFlowRoute::new( - AccessFlowRouteName::new(format!("tls-{index}")).expect("route name"), - SocketAddrV4::new(Ipv4Addr::LOCALHOST, 18080 + index), - vec![NonZeroU16::new(80).expect("port")], - endpoint, - ) - .expect("TLS route") - }) - .collect(); - let plan = AccessFlowRelayPlan::new( - routes, - presentation(), - Duration::from_secs(5), - NonZeroUsize::new(8).expect("connections"), - NonZeroUsize::new(4096).expect("buffer"), - ) - .expect("TLS plan"); - let runtime = RelayTransportRuntime::activate(&plan, Arc::new(AtomicBool::new(false))) - .await - .expect("initial activation"); - let connector = runtime.connector(); - let generation = runtime - .inner - .retained_generation() - .expect("retained generation"); - let generation_bytes = MAX_TRUST_ANCHOR_BYTES as u64 + GENERATION_CONTROL_BYTES; - - for (index, route) in plan.routes().iter().enumerate() { - let projected = connector - .resource_projection(route.endpoint()) - .expect("product projection"); - let base = runtime - .inner - .tls - .resource_projection(&generation.tls_endpoints[index]) - .expect("base projection"); - assert!(projected.retained_endpoint_bytes > base.retained_endpoint_bytes); - assert_eq!(projected.active_bytes, base.active_bytes + generation_bytes); - } - let candidate_bytes = - project_candidate_loader_peak(&collect_tls_sources(&plan).unwrap()).unwrap(); - assert_eq!( - RelayTransportRuntime::resource_reserve(&plan), - Ok(RelayTransportResourceReserve { - descriptors: TRUST_LOAD_DESCRIPTOR_ENVELOPE, - memory_bytes: candidate_bytes, + .configure_trust_budget(RelayTransportTrustBudget { + descriptors: TEST_TLS_TRUST_DESCRIPTOR_BUDGET, + memory_bytes: TEST_TLS_TRUST_BUDGET_BYTES, }) - ); - } - - #[test] - fn candidate_loader_peak_is_plan_global_and_deduplicates_trust_paths() { - let shared = PathBuf::from("/tmp/shared.pem"); - let distinct = PathBuf::from("/tmp/other.pem"); - let generation_bytes = MAX_TRUST_ANCHOR_BYTES as u64 + GENERATION_CONTROL_BYTES; - - let one = tls_sources(shared.clone(), shared.clone()) - .into_iter() - .take(1) - .collect::>(); - let one_peak = project_candidate_loader_peak(&one).unwrap(); - assert!(one_peak > 2 * MAX_TRUST_ANCHOR_BYTES as u64 + generation_bytes); - - let shared_routes = tls_sources(shared.clone(), shared); - let shared_peak = project_candidate_loader_peak(&shared_routes).unwrap(); - let second_metadata = - tls_endpoint_metadata_bytes(&shared_routes[1].address, &shared_routes[1].server_name) - .unwrap(); - let second_path = shared_routes[1] - .trust_path - .as_os_str() - .as_encoded_bytes() - .len() as u64; - assert_eq!( - shared_peak - one_peak, - 3 * second_metadata - + 2 * second_path - + generation_bytes - + MAX_TRUST_ANCHOR_BYTES as u64 - + TLS_ROUTE_CONTROL_BYTES - ); - - let distinct_routes = tls_sources(PathBuf::from("/tmp/first.pem"), distinct); - let distinct_peak = project_candidate_loader_peak(&distinct_routes).unwrap(); - let same_path_routes = tls_sources( - PathBuf::from("/tmp/first.pem"), - PathBuf::from("/tmp/first.pem"), - ); - let same_path_peak = project_candidate_loader_peak(&same_path_routes).unwrap(); - assert_eq!( - distinct_peak - same_path_peak, - MAX_TRUST_ANCHOR_BYTES as u64 - + "/tmp/other.pem".len() as u64 - + TLS_UNIQUE_SOURCE_CONTROL_BYTES - ); - } - - #[tokio::test] - async fn close_is_terminal_and_empty_tls_set_reload_is_noop() { - let directory = trusted_tempdir(); - let endpoint = unix_endpoint(&directory.path().join("relay.sock")); - let plan = AccessFlowRelayPlan::new( - vec![ - AccessFlowRoute::new( - AccessFlowRouteName::new("unix").expect("route name"), - SocketAddrV4::new(Ipv4Addr::LOCALHOST, 18081), - vec![NonZeroU16::new(80).expect("port")], - endpoint, - ) - .expect("Unix route"), - ], - presentation(), - Duration::from_secs(5), - NonZeroUsize::new(8).expect("connections"), - NonZeroUsize::new(4096).expect("buffer"), - ) - .expect("Unix plan"); + .unwrap(); + let (release, wait_for_release) = tokio::sync::oneshot::channel::<()>(); + runtime.install_test_blocked_operation(tokio::spawn(async move { + let _ = wait_for_release.await; + })); assert_eq!( - RelayTransportRuntime::resource_reserve(&plan), - Ok(RelayTransportResourceReserve { - descriptors: 0, - memory_bytes: 0, - }) + runtime.activate_prepared().await, + Err(RelayTransportError::ReloadInProgress) ); - let readiness = Arc::new(AtomicBool::new(false)); - let runtime = RelayTransportRuntime::activate(&plan, Arc::clone(&readiness)) - .await - .expect("Unix-only activation"); - assert!(readiness.load(Ordering::Acquire)); - assert!(runtime.begin_reload().expect("Unix-only reload").is_none()); - assert!(readiness.load(Ordering::Acquire)); - assert!(runtime.healthy()); - runtime.close(); - assert!(!runtime.healthy()); assert!(!readiness.load(Ordering::Acquire)); assert_eq!( - runtime.begin_reload().map(|_| ()), - Err(RelayTransportError::ShuttingDown) - ); - } - - #[test] - fn mutation_fixture_really_changes_descriptor_metadata() { - let directory = trusted_tempdir(); - let path = write_trust(directory.path(), TEST_ROOT_PEM.as_bytes()); - let before = std::fs::metadata(&path).expect("before"); - std::fs::write(&path, b"changed").expect("change"); - let after = std::fs::metadata(&path).expect("after"); - assert_eq!(before.dev(), after.dev()); - assert_eq!(before.ino(), after.ino()); - assert!( - before.len() != after.len() - || before.mtime() != after.mtime() - || before.mtime_nsec() != after.mtime_nsec() - || before.ctime() != after.ctime() - || before.ctime_nsec() != after.ctime_nsec() + runtime.activate_prepared().await, + Err(RelayTransportError::ReloadInProgress) ); + release.send(()).unwrap(); + tokio::time::timeout(Duration::from_secs(1), async { + while runtime.reload_blocked() { + tokio::task::yield_now().await; + } + }) + .await + .unwrap(); + runtime.activate_prepared().await.unwrap(); + assert!(readiness.load(Ordering::Acquire)); } } diff --git a/src/agent_control.rs b/src/agent_control.rs index 4b0577f..1f1a6f9 100644 --- a/src/agent_control.rs +++ b/src/agent_control.rs @@ -239,12 +239,16 @@ pub(crate) struct AccessFlowRelayStatus { pub(crate) ready: bool, pub(crate) active_flows: usize, pub(crate) routes: Vec, + #[serde(skip_serializing_if = "Option::is_none")] + pub(crate) trust_failure: Option, } #[derive(Debug, Clone, Deserialize, Serialize, PartialEq, Eq)] pub(crate) struct AccessFlowRelayRouteStatus { pub(crate) name: String, pub(crate) accepting: bool, + #[serde(skip_serializing_if = "Option::is_none")] + pub(crate) trust_mode: Option, } #[derive(Debug, Clone, Copy, Deserialize, Serialize, PartialEq, Eq)] diff --git a/src/config.rs b/src/config.rs index a94cee6..53135a2 100644 --- a/src/config.rs +++ b/src/config.rs @@ -15,16 +15,15 @@ mod steps; mod target; mod validation; +pub use access_tls_trust::TlsClientTrustMode; pub(crate) use agent::{ ACCESS_FLOW_RELAY_NODE, AccessFlowRelayValidationMode, CompiledAccessFlowRelayConfig, - CompiledAccessFlowRelayEndpoint, MAX_ACCESS_FLOW_TRUST_PATH_BYTES, - MAX_ACCESS_FLOW_TRUST_PATH_COMPONENTS, + CompiledAccessFlowRelayEndpoint, }; pub use agent::{ AccessFlowRelayConfig, AccessFlowRelayPresentation, AccessFlowRelayRoute, - AccessFlowRelayTransport, AccessFlowRelayTrust, ContainerAgentConfig, - ContainerAgentConfigInput, ControlSocketConfig, EnvValue, HealthCheck, RestartPolicy, - ServiceConfig, SshBridgeConfig, SshBridgeConfigInput, + AccessFlowRelayTransport, ContainerAgentConfig, ContainerAgentConfigInput, ControlSocketConfig, + EnvValue, HealthCheck, RestartPolicy, ServiceConfig, SshBridgeConfig, SshBridgeConfigInput, }; pub use http::{HttpAuthConfig, HttpAuthType, HttpConfig}; pub(crate) use launch::validate_launch_var_string_value; diff --git a/src/config/agent.rs b/src/config/agent.rs index f8870f9..3ba4c28 100644 --- a/src/config/agent.rs +++ b/src/config/agent.rs @@ -14,10 +14,11 @@ use access_flow_relay::{ use access_flow_tls::{TlsAccessFlowAddress, TlsAccessFlowServerName}; use access_flow_unix::{NormalizedUnixSocketPath, UnixAccessFlowEndpoint, UnixExecutionTarget}; use access_identity::{IdentityPresentation, SensitiveBearer}; +use access_tls_trust::{ + TlsClientTrustMode, TlsClientTrustPlan, TlsTrustFileSource, TlsTrustLoadError, +}; pub const ACCESS_FLOW_RELAY_NODE: &str = "@access-flow-relay"; -pub(crate) const MAX_ACCESS_FLOW_TRUST_PATH_BYTES: usize = 4096; -pub(crate) const MAX_ACCESS_FLOW_TRUST_PATH_COMPONENTS: usize = 256; const REMOVED_LOCAL_FLOW_RELAY_NODE: &str = "@local-flow-relay"; #[derive(Debug, Clone, Deserialize, Serialize)] @@ -691,7 +692,9 @@ pub(crate) enum CompiledAccessFlowRelayEndpoint { tls_index: usize, address: TlsAccessFlowAddress, server_name: TlsAccessFlowServerName, - trust_path: PathBuf, + trust_mode: TlsClientTrustMode, + trust_plan: TlsClientTrustPlan, + trust_path: Option, }, } @@ -804,7 +807,13 @@ impl AccessFlowRelayRoute { address, server_name, trust, - } => validate_tls_transport_templates(address, server_name, trust)?, + ca_certificate, + } => validate_tls_transport_templates( + address, + server_name, + *trust, + ca_certificate.as_deref(), + )?, } if self.listen.contains('{') { anyhow::bail!( @@ -831,7 +840,13 @@ impl AccessFlowRelayRoute { address, server_name, trust, - } => validate_tls_transport_templates(address, server_name, trust)?, + ca_certificate, + } => validate_tls_transport_templates( + address, + server_name, + *trust, + ca_certificate.as_deref(), + )?, } self.listen.parse::()? } @@ -855,10 +870,23 @@ impl AccessFlowRelayRoute { address, server_name, trust, + ca_certificate, } => { let tls_index = tls_index.expect("the relay compiler assigns an index to every TLS transport"); - let AccessFlowRelayTrust::PemBundle { path } = trust; + let trust_path = ca_certificate.as_ref().map(PathBuf::from); + let authored_source = trust_path + .clone() + .map(TlsTrustFileSource::new) + .transpose() + .map_err(map_tls_trust_source_config_error)?; + let trust_plan = TlsClientTrustPlan::new( + *trust, + authored_source + .as_ref() + .map(TlsTrustFileSource::authored_source_id), + ) + .map_err(map_tls_trust_config_error)?; CompiledAccessFlowRelayEndpoint::TlsTcp { tls_index, address: TlsAccessFlowAddress::parse(address).context( @@ -867,7 +895,9 @@ impl AccessFlowRelayRoute { server_name: TlsAccessFlowServerName::parse(server_name).context( "container_agent.access_flow_relay.routes.transport.server_name is invalid", )?, - trust_path: normalized_absolute_trust_path(path)?, + trust_mode: *trust, + trust_plan, + trust_path, } } }; @@ -915,7 +945,8 @@ fn compile_unix_endpoint(path: &str) -> anyhow::Result { fn validate_tls_transport_templates( address: &str, server_name: &str, - trust: &AccessFlowRelayTrust, + trust: TlsClientTrustMode, + ca_certificate: Option<&str>, ) -> anyhow::Result<()> { validate_template( "container_agent.access_flow_relay.routes.transport.address", @@ -927,49 +958,24 @@ fn validate_tls_transport_templates( server_name, &[], )?; - let AccessFlowRelayTrust::PemBundle { path } = trust; - validate_template( - "container_agent.access_flow_relay.routes.transport.trust.path", - path, - &[], - ) -} - -fn normalized_absolute_trust_path(path: &str) -> anyhow::Result { - if path.len() > MAX_ACCESS_FLOW_TRUST_PATH_BYTES { - anyhow::bail!( - "container_agent.access_flow_relay.routes.transport.trust.path must contain at most {MAX_ACCESS_FLOW_TRUST_PATH_BYTES} UTF-8 bytes" - ); - } - let mut segments = path.split('/'); - let rooted = segments.next() == Some(""); - if !rooted { - anyhow::bail!( - "container_agent.access_flow_relay.routes.transport.trust.path must be a normalized absolute non-root path" - ); - } - let mut component_count = 0_usize; - for segment in segments { - if segment.is_empty() || matches!(segment, "." | "..") { - anyhow::bail!( - "container_agent.access_flow_relay.routes.transport.trust.path must be a normalized absolute non-root path" - ); - } - component_count = component_count - .checked_add(1) - .ok_or_else(|| anyhow::anyhow!("trust path component count overflow"))?; - if component_count > MAX_ACCESS_FLOW_TRUST_PATH_COMPONENTS { - anyhow::bail!( - "container_agent.access_flow_relay.routes.transport.trust.path must contain at most {MAX_ACCESS_FLOW_TRUST_PATH_COMPONENTS} components" - ); - } - } - if component_count == 0 { - anyhow::bail!( - "container_agent.access_flow_relay.routes.transport.trust.path must be a normalized absolute non-root path" - ); + if let Some(path) = ca_certificate { + validate_template( + "container_agent.access_flow_relay.routes.transport.ca_certificate", + path, + &[], + )?; } - Ok(PathBuf::from(path)) + let source = ca_certificate + .map(PathBuf::from) + .map(TlsTrustFileSource::new) + .transpose() + .map_err(map_tls_trust_source_config_error)?; + TlsClientTrustPlan::new( + trust, + source.as_ref().map(TlsTrustFileSource::authored_source_id), + ) + .map(|_| ()) + .map_err(map_tls_trust_config_error) } #[derive(Debug, Clone, Deserialize, Serialize)] @@ -981,14 +987,37 @@ pub enum AccessFlowRelayTransport { TlsTcp { address: String, server_name: String, - trust: AccessFlowRelayTrust, + trust: TlsClientTrustMode, + ca_certificate: Option, }, } -#[derive(Debug, Clone, Deserialize, Serialize)] -#[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)] -pub enum AccessFlowRelayTrust { - PemBundle { path: String }, +fn map_tls_trust_config_error(error: TlsTrustLoadError) -> anyhow::Error { + match error { + TlsTrustLoadError::InvalidPlan => anyhow::anyhow!( + "container_agent.access_flow_relay.routes.transport ca_certificate does not match trust mode" + ), + TlsTrustLoadError::ResourceLimit => anyhow::anyhow!( + "container_agent.access_flow_relay.routes.transport.ca_certificate exceeds a resource bound" + ), + _ => anyhow::anyhow!( + "container_agent.access_flow_relay.routes.transport.ca_certificate is invalid" + ), + } +} + +fn map_tls_trust_source_config_error(error: TlsTrustLoadError) -> anyhow::Error { + match error { + TlsTrustLoadError::InvalidPlan => anyhow::anyhow!( + "container_agent.access_flow_relay.routes.transport.ca_certificate must be a normalized absolute non-root path" + ), + TlsTrustLoadError::ResourceLimit => anyhow::anyhow!( + "container_agent.access_flow_relay.routes.transport.ca_certificate exceeds a resource bound" + ), + _ => anyhow::anyhow!( + "container_agent.access_flow_relay.routes.transport.ca_certificate is invalid" + ), + } } fn map_relay_plan_error(error: AccessFlowRelayPlanError) -> anyhow::Error { diff --git a/src/config/tests.rs b/src/config/tests.rs index bf37a3e..f59abba 100644 --- a/src/config/tests.rs +++ b/src/config/tests.rs @@ -5807,10 +5807,8 @@ path = "/run/acl-proxy/transparent-http.sock""#, r#"kind = "tls_tcp" address = "proxy.example.test:7443" server_name = "proxy.example.test" - -[container_agent.access_flow_relay.routes.transport.trust] -kind = "pem_bundle" -path = "/etc/aw-gateway/acl-proxy-roots.pem""#, +trust = "custom" +ca_certificate = "/etc/aw-gateway/acl-proxy-roots.pem""#, ) } @@ -5833,10 +5831,8 @@ allowed_destination_ports = [443] kind = "tls_tcp" address = "proxy.example.test:7444" server_name = "proxy.example.test" - -[container_agent.access_flow_relay.routes.transport.trust] -kind = "pem_bundle" -path = "/etc/aw-gateway/acl-proxy-roots.pem" +trust = "custom" +ca_certificate = "/etc/aw-gateway/acl-proxy-roots.pem" "#; unix.replacen( @@ -5874,7 +5870,9 @@ fn container_agent_accepts_strict_tls_and_mixed_relay_transports() { tls_index, address, server_name, + trust_mode, trust_path, + .. } = compiled.plan.routes()[0].endpoint() else { panic!("TLS route compiled as the wrong transport"); @@ -5883,7 +5881,11 @@ fn container_agent_accepts_strict_tls_and_mixed_relay_transports() { assert_eq!(address.port().get(), 7443); assert_eq!(address.host().dns_name(), Some("proxy.example.test")); assert_eq!(server_name.host().dns_name(), Some("proxy.example.test")); - assert_eq!(trust_path, Path::new("/etc/aw-gateway/acl-proxy-roots.pem")); + assert_eq!(*trust_mode, access_tls_trust::TlsClientTrustMode::Custom); + assert_eq!( + trust_path.as_deref(), + Some(Path::new("/etc/aw-gateway/acl-proxy-roots.pem")) + ); } let mixed: ContainerAgentFile = toml::from_str(&mixed_access_flow_relay()).unwrap(); @@ -5905,6 +5907,56 @@ fn container_agent_accepts_strict_tls_and_mixed_relay_transports() { )); } +#[test] +fn tls_transport_enforces_explicit_four_mode_trust_matrix() { + let custom = bearer_tls_access_flow_relay(); + for accepted in [ + custom.clone(), + custom.replace("trust = \"custom\"", "trust = \"system_plus_custom\""), + custom + .replace("trust = \"custom\"\n", "trust = \"system\"\n") + .replace( + "ca_certificate = \"/etc/aw-gateway/acl-proxy-roots.pem\"\n", + "", + ), + custom + .replace("trust = \"custom\"\n", "trust = \"insecure\"\n") + .replace( + "ca_certificate = \"/etc/aw-gateway/acl-proxy-roots.pem\"\n", + "", + ), + ] { + let config: ContainerAgentFile = toml::from_str(&accepted).unwrap(); + config.validate().unwrap(); + } + + for rejected in [ + custom.replace("trust = \"custom\"\n", ""), + custom.replace( + "trust = \"custom\"", + "trust = \"system\"\nca_certificate_extra = \"forbidden\"", + ), + custom.replace( + "trust = \"custom\"\nca_certificate = \"/etc/aw-gateway/acl-proxy-roots.pem\"", + "trust = \"system\"\nca_certificate = \"/etc/aw-gateway/acl-proxy-roots.pem\"", + ), + custom.replace( + "trust = \"custom\"\nca_certificate = \"/etc/aw-gateway/acl-proxy-roots.pem\"", + "trust = \"insecure\"\nca_certificate = \"/etc/aw-gateway/acl-proxy-roots.pem\"", + ), + custom.replace( + "trust = \"custom\"\nca_certificate = \"/etc/aw-gateway/acl-proxy-roots.pem\"", + "[container_agent.access_flow_relay.routes.transport.trust]\nkind = \"pem_bundle\"\npath = \"/etc/aw-gateway/acl-proxy-roots.pem\"", + ), + ] { + let parsed = toml::from_str::(&rejected); + assert!( + parsed.as_ref().is_err() || parsed.unwrap().validate().is_err(), + "unexpectedly accepted:\n{rejected}" + ); + } +} + #[test] fn tls_route_indices_are_sequential_and_exclude_unix_routes() { let second_tls = r#" @@ -5917,10 +5969,8 @@ allowed_destination_ports = [8443] kind = "tls_tcp" address = "proxy-two.example.test:7443" server_name = "proxy-two.example.test" - -[container_agent.access_flow_relay.routes.transport.trust] -kind = "pem_bundle" -path = "/etc/aw-gateway/second-roots.pem" +trust = "custom" +ca_certificate = "/etc/aw-gateway/second-roots.pem" "#; let raw = mixed_access_flow_relay().replacen( @@ -6092,11 +6142,11 @@ fn tls_transport_rejects_invalid_addresses_names_and_trust_paths() { fn tls_trust_path_enforces_authored_byte_and_component_bounds() { let exact_bytes = format!( "/{}", - "a".repeat(super::agent::MAX_ACCESS_FLOW_TRUST_PATH_BYTES - 1) + "a".repeat(access_tls_trust::MAX_TLS_TRUST_SOURCE_LOCATOR_BYTES - 1) ); assert_eq!( exact_bytes.len(), - super::agent::MAX_ACCESS_FLOW_TRUST_PATH_BYTES + access_tls_trust::MAX_TLS_TRUST_SOURCE_LOCATOR_BYTES ); let exact_bytes_raw = bearer_tls_access_flow_relay().replace("/etc/aw-gateway/acl-proxy-roots.pem", &exact_bytes); @@ -6109,7 +6159,7 @@ fn tls_trust_path_enforces_authored_byte_and_component_bounds() { let over_bytes_config: ContainerAgentFile = toml::from_str(&over_bytes_raw).unwrap(); let over_bytes_error = over_bytes_config.validate().unwrap_err().to_string(); assert!( - over_bytes_error.contains("at most 4096 UTF-8 bytes"), + over_bytes_error.contains("exceeds a resource bound"), "{over_bytes_error}" ); assert!( @@ -6119,9 +6169,7 @@ fn tls_trust_path_enforces_authored_byte_and_component_bounds() { let exact_components = format!( "/{}", - std::iter::repeat_n("a", super::agent::MAX_ACCESS_FLOW_TRUST_PATH_COMPONENTS) - .collect::>() - .join("/") + std::iter::repeat_n("a", 256).collect::>().join("/") ); let exact_components_raw = bearer_tls_access_flow_relay() .replace("/etc/aw-gateway/acl-proxy-roots.pem", &exact_components); @@ -6135,7 +6183,7 @@ fn tls_trust_path_enforces_authored_byte_and_component_bounds() { let over_components_config: ContainerAgentFile = toml::from_str(&over_components_raw).unwrap(); let over_components_error = over_components_config.validate().unwrap_err().to_string(); assert!( - over_components_error.contains("at most 256 components"), + over_components_error.contains("exceeds a resource bound"), "{over_components_error}" ); assert!( @@ -6156,11 +6204,15 @@ fn relay_transport_schema_rejects_unknown_cross_variant_and_trust_shapes() { "address = \"proxy.example.test:7443\"", "address = \"proxy.example.test:7443\"\npath = \"/run/fallback.sock\"", ), - tls.replace("kind = \"pem_bundle\"", "kind = \"platform\""), tls.replace( - "kind = \"pem_bundle\"", - "kind = \"pem_bundle\"\ninsecure = true", + "trust = \"custom\"", + "trust = \"custom\"\ninsecure = true", + ), + tls.replace( + "trust = \"custom\"\nca_certificate = \"/etc/aw-gateway/acl-proxy-roots.pem\"", + "trust = \"custom\"\npath = \"/etc/aw-gateway/acl-proxy-roots.pem\"", ), + tls.replace("trust = \"custom\"", "trust = \"platform\""), AGENT_ACCESS_FLOW_RELAY.replace( "path = \"/run/acl-proxy/transparent-http.sock\"", "path = \"/run/acl-proxy/transparent-http.sock\"\naddress = \"proxy.example.test:7443\"", diff --git a/tests/agent_control.rs b/tests/agent_control.rs index 0c19727..5cb1879 100644 --- a/tests/agent_control.rs +++ b/tests/agent_control.rs @@ -1054,10 +1054,8 @@ allowed_destination_ports = [{destination_port}] kind = "tls_tcp" address = "{remote_address}" server_name = "access-flow.test" - -[container_agent.access_flow_relay.routes.transport.trust] -kind = "pem_bundle" -path = "{trust_path}" +trust = "custom" +ca_certificate = "{trust_path}" [[container_agent.services]] name = "base" @@ -1110,11 +1108,32 @@ depends_on = ["base"] wait_for_path(&base_ready); wait_for_path(&dependent_ready); wait_for_relay_ready(&control_socket, true); + let ready = control_request( + &control_socket, + br#"{"id":"ready-status","method":"status"}"#, + ); + assert_eq!( + ready["result"]["access_flow_relay"]["routes"][0]["trust_mode"], + "custom" + ); + assert!( + ready["result"]["access_flow_relay"] + .get("trust_failure") + .is_none() + ); std::fs::write(&trust_path, "not a PEM trust bundle").unwrap(); signal_process(&child, libc::SIGHUP); wait_for_log(&log_rx, "access flow relay trust reload failed"); wait_for_relay_ready(&control_socket, false); + let failed = control_request( + &control_socket, + br#"{"id":"failed-status","method":"status"}"#, + ); + assert_eq!( + failed["result"]["access_flow_relay"]["trust_failure"], + "invalid_material" + ); assert!(child.try_wait().unwrap().is_none()); assert!(!stop_order.exists()); diff --git a/tests/assets.rs b/tests/assets.rs index 25cc074..48ba6ed 100644 --- a/tests/assets.rs +++ b/tests/assets.rs @@ -580,7 +580,7 @@ fn validate_tls_access_flow_smoke(script: &str) -> Result<(), &'static str> { "schema_version = 4", "kind = \"tls_tcp\"", "server_name = \"proxy.access-flow.test\"", - "kind = \"pem_bundle\"", + "trust = \"custom\"", "[container_agent.access_flow_relay]", "/opt/aw-gateway/bin/aw-container-agent", "invalid bearer reached the authorization provider", @@ -1055,7 +1055,7 @@ fn tls_access_flow_cross_host_smoke_preserves_diagnostics_and_is_awk_portable() assert!(smoke.contains("max_connections = 64")); assert_eq!( smoke - .matches("path = \"/run/aw-gateway/trust/access-flow-root.pem\"") + .matches("ca_certificate = \"/run/aw-gateway/trust/access-flow-root.pem\"") .count(), 2 ); diff --git a/tests/example_configs.rs b/tests/example_configs.rs index 7250bdc..bd18ea7 100644 --- a/tests/example_configs.rs +++ b/tests/example_configs.rs @@ -1,7 +1,6 @@ use assert_cmd::Command; use aw_gateway::config::{ - AccessFlowRelayTransport, AccessFlowRelayTrust, ContainerAgentFile, ContainerMountMode, - GatewayConfig, + AccessFlowRelayTransport, ContainerAgentFile, ContainerMountMode, GatewayConfig, }; use std::path::Path; @@ -116,18 +115,14 @@ fn remote_tls_gateway_and_agent_examples_share_one_relay_contract() { AccessFlowRelayTransport::TlsTcp { address: agent_address, server_name: agent_server_name, - trust: - AccessFlowRelayTrust::PemBundle { - path: agent_trust_path, - }, + trust: agent_trust, + ca_certificate: agent_trust_path, }, AccessFlowRelayTransport::TlsTcp { address: gateway_address, server_name: gateway_server_name, - trust: - AccessFlowRelayTrust::PemBundle { - path: gateway_trust_path, - }, + trust: gateway_trust, + ca_certificate: gateway_trust_path, }, ) = (&agent_route.transport, &gateway_route.transport) else { @@ -135,10 +130,12 @@ fn remote_tls_gateway_and_agent_examples_share_one_relay_contract() { }; assert_eq!(agent_address, gateway_address); assert_eq!(agent_server_name, gateway_server_name); + assert_eq!(agent_trust, gateway_trust); + assert_eq!(*agent_trust, access_tls_trust::TlsClientTrustMode::Custom); assert_eq!(agent_trust_path, gateway_trust_path); assert_eq!( - agent_trust_path, - "/etc/aw-gateway/acl-proxy-trust/roots.pem" + agent_trust_path.as_deref(), + Some("/etc/aw-gateway/acl-proxy-trust/roots.pem") ); }