diff --git a/.github/workflows/turn-endpoint.yml b/.github/workflows/turn-endpoint.yml new file mode 100644 index 00000000..9f805ac7 --- /dev/null +++ b/.github/workflows/turn-endpoint.yml @@ -0,0 +1,77 @@ +# Does the relay credentials endpoint still answer? +# +# `mediad::turn` fetches short-lived Cloudflare credentials so a robot can offer a `relay` +# candidate to a consumer that cannot reach it directly. The endpoint it used before +# 2026-09 had been dead since June — a dangling Route53 delegation — and **nothing +# noticed for three months**, because the only symptom is a warning in a journal nobody +# reads and a candidate type nobody counts. Every other check in this repository passes +# regardless: they pair two peers on one network, which never looks at a relay. +# +# So this is the one check that would have caught it, and it is deliberately *not* on +# `pull_request`. The failure being guarded against is "nobody touched this for months", +# which a PR trigger cannot see; and a third party's outage must never block work that has +# nothing to do with it. A scheduled failure mails whoever owns the repository, which is +# the right blast radius for "an upstream service went away". +name: turn-endpoint + +on: + schedule: + # Daily. The endpoint is somebody else's Space, and a week of relay coverage lost + # before anybody hears about it is most of the damage already done. + - cron: "41 5 * * *" + workflow_dispatch: + +jobs: + credentials: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + # **Read the URL out of the source rather than repeating it here.** A check with its + # own copy of the endpoint tests whatever it was last told, which can drift from what + # the daemon compiles in — and a green check on an endpoint no robot uses is worse + # than no check, because it reads as proof. + - name: Ask the endpoint the daemon uses + env: + HF_TOKEN: ${{ secrets.HF_TOKEN }} + run: | + set -euo pipefail + + url=$(sed -n 's/^pub const DEFAULT_TURN_ENDPOINT: &str = "\(.*\)";$/\1/p' \ + mediad/src/turn.rs) + [ -n "$url" ] || { + echo "::error::could not read DEFAULT_TURN_ENDPOINT from mediad/src/turn.rs" + exit 1 + } + echo "endpoint: $url" + + # No token means this cannot check anything. Failing is the point: a check that + # skips itself into permanent silence is the failure mode this workflow exists + # to end. + [ -n "${HF_TOKEN:-}" ] || { + echo "::error::no HF_TOKEN secret. Add a Hugging Face token with no scopes" \ + "beyond sign-in; it is used only to mint TURN credentials." + exit 1 + } + + code=$(curl -sS -o body.json -w '%{http_code}' --max-time 30 \ + -H "Authorization: Bearer $HF_TOKEN" "$url?ttl=600") + [ "$code" = "200" ] || { + echo "::error::$url answered HTTP $code, not 200" + head -c 400 body.json || true + exit 1 + } + + # A 200 carrying no relay is still a robot with no relay candidate to offer — + # which is the outage, not a milder version of it. `stun:` entries do not count: + # `webrtcbin` takes a STUN server through its own property, and this is about the + # thing STUN cannot do. + relays=$(jq '[.iceServers[]? | .urls] | flatten + | map(select(startswith("turn:") or startswith("turns:"))) + | length' body.json) + echo "relay servers offered: $relays" + [ "$relays" -gt 0 ] || { + echo "::error::$url answered 200 with no turn:/turns: entries" + head -c 400 body.json || true + exit 1 + } diff --git a/Cargo.lock b/Cargo.lock index 86d85c20..e5212303 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2732,6 +2732,7 @@ dependencies = [ "tokio-tungstenite 0.30.0", "tracing", "tracing-subscriber", + "url", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 04bfe08d..05287a06 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -90,6 +90,7 @@ tracing = "0.1" tracing-subscriber = "0.3" tokio-util = "0.7" async-trait = "0.1" +url = "2" # No custom [profile.release]. Binary size is not worth optimising for: model artifacts # will dwarf a few MB of binary, and the tuning cost more than it bought — diff --git a/docs/design/remote-access-design.md b/docs/design/remote-access-design.md index 275b40f0..0c6f5d05 100644 --- a/docs/design/remote-access-design.md +++ b/docs/design/remote-access-design.md @@ -762,10 +762,11 @@ error, deliberately — safety refusing to run a policy on a fallen robot is not page that only catches errors reports every one of those as a success and leaves a motionless robot unexplained. -**And the transport is the one thing this cannot prove from a Space yet.** The control channel is -SCTP over the same candidate pair as the media, so §6's dead TURN endpoint takes the click with -it: from a data centre the session negotiates and may then carry nothing. That is why the status -line names the stage it reached rather than saying "connecting…" — and why the page has a second +**And the transport is what this leans on hardest from a Space.** The control channel is SCTP over +the same candidate pair as the media, so a relay that is not there takes the click with it: from a +data centre the session negotiates and may then carry nothing. §6's endpoint answers now, so the +ordinary case is covered — but a metered dependency is still a dependency, which is why the status +line names the stage it reached rather than saying "connecting…", and why the page has a second way in. **`lan.py` is that second way, and it is a transport rather than a second design.** The robot is @@ -856,11 +857,11 @@ matters more than it sounds, because `aiortc`'s STUN client works where its TURN so a Python consumer *cannot* be the side that relays. `reachy_mini`'s #1182 established this arrangement and `mediad::turn` is the same one. -The credentials are Cloudflare's, minted per account by a proxy Hugging Face hosts -(`turn.fastrtc.org/credentials`) and authenticated with **the same token the relay signs in with** -— so a robot that belongs to somebody can offer a relay and one that belongs to nobody cannot, -which is the same line §2 draws everywhere else. They are short-lived: a task refreshes at half of -a 600 s lifetime, and retries in thirty seconds after a *transient* failure only. A robot nobody +The credentials are Cloudflare's, minted per account by a proxy Hugging Face hosts and +authenticated with **the same token the relay signs in with** — so a robot that belongs to +somebody can offer a relay and one that belongs to nobody cannot, which is the same line §2 draws +everywhere else. They are short-lived: a task refreshes at half of a 600 s lifetime, and retries +in thirty seconds after a *transient* failure only. A robot nobody has signed in has nothing to retry for, and a warning every thirty seconds for the life of the daemon is how a log stops being read. @@ -872,22 +873,61 @@ blocks (a `try_read` that yields nothing rather than waiting) and never fails. A the ordinary state for the first few seconds after boot and forever on a robot with no account, and it means host and srflx only, which is all anything on the same network needs. -**And the endpoint is not answering, which is where this stands.** `turn.fastrtc.org` has no A -record and `fastrtc.org` has no NS records at all, from three public resolvers and from the board -— so the proxy both `fastrtc`'s own current code and `reachy_mini`'s #1182 point at cannot be -reached by anybody. Their documentation describes it as a live Hugging Face–Cloudflare arrangement -(10 GB a month free with an account), so this reads as a lapsed registration or an outage rather -than a moved URL, and it means the mini fleet's relay path is down too. Worth telling whoever owns -`fastrtc`. - -Three ways on, in the order they should be considered: - -- **Wait, having reported it.** The robot degrades exactly as designed — a warning every thirty - seconds and host/srflx candidates — so nothing is broken except reaching a robot from a network - that needs a relay. -- **Our own proxy**, which is what that endpoint is: a small service holding a Cloudflare Calls key - and minting short-lived credentials for a caller presenting a valid HF token. `--turn-url` is - already the seam it plugs into, and the key stays in one place rather than on robots. +**The proxy is the Space, and the name in front of it was the dead part.** `turn.fastrtc.org` — +what `fastrtc`'s own code points at and what `reachy_mini` #1182 copied into this arrangement — is +a dangling delegation, not an outage: the `.org` registry names four Route53 nameservers for the +zone, the registration is healthy and locked until 2027, and all four nameservers answer `REFUSED` +for the zone they are authoritative for, which is what Route53 says when the hosted zone behind +them is gone. `fastrtc/turn-service`, the Space that alias pointed at, never stopped answering. +So `DEFAULT_TURN_ENDPOINT` addresses it directly, at +`https://fastrtc-turn-service.hf.space/credentials`, and the vanity record is out of the path. + +**That also closes a token-exfiltration route, which is the half that mattered more than the +outage.** A dangling Route53 delegation is a known takeover shape — create hosted zones until AWS +assigns you one of the four delegated nameservers, and one is enough, because a resolver needs +only one authoritative answer. Whoever landed it would serve records for the name, pass DNS +validation for a certificate on it, and be handed the account token every signed-in robot sends +as a bearer header every five minutes. That token is the robot's whole credential; §2.4 is about +how broad its scopes are. + +**The endpoint is checked before the token can reach it.** `turn::parse_endpoint` is a `clap` +`value_parser` on `--turn-url`: `https` unless the host is loopback, no userinfo, no query and no +fragment. A wrong value stops the daemon at argument parsing rather than becoming a warning every +thirty seconds. Redirects need no separate guard — `reqwest` strips `Authorization` when a +redirect crosses scheme, host or port (`src/redirect.rs`, `remove_sensitive_headers`), so the +token cannot be walked to a third-party origin. + +**The allowance is the robot owner's, and it is finite** — 10 GB a month on a free Hugging Face +account. A relayed video session is roughly a gigabyte an hour, so a robot driven hard over a +relay can spend it, after which the proxy answers with no relay servers and `turn.rs` logs "the +TURN proxy offered no relay servers" at info level. That is indistinguishable from a robot that +was never offered one, which is a legibility gap worth closing when somebody hits it. It is also +the argument `stream.rs` makes for sending frames outbound rather than through a relay. + +**Something now notices when it dies, which is the reason this went unnoticed for three months.** +The endpoint was already dead when #1182 shipped it, and the only symptom was a warning in a log +and a candidate type nobody counted. `.github/workflows/turn-endpoint.yml` runs daily: one +authenticated `GET` against `DEFAULT_TURN_ENDPOINT`, asserting 200 and at least one `turn:`/ +`turns:` entry. Every other check in the repository passes regardless, because every one of them +pairs two peers on one network, which never looks at a relay. + +Three things about its shape are deliberate. It reads the URL **out of `turn.rs` with `sed`** +rather than keeping a copy, because a check holding its own endpoint tests whatever it was last +told and can drift from what the daemon compiles in — and a green check on a URL no robot uses is +worse than no check, since it reads as proof. It is **not** on `pull_request`: the failure being +guarded against is "nobody touched this for months", which a PR trigger cannot see, and a third +party's outage must never block unrelated work. And a **missing `HF_TOKEN` secret fails** rather +than skipping, because a check that quietly skips itself into permanent silence is the exact +failure mode it exists to end. + +Two things this deliberately does not do: + +- **Our own proxy.** This endpoint is a small service holding a Cloudflare Calls key and minting + short-lived credentials for a caller with a valid HF token, and running one ourselves would end + the dependency on a dormant project's Space — `--turn-url` is already the seam it plugs into. + Worth doing, and not worth blocking relay coverage on: `*.hf.space` is `{owner}-{space}`, so it + breaks if `fastrtc` renames or removes the Space, and there is no CNAME layer left to repoint. + That is the residual risk, named rather than closed. - **A Cloudflare key on the robot**, using `TURN_KEY_ID` and `TURN_KEY_API_TOKEN` directly. Fastest and worst: a long-lived API token on every board, which is the shape of mistake §2.4 exists to stop making. @@ -949,8 +989,10 @@ Five slices, and the first two are independently useful and need no client: bridge to, which is a robot whose pipeline never reached PLAYING. 5. **STUN decided; TURN offered by the robot.** §6. **Done**: `stun.l.google.com:19302` on both ends, and `mediad::turn` keeps Cloudflare credentials fresh so every consumer's offer carries a - `relay` candidate. §6 has the argument; the two things to carry away are that only the robot - needs credentials, and that reading them must never block the thread building an offer. + `relay` candidate — from `fastrtc-turn-service.hf.space`, the Space itself, because the alias + in front of it is a dangling delegation. §6 has the argument; the three things to carry away + are that only the robot needs credentials, that reading them must never block the thread + building an offer, and that no check in this repo would notice if that endpoint died again. ## 9. What is open, and who can close it @@ -960,12 +1002,15 @@ Five slices, and the first two are independently useful and need no client: | a calibration for the camera | `media.video` publishes the module's design figures with `calibrated: false`, which is enough to map a room and not enough for metrology. Measuring one robot and writing `[media.intrinsics]` closes it for that robot; a per-unit calibration in provisioning closes it for the family. §11 of `remote-webrtc.md` | | everything on the wire should be timestamped at source | `remote-webrtc.md` §11: `abs-capture-time` on the media, checked against what `webrtcsink`, a browser and `aiortc` actually surface; and a monotonic-plus-epoch field on every control-channel notification that describes a moment. Wanted for any consumer that has to relate what the robot saw to what it felt — visual-inertial SLAM is the case that makes it concrete — and it wants its own version bump rather than riding along with a transport | | §2.6 `logout` revokes nothing | whether Hugging Face accepts a revocation for the first-party device-code client, checked rather than assumed. Not blocking — signing out stops the robot being reachable, and a stolen board is answered on hf.co — but it is the difference between "forgotten" and "revoked" | +| §6 the relay check needs a token | `.github/workflows/turn-endpoint.yml` exists and runs daily, and fails until an `HF_TOKEN` secret is set on the repository — a Hugging Face token with no scope beyond sign-in, used only to mint TURN credentials. Failing loudly is deliberate; the alternative is a check that skips itself into silence | +| §6 the relay is somebody else's Space | a credentials proxy of our own, holding the Cloudflare key in one place instead of trusting a dormant project's Space to keep its name. `--turn-url` is the seam. Not blocking — the Space answers — but `*.hf.space` is `{owner}-{space}` and there is no alias left to repoint if it moves | Closed since this page was written: the OAuth client (§2.3 — Hugging Face ships one), whether the token expires (§2.7 — thirty days, with a rotating refresh token), which rendezvous to use (§4 — the mini's), whether we can read it (§4 — we maintain it; the "private repo" in an earlier draft -was a wrong-name 401), and where the client is served (§5 — a static Space with `hf_oauth`, because -the question was never hosting but how a page gets a token). +was a wrong-name 401), where the client is served (§5 — a static Space with `hf_oauth`, because +the question was never hosting but how a page gets a token), and which relay endpoint to use (§6 — +the Space itself; the alias in front of it is a dangling delegation). One item this page created and closed: **peers are keyed by token**, so two things sharing one take turns being reachable. Not a provisioning problem — images are built from scratch, not cloned, diff --git a/mediad/Cargo.toml b/mediad/Cargo.toml index 0581fdc3..36ab01b0 100644 --- a/mediad/Cargo.toml +++ b/mediad/Cargo.toml @@ -50,6 +50,11 @@ reqwest = { version = "0.13.4", default-features = false, features = ["rustls", # reconnection field. Sixty lines to hand-roll and one of them would be the bug: a `data:` split # across two TCP reads. eventsource-stream = "0.2.3" +# Parsing, not fetching: the one endpoint this daemon sends the account token to is +# checked before the token can reach it, and "is this loopback" and "does this carry +# userinfo" are the two questions a hand-rolled split on `:` gets wrong. Already in the +# lock via `reqwest`. +url.workspace = true # `StreamExt`, for the byte stream `reqwest` hands over. Already in the lock via `updater`. futures-util = "0.3.33" # The local half of the bridge: a WebSocket client to `webrtcsink`'s own signalling server on diff --git a/mediad/src/main.rs b/mediad/src/main.rs index 448c9e42..6d5255d0 100644 --- a/mediad/src/main.rs +++ b/mediad/src/main.rs @@ -55,7 +55,11 @@ struct Args { /// Hugging Face hosts this proxy and mints Cloudflare credentials for the account the token /// belongs to, which is why offering a relay needs no new secret on the robot. A flag for /// pointing a board at a fake; there is nothing to choose on a real one. - #[arg(long, default_value = mediad::turn::DEFAULT_TURN_ENDPOINT)] + /// + /// Checked here rather than trusted: this is the one URL the account token is sent to, and + /// `parse_endpoint` says what it will not send it over. + #[arg(long, default_value = mediad::turn::DEFAULT_TURN_ENDPOINT, + value_parser = mediad::turn::parse_endpoint)] turn_url: String, /// Do not register with the rendezvous service, whatever the token file says. diff --git a/mediad/src/relay.rs b/mediad/src/relay.rs index 5e118114..68964a39 100644 --- a/mediad/src/relay.rs +++ b/mediad/src/relay.rs @@ -21,10 +21,10 @@ //! //! The bridge above carries a *negotiation*: SDP and ICE, so that a consumer and the robot can //! find a path between them and speak WebRTC over it. When they cannot find one, everything built -//! on it is gone — and right now they frequently cannot, because a relay candidate needs -//! `turn.fastrtc.org`, which has no A record and whose zone has no NS records (§6). Signalling -//! crosses, media does not, and the control channel goes with it because SCTP rides the same -//! candidate pair. +//! on it is gone. A relay candidate is what keeps that from happening (§6) — and it is a +//! dependency rather than a guarantee: it is somebody else's service, metered per account, and +//! the control channel is SCTP over whatever pair ICE settled on, so a relay that stops being +//! available takes a JSON-RPC call of a few hundred bytes down with the video. //! //! So the JSON-RPC a consumer wants to send does not have to go through WebRTC at all, and the //! rendezvous turns out to already carry it: `handle_peer_message` in their `app.py` relays diff --git a/mediad/src/stream.rs b/mediad/src/stream.rs index c1f0a492..61656303 100644 --- a/mediad/src/stream.rs +++ b/mediad/src/stream.rs @@ -5,11 +5,10 @@ //! The goal is a Space on Hugging Face hardware processing this camera. The obvious route is the //! one `vision-demo` takes — a WebRTC consumer pulls the stream through the rendezvous — and it //! runs into the one thing WebRTC cannot do without help: a robot behind a home router and a -//! container behind a data centre's NAT need a **relay candidate** to pair, and -//! `turn.fastrtc.org` has no A record and its zone no NS records at all -//! (`remote-access-design.md` §6). Signalling crosses, media does not. +//! container behind a data centre's NAT need a **relay candidate** to pair +//! (`remote-access-design.md` §6). Signalling crosses; media needs somebody's relay. //! -//! An outbound WebSocket has no such problem. **The robot already proves this every second it is +//! An outbound WebSocket needs nobody's. **The robot already proves this every second it is //! reachable**: `relay.rs` holds an outbound HTTPS stream to a Space right now, and nothing about //! a home router objects. So the frames go the same way the registration does — outward — and NAT //! stops being a participant. @@ -23,6 +22,13 @@ //! this scale where relaying payload through a shared service would not: one small envelope per //! session, on a service the mini fleet also depends on, and the bytes go point to point. //! +//! **This is not a workaround for a relay that does not exist**, and it used to read like one. +//! `turn.rs` offers relay candidates and they work. The argument above is the argument either +//! way: a relay is metered per Hugging Face account, so every one of these frames would be spent +//! against an allowance (§6: 10 GB a month) that the robot's owner also needs for being +//! *watched* — and the shortest path between a board and a data centre is not through a third +//! one. A relay is the fallback for a session that cannot be made direct. This one can. +//! //! # What it is not //! //! Not a replacement for a relay candidate in general. There is no return media path, so nothing diff --git a/mediad/src/turn.rs b/mediad/src/turn.rs index 9850851b..a5cf51e4 100644 --- a/mediad/src/turn.rs +++ b/mediad/src/turn.rs @@ -37,8 +37,59 @@ use std::time::Duration; use serde::Deserialize; -/// Hugging Face's TURN credentials proxy, as `reachy_mini` uses it. -pub const DEFAULT_TURN_ENDPOINT: &str = "https://turn.fastrtc.org/credentials"; +/// Hugging Face's TURN credentials proxy, addressed as the Space it is. +/// +/// **Not `turn.fastrtc.org`**, the vanity name `fastrtc`'s own code points at and `reachy_mini` +/// #1182 copied. That name is a dangling delegation: the `.org` registry still names four Route53 +/// nameservers for the zone, the hosted zone behind them is gone, and all four answer `REFUSED` +/// for the zone they are authoritative for. The service never stopped answering — only the name +/// in front of it did — so this addresses `fastrtc/turn-service` directly. +/// +/// Which also takes a name that can be *taken over* out of the path: a signed-in robot sends its +/// account token down this URL every five minutes, and whoever wins a race to have AWS assign +/// them one of those four delegated nameservers would serve records for the name, pass DNS +/// validation for a certificate on it, and be handed the token. `reachy_mini` #1408 made the same +/// move and measured a relay pair carrying video through it. +pub const DEFAULT_TURN_ENDPOINT: &str = "https://fastrtc-turn-service.hf.space/credentials"; + +/// A `--turn-url` worth handing the account token to, or the reason it is not one. +/// +/// The token goes out as a bearer header on every refresh, so the destination is checked before +/// it can: **`https`**, unless the host is loopback and the endpoint is therefore a test fake or +/// a stand-in on the board itself. Userinfo is refused because one credential per request is +/// enough, and a query or fragment because [`fetch`] appends `?ttl=` to whatever it is given — +/// silently landing the TTL in a fragment, or as a second value of an existing parameter. +/// +/// A `clap` `value_parser`, so a wrong value stops the daemon while somebody is still looking at +/// the terminal. An endpoint that is wrong rather than refused becomes a warning every thirty +/// seconds for the life of the daemon, which is how a log stops being read. +pub fn parse_endpoint(value: &str) -> Result { + let url = url::Url::parse(value).map_err(|why| format!("not a URL: {why}"))?; + + if !url.username().is_empty() || url.password().is_some() { + return Err("carries userinfo, and one credential per request is enough".to_owned()); + } + if url.query().is_some() || url.fragment().is_some() { + return Err("carries a query or a fragment, and the ttl is appended to it".to_owned()); + } + + let loopback = match url.host() { + Some(url::Host::Domain(name)) => name == "localhost", + Some(url::Host::Ipv4(address)) => address.is_loopback(), + Some(url::Host::Ipv6(address)) => address.is_loopback(), + None => return Err("names no host".to_owned()), + }; + match url.scheme() { + "https" => {} + "http" if loopback => {} + "http" => { + return Err("is plain http, which puts the account token on the wire".to_owned()); + } + other => return Err(format!("{other} is not a scheme these can be fetched over")), + } + + Ok(url.into()) +} /// How long to ask for the credentials to be valid. const TTL: Duration = Duration::from_secs(600); @@ -334,6 +385,47 @@ mod tests { assert!(turn_uris(&servers(r#"{"iceServers":[]}"#)).is_empty()); } + /// **The default is one the guard accepts.** A default that fails its own check is a daemon + /// that will not start at all, on every robot at once. + #[test] + fn the_default_endpoint_is_one_the_token_may_be_sent_to() { + assert_eq!( + parse_endpoint(DEFAULT_TURN_ENDPOINT).as_deref(), + Ok(DEFAULT_TURN_ENDPOINT), + "and it survives the round trip unchanged, so `fetch` appends `?ttl=` to what was set" + ); + } + + /// What the guard is for: the destinations the account token must not go to. + #[test] + fn an_endpoint_that_would_leak_the_token_is_refused_at_argument_parsing() { + for (endpoint, because) in [ + ("http://turn.example/credentials", "plain http"), + ("https://user:pass@turn.example/credentials", "userinfo"), + ("https://turn.example/credentials?ttl=1", "query"), + ("https://turn.example/credentials#f", "fragment"), + ("ftp://turn.example/credentials", "scheme"), + ("/credentials", "relative"), + ] { + assert!( + parse_endpoint(endpoint).is_err(), + "{endpoint} was accepted, and it should have been refused for its {because}" + ); + } + } + + /// And loopback http is not one of them — every test below dials one. + #[test] + fn a_loopback_fake_needs_no_certificate() { + for endpoint in [ + "http://127.0.0.1:8080/credentials", + "http://localhost:8080/credentials", + "http://[::1]:8080/credentials", + ] { + assert!(parse_endpoint(endpoint).is_ok(), "{endpoint}"); + } + } + /// An unreachable endpoint says *why* it was unreachable. #[tokio::test] async fn a_failure_names_its_cause_and_not_just_itself() { diff --git a/spaces/policy-shop/README.md b/spaces/policy-shop/README.md index 0279cbf9..e1899e7a 100644 --- a/spaces/policy-shop/README.md +++ b/spaces/policy-shop/README.md @@ -65,17 +65,17 @@ A `perpetual` policy has no length of its own, so a one-shot made out of one is unwind. `robotctl policy add` refuses without `--hold` rather than picking a number; this page asks instead, and that is the seconds box above the rows. -## Two ways in, and right now only one of them works +## Two ways in **From anywhere** is the rendezvous: the robot registers as a producer holding its account token, this signs in as the visitor, and the service pairs them. It is the path that reaches a duck behind -its owner's router, and it needs a candidate pair that works — which today it may not have. -`turn.fastrtc.org` has no A record and its zone has no NS records at all -(`remote-access-design.md` §6), so neither side offers a `relay` and a session between a home -router and a data centre falls back to host and srflx. Often that punches a hole. Often it does -not, and because the control channel is SCTP over the same candidate pair, when it does not punch -nothing here works at all. The status line names the stage it reached instead of saying -"connecting…", so this failure is legible rather than mysterious. +its owner's router, and it needs a candidate pair that works. The robot offers a `relay` candidate +(`remote-access-design.md` §6), so a session between a home router and a data centre has a +fallback for when host and srflx do not punch a hole — which they often do not. That fallback is +somebody's bandwidth and it is metered, and because the control channel is SCTP over the same +candidate pair, a relay that stops being available takes every button on this page with it. The +status line names the stage it reached instead of saying "connecting…", so that failure is +legible rather than mysterious. **On this network** needs none of that. `mediad` is already serving `webrtcsink`'s signalling server on `ws://:8443` — it is what the robot's own console talks to — and it carries the diff --git a/spaces/policy-shop/app.py b/spaces/policy-shop/app.py index 5f446b69..12e218b4 100644 --- a/spaces/policy-shop/app.py +++ b/spaces/policy-shop/app.py @@ -21,11 +21,12 @@ ## Two things that will bite before the robot does -**Media may not connect from a data centre, and it is not this Space's fault.** A relay candidate -needs `turn.fastrtc.org`, which has no DNS at all right now (§6), so the session falls back to -host and srflx — often enough to punch a hole, and often enough not. The control channel is SCTP -over that same candidate pair, so when it does not punch, nothing here works. Running this file on -a laptop on the robot's own network is the way through — `uv run app.py`, and the README's +**Media from a data centre now rests on the relay, and that is worth knowing before it bites.** +The robot offers a `relay` candidate again (§6), so the session no longer depends on host and +srflx punching a hole. But the control channel is SCTP over whatever pair ICE settled on, so +anything that takes the relay away — an allowance spent, a proxy not answering — takes every +method on this page with it, for calls whose payload is a few hundred bytes. Running this file on +a laptop on the robot's own network needs no relay at all — `uv run app.py`, and the README's local-run section is the two lines that get there. **One consumer at a time.** That is the rendezvous's rule, not a simplification here: while this @@ -792,8 +793,8 @@ def open_session(peer_id: str | None, oauth: gr.OAuthToken | None) -> str: "**And there is no WebRTC in this path at all.** The calls are relayed as HTTP: the " "rendezvous forwards an `rpc` key on a `peer` envelope verbatim, and `mediad`'s " "control lane answers it out of the same routing table the datachannel uses. No ICE, " - "no DTLS, no relay candidate — so nothing here can be defeated by a NAT, which the " - "WebRTC version was, every time, while `turn.fastrtc.org` has no DNS (§6).\n\n" + "no DTLS, no relay candidate — so nothing here can be defeated by a NAT, and no " + "click waits on somebody's relay staying up (§6).\n\n" "The cost is pixels: video is RTP on the media path, so there is none on this tab. " "`robot.policies` says what the robot did; the tab beside this one shows it." ) diff --git a/spaces/policy-shop/lan.py b/spaces/policy-shop/lan.py index f64f12b7..b207ce6c 100644 --- a/spaces/policy-shop/lan.py +++ b/spaces/policy-shop/lan.py @@ -1,11 +1,11 @@ -"""The same session over the robot's own signalling server, for while TURN is down. - -The rendezvous path is the one that reaches a duck from anywhere, and it needs a relay candidate -to do it: between a home router and a data centre, srflx-to-srflx punches often enough to be worth -trying and not often enough to rely on. `turn.fastrtc.org` has no A record and its zone has no NS -records at all (`docs/design/remote-access-design.md` §6), so today the honest answer for a -consumer that wants to *drive* a duck is to be on the duck's network — where ICE needs no relay -because both sides offer host candidates. +"""The same session over the robot's own signalling server, for a consumer already on the LAN. + +The rendezvous path is the one that reaches a duck from anywhere, and it leans on a relay +candidate to do it: between a home router and a data centre, srflx-to-srflx punches often enough +to be worth trying and not often enough to rely on, so the fallback carries the session +(`docs/design/remote-access-design.md` §6). On the duck's own network none of that is needed — +both sides offer host candidates and ICE pairs them immediately — and a driving session is +exactly the kind that should not be paying a relay's latency to reach a robot two metres away. **This is a transport, not a second design.** `webrtcsink`'s signalling server is already running on the robot at `ws://:8443`, and it carries the same gst envelopes the rendezvous carries diff --git a/spaces/shared/wire.py b/spaces/shared/wire.py index 26ec07e4..398e9898 100644 --- a/spaces/shared/wire.py +++ b/spaces/shared/wire.py @@ -1,10 +1,11 @@ """JSON-RPC to a duck over the rendezvous, with no WebRTC in the path at all. The other two transports negotiate: they exchange SDP and ICE so the two ends can find a route -between them, and then speak WebRTC over it. That works on one network and it works across NATs -*when a relay candidate exists* — which right now it does not, because `turn.fastrtc.org` has no A -record and its zone has no NS records (`remote-access-design.md` §6). Signalling crosses, media -does not, and the control channel goes with it because SCTP rides the same candidate pair. +between them, and then speak WebRTC over it. That works on one network, and across NATs it works +*when a relay candidate exists* — which now it does (`remote-access-design.md` §6), but the +control channel is hostage to it either way, because SCTP rides the same candidate pair as the +media. A relay that is rate-limited, metered out, or briefly not answering takes the click with +it, for a call whose whole payload is a few hundred bytes. This needs none of it. The rendezvous already relays what is wanted: `handle_peer_message` in its `app.py` forwards **every key of a `peer` envelope except `type` and `sessionId`** verbatim to the diff --git a/spaces/vision-demo/README.md b/spaces/vision-demo/README.md index dfa24ee4..cf2f3756 100644 --- a/spaces/vision-demo/README.md +++ b/spaces/vision-demo/README.md @@ -21,10 +21,12 @@ over each decoded frame here. Nothing is on the robot's network and no relay is ## The robot dials us, and that is the whole design Pulling the camera through the rendezvous means WebRTC, and WebRTC between a robot behind a home -router and a container in a data centre needs a **relay candidate**. `turn.fastrtc.org` has no A -record and `fastrtc.org` no NS records at all (`remote-access-design.md` §6), so there is none: -signalling crosses and media never does. That was this Space's whole history — "signalling worked -and media did not" was its documented expected outcome. +router and a container in a data centre needs a **relay candidate** to fall back on. For this +Space's whole history there was none — the relay default every robot shipped with named a host +with no DNS behind it — and "signalling worked and media did not" was its documented expected +outcome. That default is fixed (`remote-access-design.md` §6), and inverting the direction is +still what this Space wants: a relay spends the robot owner's metered allowance on every frame, +and these frames are only ever decoded by a program. So the direction is inverted: diff --git a/spaces/vision-demo/app.py b/spaces/vision-demo/app.py index 524b002b..dc4f65b8 100644 --- a/spaces/vision-demo/app.py +++ b/spaces/vision-demo/app.py @@ -3,9 +3,11 @@ The robot is behind somebody's router and this runs in a container in a data centre. The obvious way to join them is the one this Space used to take: a WebRTC consumer pulls the stream through `reachy_mini_central`. That ran into the one thing WebRTC cannot do unaided — a relay candidate — -and `turn.fastrtc.org` has no A record and its zone no NS records at all -(`docs/design/remote-access-design.md` §6). Signalling crossed and media never did, from every -Space, every time. +and the relay default every robot shipped with pointed at a name with no DNS behind it, so +signalling crossed and media never did, from every Space, every time. That default is fixed +(`docs/design/remote-access-design.md` §6) and this design is still the right one: a relay costs +somebody's bandwidth, metered against the robot owner's Hugging Face allowance, for every frame +of a stream only a program is going to look at. **So the direction is inverted and the problem disappears.** The rendezvous is used for one small thing — telling the robot where to send frames — and the frames come **outbound from the robot** to @@ -352,8 +354,8 @@ def render(filter_name: str, oauth: gr.OAuthProfile | None) -> tuple[Any, str]: # A duck's camera, processed in a data centre The robot **dials this Space** and pushes JPEG frames; OpenCV runs over them here. Nothing - is on the robot's network, and no relay candidate is involved — which is why this works - where pulling the stream over WebRTC does not, while `turn.fastrtc.org` has no DNS. + is on the robot's network, and no relay candidate is involved — so no shared relay carries + the pixels, and none of this is spent against the robot owner's TURN allowance. Frames arrive at `{RECEIVER_URL}` — change it below if the robot cannot reach that. """ diff --git a/spaces/vision-demo/receiver.py b/spaces/vision-demo/receiver.py index c644751d..6cc5e850 100644 --- a/spaces/vision-demo/receiver.py +++ b/spaces/vision-demo/receiver.py @@ -2,11 +2,11 @@ **The robot connects to us, which is the whole reason this exists.** Pulling the camera through the rendezvous means WebRTC, and WebRTC between a robot behind a home router and a container in a -data centre needs a relay candidate — `turn.fastrtc.org` has no A record and its zone no NS -records (`remote-access-design.md` §6), so it negotiates and carries nothing. An outbound -WebSocket has no such problem: the robot already holds one to a Space every second it is -reachable. So the direction is inverted, NAT stops being a participant, and this is the socket at -the far end. +data centre needs a relay candidate to fall back on (`remote-access-design.md` §6) — somebody's +bandwidth, metered against the robot owner's Hugging Face allowance, for every frame of a stream +that is only going to be decoded by a program. An outbound WebSocket needs nobody's: the robot +already holds one to a Space every second it is reachable. So the direction is inverted, NAT +stops being a participant, and this is the socket at the far end. `mediad`'s `stream.rs` is the other half. It sends one text frame describing what is coming, then one binary message per JPEG, upright, at the rate it was asked for.