diff --git a/.github/workflows/_build-release.yml b/.github/workflows/_build-release.yml index 545eb51d..9645e5db 100644 --- a/.github/workflows/_build-release.yml +++ b/.github/workflows/_build-release.yml @@ -168,14 +168,14 @@ jobs: --include "robotd/systemd/robotd.service=systemd/robotd.service" \ --include "hooks/postinstall=hooks/postinstall" \ --include "scripts/setup-gstreamer.sh=scripts/setup-gstreamer.sh" \ - --include "duck-detect/models/duck_detect.rknn=models/duck_detect.rknn" \ - --include "duck-detect/models/duck_detect.onnx=models/duck_detect.onnx" \ --include "scripts/setup-npu.sh=scripts/setup-npu.sh" \ --include "deploy/overlays/rk3568-npu-enable.dts=deploy/overlays/rk3568-npu-enable.dts" \ --include "scripts/setup-rkaiq.sh=scripts/setup-rkaiq.sh" \ --include "scripts/rkaiq-modinfo-shim.c=scripts/rkaiq-modinfo-shim.c" \ --include "scripts/setup-login.sh=scripts/setup-login.sh" \ + --include "scripts/setup-quiet-boot.sh=scripts/setup-quiet-boot.sh" \ --include "scripts/seed-policies.sh=scripts/seed-policies.sh" \ + --include "scripts/seed-detector.sh=scripts/seed-detector.sh" \ --include "scripts/robot-rescue=scripts/robot-rescue" \ --include "scripts/robot-boot-check=scripts/robot-boot-check" \ --include "updater/systemd/robot-boot-check.service=systemd/robot-boot-check.service" \ diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9a78882c..270242ec 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -102,7 +102,7 @@ jobs: # a bashism would work on a dev box and fail on the board. - name: Lint the installer run: | - for script in scripts/install.sh scripts/setup-board.sh scripts/setup-gstreamer.sh scripts/setup-login.sh scripts/migrate-network.sh scripts/provision.sh scripts/provision-board.sh scripts/ci-release-notes.sh scripts/robot-rescue scripts/dev-push.sh scripts/pad-link-test.sh scripts/pad-stack-report.sh scripts/seed-policies.sh; do + for script in scripts/install.sh scripts/setup-board.sh scripts/setup-gstreamer.sh scripts/setup-login.sh scripts/setup-quiet-boot.sh scripts/migrate-network.sh scripts/provision.sh scripts/provision-board.sh scripts/ci-release-notes.sh scripts/robot-rescue scripts/dev-push.sh scripts/pad-link-test.sh scripts/pad-stack-report.sh scripts/seed-policies.sh scripts/seed-detector.sh; do sh -n "$script" shellcheck --shell=sh "$script" # The one-liner is only correct if the file is executable and self-contained. diff --git a/.github/workflows/dev.yml b/.github/workflows/dev.yml index da20d356..52b039d9 100644 --- a/.github/workflows/dev.yml +++ b/.github/workflows/dev.yml @@ -154,14 +154,14 @@ jobs: --include "robotd/systemd/robotd.service=systemd/robotd.service" \ --include "hooks/postinstall=hooks/postinstall" \ --include "scripts/setup-gstreamer.sh=scripts/setup-gstreamer.sh" \ - --include "duck-detect/models/duck_detect.rknn=models/duck_detect.rknn" \ - --include "duck-detect/models/duck_detect.onnx=models/duck_detect.onnx" \ --include "scripts/setup-npu.sh=scripts/setup-npu.sh" \ --include "deploy/overlays/rk3568-npu-enable.dts=deploy/overlays/rk3568-npu-enable.dts" \ --include "scripts/setup-rkaiq.sh=scripts/setup-rkaiq.sh" \ --include "scripts/rkaiq-modinfo-shim.c=scripts/rkaiq-modinfo-shim.c" \ --include "scripts/setup-login.sh=scripts/setup-login.sh" \ + --include "scripts/setup-quiet-boot.sh=scripts/setup-quiet-boot.sh" \ --include "scripts/seed-policies.sh=scripts/seed-policies.sh" \ + --include "scripts/seed-detector.sh=scripts/seed-detector.sh" \ --include "scripts/robot-rescue=scripts/robot-rescue" \ --include "scripts/robot-boot-check=scripts/robot-boot-check" \ --include "updater/systemd/robot-boot-check.service=systemd/robot-boot-check.service" \ diff --git a/.github/workflows/turn-endpoint.yml b/.github/workflows/turn-endpoint.yml new file mode 100644 index 00000000..9f805ac7 --- /dev/null +++ b/.github/workflows/turn-endpoint.yml @@ -0,0 +1,77 @@ +# Does the relay credentials endpoint still answer? +# +# `mediad::turn` fetches short-lived Cloudflare credentials so a robot can offer a `relay` +# candidate to a consumer that cannot reach it directly. The endpoint it used before +# 2026-09 had been dead since June — a dangling Route53 delegation — and **nothing +# noticed for three months**, because the only symptom is a warning in a journal nobody +# reads and a candidate type nobody counts. Every other check in this repository passes +# regardless: they pair two peers on one network, which never looks at a relay. +# +# So this is the one check that would have caught it, and it is deliberately *not* on +# `pull_request`. The failure being guarded against is "nobody touched this for months", +# which a PR trigger cannot see; and a third party's outage must never block work that has +# nothing to do with it. A scheduled failure mails whoever owns the repository, which is +# the right blast radius for "an upstream service went away". +name: turn-endpoint + +on: + schedule: + # Daily. The endpoint is somebody else's Space, and a week of relay coverage lost + # before anybody hears about it is most of the damage already done. + - cron: "41 5 * * *" + workflow_dispatch: + +jobs: + credentials: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + # **Read the URL out of the source rather than repeating it here.** A check with its + # own copy of the endpoint tests whatever it was last told, which can drift from what + # the daemon compiles in — and a green check on an endpoint no robot uses is worse + # than no check, because it reads as proof. + - name: Ask the endpoint the daemon uses + env: + HF_TOKEN: ${{ secrets.HF_TOKEN }} + run: | + set -euo pipefail + + url=$(sed -n 's/^pub const DEFAULT_TURN_ENDPOINT: &str = "\(.*\)";$/\1/p' \ + mediad/src/turn.rs) + [ -n "$url" ] || { + echo "::error::could not read DEFAULT_TURN_ENDPOINT from mediad/src/turn.rs" + exit 1 + } + echo "endpoint: $url" + + # No token means this cannot check anything. Failing is the point: a check that + # skips itself into permanent silence is the failure mode this workflow exists + # to end. + [ -n "${HF_TOKEN:-}" ] || { + echo "::error::no HF_TOKEN secret. Add a Hugging Face token with no scopes" \ + "beyond sign-in; it is used only to mint TURN credentials." + exit 1 + } + + code=$(curl -sS -o body.json -w '%{http_code}' --max-time 30 \ + -H "Authorization: Bearer $HF_TOKEN" "$url?ttl=600") + [ "$code" = "200" ] || { + echo "::error::$url answered HTTP $code, not 200" + head -c 400 body.json || true + exit 1 + } + + # A 200 carrying no relay is still a robot with no relay candidate to offer — + # which is the outage, not a milder version of it. `stun:` entries do not count: + # `webrtcbin` takes a STUN server through its own property, and this is about the + # thing STUN cannot do. + relays=$(jq '[.iceServers[]? | .urls] | flatten + | map(select(startswith("turn:") or startswith("turns:"))) + | length' body.json) + echo "relay servers offered: $relays" + [ "$relays" -gt 0 ] || { + echo "::error::$url answered 200 with no turn:/turns: entries" + head -c 400 body.json || true + exit 1 + } diff --git a/.gitignore b/.gitignore index 8bc02f5f..ad6ab663 100644 --- a/.gitignore +++ b/.gitignore @@ -20,3 +20,11 @@ secret* # Playground and scratch state (see updater/examples/playground.rs). /verify + +# Python bytecode from the demo Space, which is source rather than a build here. +__pycache__/ + +# A venv beside a Space's source, for running one locally. `uv venv` writes its own `.gitignore` +# holding `*` so it self-ignores; this is for the person who reaches for `python -m venv`, which +# does not, and it is the same accident `__pycache__` above was added for. +.venv/ diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 00000000..7ddaf1b1 --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,41 @@ +# Working in this repository + +`CONTRIBUTING.md` is the reference: building, testing, layout, conventions, releasing. This page +is the short list of things that are easy to get backwards, and where the answer lives when they +are not here. + +## Docs own mechanisms; one page each + +`docs/README.md` assigns every mechanism to one design doc. When a fact belongs to a page listed +there, every other page says one sentence and links. When two pages disagree, the one that does +not own the mechanism is the bug — and when behaviour and a design doc disagree, the doc is the +bug. [`docs/faq.md`](docs/faq.md) is the task-shaped front door for someone building against a +robot rather than changing it. + +## A consumer uses WebRTC. `media.stream` is the fallback + +The robot publishes H.264 over WebRTC, and that is the default for anything consuming a duck's +camera — it is encrypted end to end, it carries the control channel on the same session, and it +has a return path. It works from a data centre because the robot offers a relay candidate +(`remote-access-design.md` §6). + +`media.stream` — the robot dialling an outbound WebSocket and pushing frames to you — is the +fallback for a **program** consuming **frames only** on a **long-running** stream, where relay +metering is the thing that matters. It has no return path and no control channel. + +This is worth stating because the repository reads the other way round if you only follow the +code: `media.stream` was built when the relay endpoint was dead and WebRTC genuinely could not +connect from a data centre, so its module doc argues its own case at length. That endpoint is +fixed. Do not conclude from the volume of prose that it is the preferred path. + +## Never design around a version difference + +One user, one robot. An old component's limits are a question to raise, not something to route +around — bump `API_VERSION` and name the install consequence. A version skew is logged and served, +never refused; only a genuinely missing route or an unknown parameter may refuse. + +## Releases are how a fix reaches a robot + +`main` being fixed is not a robot being fixed. Robots on the stable channel move when a release is +cut, and a dev build from a branch is superseded by the next `daemon-dev-main` the board's +six-hourly check finds. `docs/design/updater-design.md` owns the mechanism. diff --git a/CLAUDE.md b/CLAUDE.md new file mode 120000 index 00000000..47dc3e3d --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1 @@ +AGENTS.md \ No newline at end of file diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 2546aa05..f1036e4a 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -100,6 +100,7 @@ hooks/ preinstall · postinstall — what runs inside an update, from t scripts/ provision-board.sh · dev-push.sh + dev-build.Dockerfile (from your machine) · provision.sh → setup-board.sh → setup-gstreamer.sh · setup-rkaiq.sh · migrate-network.sh · install.sh (on the board) · + setup-login.sh · setup-quiet-boot.sh (install.sh and postinstall both run these) · robot-boot-check · robot-rescue (recovery, installed to /usr/local/sbin) · pad-link-test.sh · pad-stack-report.sh (gamepad radio, on the board) · board-test.sh · systemd-test.sh (CI) · cross-sysroot.sh (cross-builds) · diff --git a/Cargo.lock b/Cargo.lock index f115c255..93bc8c5d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,6 +2,12 @@ # It is not intended for manual editing. version = 4 +[[package]] +name = "adler2" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" + [[package]] name = "aho-corasick" version = "1.1.4" @@ -11,6 +17,17 @@ dependencies = [ "memchr", ] +[[package]] +name = "ahrs" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4060d852f1a761d677c6f50499475735d8619f622a5d32b9fa8044e349827aef" +dependencies = [ + "nalgebra", + "num-traits", + "simba", +] + [[package]] name = "allocator-api2" version = "0.2.21" @@ -205,6 +222,28 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "async-stream" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b5a71a6f37880a80d1d7f19efd781e4b5de42c88f0722cc13bcb6cc2cfe8476" +dependencies = [ + "async-stream-impl", + "futures-core", + "pin-project-lite", +] + +[[package]] +name = "async-stream-impl" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7c24de15d275a1ecfd47a380fb4d5ec9bfe0933f309ed5e705b775596a3574d" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + [[package]] name = "async-task" version = "4.7.1" @@ -270,6 +309,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "31b698c5f9a010f6573133b09e0de5408834d0c82f8d7475a89fc1867a71cd90" dependencies = [ "axum-core", + "base64", "bytes", "form_urlencoded", "futures-util", @@ -288,8 +328,10 @@ dependencies = [ "serde_json", "serde_path_to_error", "serde_urlencoded", + "sha1 0.10.7", "sync_wrapper", "tokio", + "tokio-tungstenite 0.29.0", "tower", "tower-layer", "tower-service", @@ -443,9 +485,19 @@ dependencies = [ "dbus", ] +[[package]] +name = "bmi088" +version = "0.1.2" +source = "git+https://github.com/pollen-robotics/bmi088-rs?tag=v0.1.2#bd113f317b14923f50f9a12a06678fe162b7fe06" +dependencies = [ + "ahrs", + "embedded-hal", + "nalgebra", +] + [[package]] name = "btd" -version = "0.10.0" +version = "0.12.0" dependencies = [ "bluer", "clap", @@ -506,6 +558,12 @@ version = "1.25.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "95832e849adfb21180ccb6826a99da14e5d266ae5c2e668e1602cf234f153797" +[[package]] +name = "byteorder" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" + [[package]] name = "byteorder-lite" version = "0.1.0" @@ -518,6 +576,12 @@ version = "1.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04" +[[package]] +name = "cast" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" + [[package]] name = "castaway" version = "0.2.4" @@ -575,7 +639,7 @@ checksum = "d524456ba66e72eb8b115ff89e01e497f8e6d11d78b70b1aa13c0fbd97540a81" dependencies = [ "cfg-if", "cpufeatures 0.3.0", - "rand_core", + "rand_core 0.10.1", ] [[package]] @@ -687,7 +751,7 @@ dependencies = [ [[package]] name = "configd" -version = "0.10.0" +version = "0.12.0" dependencies = [ "async-trait", "clap", @@ -753,6 +817,15 @@ dependencies = [ "libc", ] +[[package]] +name = "crc32fast" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8498c871161e1742aaa9d52551b2d6ebdd4c3d45a3be423e3728f33b955be550" +dependencies = [ + "cfg-if", +] + [[package]] name = "critical-section" version = "1.2.0" @@ -935,6 +1008,12 @@ dependencies = [ "parking_lot_core", ] +[[package]] +name = "data-encoding" +version = "2.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4583a4551df46e2792f82ceeac45e850d2e2d5debba0b91f102385cda5b11f06" + [[package]] name = "dbus" version = "0.9.12" @@ -1050,21 +1129,23 @@ dependencies = [ [[package]] name = "duck-control" -version = "0.10.0" +version = "0.12.0" dependencies = [ "duck-ipc-proto", "libloading 0.8.9", "ort", "rustypot", "serde", + "serde_json", "serialport", + "sha2 0.11.0", "thiserror 2.0.19", "tracing", ] [[package]] name = "duck-detect" -version = "0.10.0" +version = "0.12.0" dependencies = [ "anyhow", "clap", @@ -1075,10 +1156,24 @@ dependencies = [ "tracing-subscriber", ] +[[package]] +name = "duck-ether" +version = "0.12.0" +dependencies = [ + "clap", + "duck-ipc-proto", + "serde_json", + "thiserror 2.0.19", + "tokio", + "tracing", + "tracing-subscriber", +] + [[package]] name = "duck-ipc-proto" -version = "0.10.0" +version = "0.12.0" dependencies = [ + "libc", "semver", "serde", "serde_json", @@ -1087,7 +1182,7 @@ dependencies = [ [[package]] name = "duckctl" -version = "0.10.0" +version = "0.12.0" dependencies = [ "btd", "btleplug", @@ -1111,6 +1206,22 @@ version = "1.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9e5e8f6c15a24b9a3ee5efec809ccd006d3b30e8b3bb63c39af737c7f87daa1d" +[[package]] +name = "embedded-hal" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "361a90feb7004eca4019fb28352a9465666b24f840f5c3cddf0ff13920590b89" + +[[package]] +name = "embedded-hal-nb" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fba4268c14288c828995299e59b12babdbe170f6c6d73731af1b4648142e8605" +dependencies = [ + "embedded-hal", + "nb", +] + [[package]] name = "encoding_rs" version = "0.8.35" @@ -1208,12 +1319,32 @@ dependencies = [ "pin-project-lite", ] +[[package]] +name = "eventsource-stream" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "74fef4569247a5f429d9156b9d0a2599914385dd189c539334c625d8099d90ab" +dependencies = [ + "futures-core", + "nom", + "pin-project-lite", +] + [[package]] name = "fastrand" version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "da7c62ceae207dd37ea5b845da6a0696c799f85e97da1ab5b7910be3c1c80223" +[[package]] +name = "fdeflate" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e6853b52649d4ac5c0bd02320cddc5ba956bdb407c4b75a2c6b75bf51500f8c" +dependencies = [ + "simd-adler32", +] + [[package]] name = "filetime" version = "0.2.29" @@ -1230,6 +1361,17 @@ version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" +[[package]] +name = "flate2" +version = "1.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e634e2e0ebac1ee034020da1ca582e17ffe4e0f5e985823721e168928136dcb" +dependencies = [ + "crc32fast", + "miniz_oxide 0.9.1", + "zlib-rs", +] + [[package]] name = "fnv" version = "1.0.7" @@ -1397,6 +1539,18 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "getrandom" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" +dependencies = [ + "cfg-if", + "libc", + "r-efi 5.3.0", + "wasip2", +] + [[package]] name = "getrandom" version = "0.4.3" @@ -1406,8 +1560,8 @@ dependencies = [ "cfg-if", "js-sys", "libc", - "r-efi", - "rand_core", + "r-efi 6.0.0", + "rand_core 0.10.1", "wasm-bindgen", ] @@ -1458,6 +1612,114 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "glam" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "333928d5eb103c5d4050533cec0384302db6be8ef7d3cebd30ec6a35350353da" + +[[package]] +name = "glam" +version = "0.15.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3abb554f8ee44336b72d522e0a7fe86a29e09f839a36022fa869a7dfe941a54b" + +[[package]] +name = "glam" +version = "0.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4126c0479ccf7e8664c36a2d719f5f2c140fbb4f9090008098d2c291fa5b3f16" + +[[package]] +name = "glam" +version = "0.17.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e01732b97afd8508eee3333a541b9f7610f454bb818669e66e90f5f57c93a776" + +[[package]] +name = "glam" +version = "0.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "525a3e490ba77b8e326fb67d4b44b4bd2f920f44d4cc73ccec50adc68e3bee34" + +[[package]] +name = "glam" +version = "0.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b8509e6791516e81c1a630d0bd7fbac36d2fa8712a9da8662e716b52d5051ca" + +[[package]] +name = "glam" +version = "0.20.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f43e957e744be03f5801a55472f593d43fabdebf25a4585db250f04d86b1675f" + +[[package]] +name = "glam" +version = "0.21.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "518faa5064866338b013ff9b2350dc318e14cc4fcd6cb8206d7e7c9886c98815" + +[[package]] +name = "glam" +version = "0.22.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "12f597d56c1bd55a811a1be189459e8fad2bbc272616375602443bdfb37fa774" + +[[package]] +name = "glam" +version = "0.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e4afd9ad95555081e109fe1d21f2a30c691b5f0919c67dfa690a2e1eb6bd51c" + +[[package]] +name = "glam" +version = "0.24.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5418c17512bdf42730f9032c74e1ae39afc408745ebb2acf72fbc4691c17945" + +[[package]] +name = "glam" +version = "0.25.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "151665d9be52f9bb40fc7966565d39666f2d1e69233571b71b87791c7e0528b3" + +[[package]] +name = "glam" +version = "0.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e05e7e6723e3455f4818c7b26e855439f7546cf617ef669d1adedb8669e5cb9" + +[[package]] +name = "glam" +version = "0.28.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "779ae4bf7e8421cf91c0b3b64e7e8b40b862fba4d393f59150042de7c4965a94" + +[[package]] +name = "glam" +version = "0.29.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8babf46d4c1c9d92deac9f7be466f76dfc4482b6452fc5024b5e8daf6ffeb3ee" + +[[package]] +name = "glam" +version = "0.30.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19fc433e8437a212d1b6f1e68c7824af3aed907da60afa994e7f542d18d12aa9" + +[[package]] +name = "glam" +version = "0.31.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "556f6b2ea90b8d15a74e0e7bb41671c9bdf38cd9f78c284d750b9ce58a2b5be7" + +[[package]] +name = "glam" +version = "0.32.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f70749695b063ecbf6b62949ccccde2e733ec3ecbbd71d467dca4e5c6c97cca0" + [[package]] name = "glib" version = "0.21.5" @@ -1513,6 +1775,17 @@ dependencies = [ "system-deps", ] +[[package]] +name = "gpio-cdev" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09831ec59b80be69e75d29cf36e16afbbe5fd1af9c1bf4689ad91c77db5aa6a6" +dependencies = [ + "bitflags 2.13.1", + "libc", + "nix 0.27.1", +] + [[package]] name = "gstreamer" version = "0.24.5" @@ -1750,6 +2023,21 @@ version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" +[[package]] +name = "hf-robot-account" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8f937783a058985f73be1663ef122f19682723c7504ff32ea10879c1ce9bef4" +dependencies = [ + "libc", + "reqwest", + "serde", + "serde_json", + "thiserror 2.0.19", + "tokio", + "tracing", +] + [[package]] name = "hmac" version = "0.12.1" @@ -1895,6 +2183,18 @@ dependencies = [ "tracing", ] +[[package]] +name = "i2cdev" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b940f7497c4f95b863b21cd34c3737b53a67d80d94cf29055d7f7eeca6ffdb4" +dependencies = [ + "bitflags 2.13.1", + "byteorder", + "libc", + "nix 0.26.4", +] + [[package]] name = "icu_collections" version = "2.2.0" @@ -2014,6 +2314,7 @@ dependencies = [ "byteorder-lite", "moxcms", "num-traits", + "png", "zune-core", "zune-jpeg", ] @@ -2248,7 +2549,7 @@ dependencies = [ [[package]] name = "kinematics" -version = "0.10.0" +version = "0.12.0" dependencies = [ "roxmltree", "serde", @@ -2332,6 +2633,24 @@ dependencies = [ "bitflags 2.13.1", ] +[[package]] +name = "linux-embedded-hal" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a8a605c95f708c78554738a12153b213f107d3bd5323f7ce32d6deb3faafb40" +dependencies = [ + "cast", + "embedded-hal", + "embedded-hal-nb", + "gpio-cdev", + "i2cdev", + "nb", + "nix 0.27.1", + "serialport", + "spidev", + "sysfs_gpio", +] + [[package]] name = "linux-raw-sys" version = "0.12.1" @@ -2422,24 +2741,35 @@ dependencies = [ [[package]] name = "mediad" -version = "0.10.0" +version = "0.12.0" dependencies = [ "anyhow", + "async-stream", "axum", "clap", "duck-detect", "duck-ipc-proto", + "eventsource-stream", + "futures-util", "glib", "gstreamer", "gstreamer-app", "gstreamer-video", "gstreamer-webrtc", + "hf-robot-account", + "image", + "libc", + "rand 0.10.2", + "reqwest", "robotd-params", + "serde", "serde_json", "tempfile", "tokio", + "tokio-tungstenite 0.30.0", "tracing", "tracing-subscriber", + "url", ] [[package]] @@ -2448,6 +2778,24 @@ version = "2.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" +[[package]] +name = "memoffset" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5aa361d4faea93603064a027415f07bd8e1d5c88c9fbf68bf56a285428fd79ce" +dependencies = [ + "autocfg", +] + +[[package]] +name = "memoffset" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5de893c32cde5f383baa4c04c5d6dbdd735cfd4a794b0debdb2bb1b421da5ff4" +dependencies = [ + "autocfg", +] + [[package]] name = "memoffset" version = "0.9.1" @@ -2463,6 +2811,12 @@ version = "0.3.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" +[[package]] +name = "minimal-lexical" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" + [[package]] name = "minisign" version = "0.9.1" @@ -2481,6 +2835,26 @@ version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "22f9645cb765ea72b8111f36c522475d2daa0d22c957a9826437e97534bc4e9e" +[[package]] +name = "miniz_oxide" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316" +dependencies = [ + "adler2", + "simd-adler32", +] + +[[package]] +name = "miniz_oxide" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b63fbc4a50860e98e7b2aa7804ded1db5cbc3aff9193adaff57a6931bf7c4b4c" +dependencies = [ + "adler2", + "simd-adler32", +] + [[package]] name = "mio" version = "1.2.2" @@ -2509,6 +2883,57 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "956787520e75e9bd233246045d19f42fb73242759cc57fba9611d940ae96d4b0" +[[package]] +name = "nalgebra" +version = "0.34.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df76ea0ff5c7e6b88689085804d6132ded0ddb9de5ca5b8aeb9eeadc0508a70a" +dependencies = [ + "approx", + "glam 0.14.0", + "glam 0.15.2", + "glam 0.16.0", + "glam 0.17.3", + "glam 0.18.0", + "glam 0.19.0", + "glam 0.20.5", + "glam 0.21.3", + "glam 0.22.0", + "glam 0.23.0", + "glam 0.24.2", + "glam 0.25.0", + "glam 0.27.0", + "glam 0.28.0", + "glam 0.29.3", + "glam 0.30.10", + "glam 0.31.1", + "glam 0.32.1", + "matrixmultiply", + "nalgebra-macros", + "num-complex", + "num-rational", + "num-traits", + "simba", + "typenum", +] + +[[package]] +name = "nalgebra-macros" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "973e7178a678cfd059ccec50887658d482ce16b0aa9da3888ddeab5cd5eb4889" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "nb" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8d5439c4ad607c3c23abf66de8c8bf57ba8adcd1f129e699851a6e43935d339d" + [[package]] name = "ndarray" version = "0.17.2" @@ -2530,6 +2955,19 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "27b02d87554356db9e9a873add8782d4ea6e3e58ea071a9adb9a2e8ddb884a8b" +[[package]] +name = "nix" +version = "0.23.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f3790c00a0150112de0f4cd161e3d7fc4b2d8a5542ffc35f099a2562aecb35c" +dependencies = [ + "bitflags 1.3.2", + "cc", + "cfg-if", + "libc", + "memoffset 0.6.5", +] + [[package]] name = "nix" version = "0.26.4" @@ -2539,6 +2977,19 @@ dependencies = [ "bitflags 1.3.2", "cfg-if", "libc", + "memoffset 0.7.1", + "pin-utils", +] + +[[package]] +name = "nix" +version = "0.27.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2eb04e9c688eff1c89d72b407f168cf79bb9e867a9d3323ed6c01519eb9cc053" +dependencies = [ + "bitflags 2.13.1", + "cfg-if", + "libc", ] [[package]] @@ -2565,6 +3016,16 @@ dependencies = [ "libc", ] +[[package]] +name = "nom" +version = "7.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" +dependencies = [ + "memchr", + "minimal-lexical", +] + [[package]] name = "nu-ansi-term" version = "0.50.3" @@ -2574,6 +3035,16 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "num-bigint" +version = "0.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c89e69e7e0f03bea5ef08013795c25018e101932225a656383bd384495ecc367" +dependencies = [ + "num-integer", + "num-traits", +] + [[package]] name = "num-complex" version = "0.4.6" @@ -2615,6 +3086,7 @@ version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f83d14da390562dca69fc84082e73e548e1ad308d24accdedd2720017cb37824" dependencies = [ + "num-bigint", "num-integer", "num-traits", ] @@ -2759,7 +3231,7 @@ dependencies = [ [[package]] name = "odometry" -version = "0.10.0" +version = "0.12.0" dependencies = [ "duck-ipc-proto", "kinematics", @@ -2821,18 +3293,27 @@ version = "2.0.0-rc.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "06503bb33f294c5f1ba484011e053bfa6ae227074bdb841e9863492dc5960d4b" +[[package]] +name = "pad-imu" +version = "0.12.0" +dependencies = [ + "duck-ipc-proto", +] + [[package]] name = "padd" -version = "0.10.0" +version = "0.12.0" dependencies = [ "clap", "duck-ipc-proto", "evdev", "gilrs", "libc", + "pad-imu", "robotd-params", "serde", "serde_json", + "tempfile", "tracing", "tracing-subscriber", ] @@ -2929,7 +3410,7 @@ checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" [[package]] name = "pet-detect" -version = "0.10.0" +version = "0.12.0" dependencies = [ "anyhow", "clap", @@ -2965,6 +3446,12 @@ version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" +[[package]] +name = "pin-utils" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" + [[package]] name = "piper" version = "0.2.5" @@ -2982,6 +3469,19 @@ version = "0.3.33" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "19f132c84eca552bf34cab8ec81f1c1dcc229b811638f9d283dceabe58c5569e" +[[package]] +name = "png" +version = "0.18.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60769b8b31b2a9f263dae2776c37b1b28ae246943cf719eb6946a1db05128a61" +dependencies = [ + "bitflags 2.13.1", + "crc32fast", + "fdeflate", + "flate2", + "miniz_oxide 0.8.9", +] + [[package]] name = "polling" version = "3.11.0" @@ -3026,6 +3526,15 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" +[[package]] +name = "ppv-lite86" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" +dependencies = [ + "zerocopy", +] + [[package]] name = "primal-check" version = "0.3.4" @@ -3089,7 +3598,7 @@ dependencies = [ "bytes", "getrandom 0.4.3", "lru-slab", - "rand", + "rand 0.10.2", "rand_pcg", "ring", "rustc-hash", @@ -3125,6 +3634,12 @@ dependencies = [ "proc-macro2", ] +[[package]] +name = "r-efi" +version = "5.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" + [[package]] name = "r-efi" version = "6.0.0" @@ -3137,6 +3652,16 @@ version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dc33ff2d4973d518d823d61aa239014831e521c75da58e3df4840d3f47749d09" +[[package]] +name = "rand" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9ef1d0d795eb7d84685bca4f72f3649f064e6641543d3a8c415898726a57b41" +dependencies = [ + "rand_chacha", + "rand_core 0.9.5", +] + [[package]] name = "rand" version = "0.10.2" @@ -3145,7 +3670,26 @@ checksum = "c7f5fa3a058cd35567ef9bfa5e75732bee0f9e4c55fa90477bef2dfcdbc4be80" dependencies = [ "chacha20", "getrandom 0.4.3", - "rand_core", + "rand_core 0.10.1", +] + +[[package]] +name = "rand_chacha" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" +dependencies = [ + "ppv-lite86", + "rand_core 0.9.5", +] + +[[package]] +name = "rand_core" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76afc826de14238e6e8c374ddcc1fa19e374fd8dd986b0d2af0d02377261d83c" +dependencies = [ + "getrandom 0.3.4", ] [[package]] @@ -3160,7 +3704,7 @@ version = "0.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "caa0f4137e1c0a72f4c651489402276c8e8e1cf081f3b0ba156d2cbeef09e86a" dependencies = [ - "rand_core", + "rand_core 0.10.1", ] [[package]] @@ -3334,13 +3878,14 @@ dependencies = [ [[package]] name = "robotctl" -version = "0.10.0" +version = "0.12.0" dependencies = [ "clap", "clap_complete", "duck-ipc-proto", "kinematics", "libc", + "pad-imu", "ratatui", "robotd-params", "semver", @@ -3353,7 +3898,7 @@ dependencies = [ [[package]] name = "robotd" -version = "0.10.0" +version = "0.12.0" dependencies = [ "arc-swap", "clap", @@ -3384,7 +3929,7 @@ dependencies = [ [[package]] name = "robotd-params" -version = "0.10.0" +version = "0.12.0" dependencies = [ "duck-ipc-proto", "kinematics", @@ -3572,6 +4117,15 @@ version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" +[[package]] +name = "safe_arch" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96b02de82ddbe1b636e6170c21be622223aea188ef2e139be0a5b219ec215323" +dependencies = [ + "bytemuck", +] + [[package]] name = "salsa20" version = "0.10.2" @@ -3765,6 +4319,28 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "sha1" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a978451301f4db1d02937a4ab3ccce137717b81826e79b7d49ffe3244a13c3b8" +dependencies = [ + "cfg-if", + "cpufeatures 0.2.17", + "digest 0.10.7", +] + +[[package]] +name = "sha1" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aacc4cc499359472b4abe1bf11d0b12e688af9a805fa5e3016f9a386dc2d0214" +dependencies = [ + "cfg-if", + "cpufeatures 0.3.0", + "digest 0.11.3", +] + [[package]] name = "sha2" version = "0.10.9" @@ -3833,6 +4409,26 @@ dependencies = [ "libc", ] +[[package]] +name = "simba" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c99284beb21666094ba2b75bbceda012e610f5479dfcc2d6e2426f53197ffd95" +dependencies = [ + "approx", + "libm", + "num-complex", + "num-traits", + "paste", + "wide", +] + +[[package]] +name = "simd-adler32" +version = "0.3.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a219298ac11a56ea9a6d2120044824d6f01aeb034955e7af7bc16858527deea" + [[package]] name = "simd_cesu8" version = "1.2.0" @@ -3873,7 +4469,7 @@ dependencies = [ [[package]] name = "sounds" -version = "0.10.0" +version = "0.12.0" dependencies = [ "anyhow", "clap", @@ -3881,6 +4477,17 @@ dependencies = [ "sha2 0.11.0", ] +[[package]] +name = "spidev" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32dadd0a877f0652fa52dbc4d2ed9f4877bea5cd30725507b36e1970a5ef0519" +dependencies = [ + "bitflags 2.13.1", + "libc", + "nix 0.26.4", +] + [[package]] name = "stable_deref_trait" version = "1.2.1" @@ -3996,6 +4603,15 @@ dependencies = [ "syn 2.0.119", ] +[[package]] +name = "sysfs_gpio" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8808c55bc926565c62ef7838bcaa8add51585236803e2bdfa1472e3a3ab5e17" +dependencies = [ + "nix 0.23.2", +] + [[package]] name = "system-deps" version = "7.0.8" @@ -4056,7 +4672,7 @@ dependencies = [ [[package]] name = "test-support" -version = "0.10.0" +version = "0.12.0" dependencies = [ "clap", "minisign", @@ -4165,15 +4781,19 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tof" -version = "0.10.0" +version = "0.12.0" dependencies = [ "anyhow", + "bmi088", "cc", "clap", "duck-ipc-proto", "libc", + "linux-embedded-hal", + "robotd-params", "serde", "serde_json", + "tempfile", "tokio", "tracing", "tracing-subscriber", @@ -4228,6 +4848,34 @@ dependencies = [ "tokio-util", ] +[[package]] +name = "tokio-tungstenite" +version = "0.29.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f72a05e828585856dacd553fba484c242c46e391fb0e58917c942ee9202915c" +dependencies = [ + "futures-util", + "log", + "tokio", + "tungstenite 0.29.0", +] + +[[package]] +name = "tokio-tungstenite" +version = "0.30.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "17a073bfed563fa236697a068031408a93cd9522e08abf9933ead3e73411bd71" +dependencies = [ + "futures-util", + "log", + "rustls", + "rustls-pki-types", + "tokio", + "tokio-rustls", + "tungstenite 0.30.0", + "webpki-roots 0.26.11", +] + [[package]] name = "tokio-util" version = "0.7.19" @@ -4454,6 +5102,40 @@ version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" +[[package]] +name = "tungstenite" +version = "0.29.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c01152af293afb9c7c2a57e4b559c5620b421f6d133261c60dd2d0cdb38e6b8" +dependencies = [ + "bytes", + "data-encoding", + "http", + "httparse", + "log", + "rand 0.9.5", + "sha1 0.10.7", + "thiserror 2.0.19", +] + +[[package]] +name = "tungstenite" +version = "0.30.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e48ac77174b19c110a50ab2128b24215ac9cb40e0e12e093fb602d175c569d22" +dependencies = [ + "bytes", + "data-encoding", + "http", + "httparse", + "log", + "rand 0.10.2", + "rustls", + "rustls-pki-types", + "sha1 0.11.0", + "thiserror 2.0.19", +] + [[package]] name = "typenum" version = "1.20.1" @@ -4466,7 +5148,7 @@ version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f2f6fb2847f6742cd76af783a2a2c49e9375d0a111c7bef6f71cd9e738c72d6e" dependencies = [ - "memoffset", + "memoffset 0.9.1", "tempfile", "windows-sys 0.61.2", ] @@ -4517,7 +5199,7 @@ checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" [[package]] name = "updater" -version = "0.10.0" +version = "0.12.0" dependencies = [ "async-trait", "axum", @@ -4525,6 +5207,7 @@ dependencies = [ "duck-ipc-proto", "fs4", "futures-util", + "hf-robot-account", "humantime-serde", "libc", "minisign", @@ -4631,6 +5314,15 @@ version = "0.11.1+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" +[[package]] +name = "wasip2" +version = "1.0.4+wasi-0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b67efb37e106e55ce722a510d6b5f9c17f083e5fc79afc2badeb12cc313d9487" +dependencies = [ + "wit-bindgen", +] + [[package]] name = "wasm-bindgen" version = "0.2.126" @@ -4744,6 +5436,34 @@ dependencies = [ "rustls-pki-types", ] +[[package]] +name = "webpki-roots" +version = "0.26.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "521bc38abb08001b01866da9f51eb7c5d647a19260e00054a8c7fd5f9e57f7a9" +dependencies = [ + "webpki-roots 1.0.8", +] + +[[package]] +name = "webpki-roots" +version = "1.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf85cb06032201fa7c6f829d7db5a7e5aa45bcc0655327713065f6f0576731bf" +dependencies = [ + "rustls-pki-types", +] + +[[package]] +name = "wide" +version = "0.7.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ce5da8ecb62bcd8ec8b7ea19f69a51275e91299be594ea5cc6ef7819e16cd03" +dependencies = [ + "bytemuck", + "safe_arch", +] + [[package]] name = "winapi" version = "0.3.9" @@ -5083,6 +5803,12 @@ dependencies = [ "memchr", ] +[[package]] +name = "wit-bindgen" +version = "0.57.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" + [[package]] name = "writeable" version = "0.6.3" @@ -5116,7 +5842,7 @@ checksum = "636f85e5ca6488e96401b61eb7de54f4e44755c988af0f52cf90230c312a1a89" [[package]] name = "xtask" -version = "0.10.0" +version = "0.12.0" dependencies = [ "clap", "minisign", @@ -5215,6 +5941,26 @@ dependencies = [ "zvariant", ] +[[package]] +name = "zerocopy" +version = "0.8.56" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "556764e583adb45a9f8d413c2a147fa7e8d821e48e12b14fd560b607998b75eb" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.56" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2ab42fc20575779bd240faa45f94a74256f755c0fa9e89f0ede20d91d0cdfc1" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + [[package]] name = "zerofrom" version = "0.1.8" @@ -5275,6 +6021,12 @@ dependencies = [ "syn 2.0.119", ] +[[package]] +name = "zlib-rs" +version = "0.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34b31d188d9d685a4f9c7b46d6e36631b07058d2cfe190267adce54dc230bf12" + [[package]] name = "zmij" version = "1.0.23" diff --git a/Cargo.toml b/Cargo.toml index a2923286..d2a050ab 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -4,7 +4,7 @@ # docs/design/architecture.md §1. `robotd`, `btd` and `mediad` are all siblings now. [workspace] resolver = "3" -members = ["btd", "configd", "duck-control", "duck-detect", "duck-ipc-proto", "duckctl", "kinematics", "mediad", "odometry", "padd", "pet-detect", "sounds", "tof", "updater", "robotctl", "robotd", "robotd-params", "test-support", "xtask"] +members = ["btd", "configd", "duck-control", "duck-detect", "duck-ether", "duck-ipc-proto", "duckctl", "kinematics", "mediad", "odometry", "pad-imu", "padd", "pet-detect", "sounds", "tof", "updater", "robotctl", "robotd", "robotd-params", "test-support", "xtask"] # Everything except `duckctl`, and that exception is the whole reason this key exists. # @@ -18,7 +18,7 @@ members = ["btd", "configd", "duck-control", "duck-detect", "duck-ipc-proto", "d # writing down. One list here, and a new daemon is picked up by both without anybody remembering. # # `--workspace` is unaffected, so CI still lints and tests `duckctl` exactly as before. -default-members = ["btd", "configd", "duck-control", "duck-detect", "duck-ipc-proto", "kinematics", "mediad", "odometry", "padd", "pet-detect", "sounds", "tof", "updater", "robotctl", "robotd", "robotd-params", "test-support", "xtask"] +default-members = ["btd", "configd", "duck-control", "duck-detect", "duck-ipc-proto", "kinematics", "mediad", "odometry", "pad-imu", "padd", "pet-detect", "sounds", "tof", "updater", "robotctl", "robotd", "robotd-params", "test-support", "xtask"] # ONNX Runtime, which robotd dlopens to run a policy. One source of truth: `xtask package` # bakes these into the release's preinstall hook, and a test asserts scripts/setup-board.sh @@ -49,12 +49,23 @@ runtime = "v2.3.2" # inside a release and cannot read this file, so it carries the literals and a test asserts the # two agree. # -# `version` is a tag in the Hub repo, and it is a *floor*: it decides what a freshly provisioned -# board installs, not what every board runs. Moving past it is `robotctl policy update`, which -# needs no daemon release — bumping this does, since it ships inside one. +# `version` is a tag in the Hub repo, and it is a *minimum*: what a freshly provisioned board +# installs, and the oldest official set this daemon's defaults load — a board below it is moved +# up to it by the post-install hook, one past it is left alone. Moving past it is `robotctl +# policy update`, which needs no daemon release — bumping this does, since it ships inside one, +# and it is bumped when a slot's default names a file only the new set carries. [workspace.metadata.policies] repo = "pollen-robotics/microduck-policies" -version = "v1" +version = "v5" + +# The duck detector, the same way: trained in pollen-robotics/duck_detector, published on the Hub, +# and seeded onto a board by `scripts/seed-detector.sh` from this pin — a floor, moved past with +# `robotctl duck-detector update`. The model repo shares its name with the *dataset* repo; the robot only +# ever addresses the model (`huggingface.co//resolve/…`, `api/models/`), and the +# dataset lives under `datasets/`, so the shared name cannot be confused on the wire. +[workspace.metadata.detector] +repo = "pollen-robotics/microduck-duck-detector" +version = "duck-v1" # The prebuilt GStreamer plugins `mediad` needs, built in CI from pinned upstream sources at # https://github.com/pollen-robotics/microduck-gst-plugins — `mpph264enc` (hardware H.264 through @@ -69,7 +80,7 @@ repo = "pollen-robotics/microduck-gst-plugins" version = "v3" [workspace.package] -version = "0.10.0" +version = "0.12.0" edition = "2024" # 1.89 for `std::fs::File::try_lock`, which is how the single-flight update lock works # without a dependency (updater/src/journal.rs). Edition 2024 needs 1.85, so this covers @@ -90,6 +101,7 @@ tracing = "0.1" tracing-subscriber = "0.3" tokio-util = "0.7" async-trait = "0.1" +url = "2" # No custom [profile.release]. Binary size is not worth optimising for: model artifacts # will dwarf a few MB of binary, and the tuning cost more than it bought — diff --git a/README.md b/README.md index d5dae0ad..2aef6df3 100644 --- a/README.md +++ b/README.md @@ -85,6 +85,7 @@ It also sits, kicks a ball, rolls forward on command, and quacks in a voice that | [Set up a dev board](docs/robot/install-dev.md) | From a blank board to a robot that takes branch builds. | | [Dev cheat sheet](docs/robot/cheatsheet-dev.md) | Branch builds, release candidates, driving from a laptop, and the restart traps after an update. | | [Push your branch](docs/robot/dev-push.md) | Build on your machine, install over ssh, about a minute. | +| [The simulated duck](docs/robot/simulation.md) | No robot on the desk? `scripts/duck-sim` runs the real daemons against a body in MuJoCo — one duck in a window, or four as machines you log into. | | [CONTRIBUTING.md](CONTRIBUTING.md) | Building, testing, layout, conventions, releasing. | | [Docs index](docs/README.md) | Everything, including the design pages and the open problems. | diff --git a/btd/src/bluez.rs b/btd/src/bluez.rs index 900be631..86742986 100644 --- a/btd/src/bluez.rs +++ b/btd/src/bluez.rs @@ -20,8 +20,13 @@ //! So: one session for the service's lifetime, one notify pump, and a write callback that pushes //! bytes into it. //! -//! **Untested against hardware.** It type-checks for aarch64 and has never met a real central. -//! Treat what follows as intent until someone connects a phone. +//! **What the callback model costs is flow control**, and that bill came due the first time a +//! reply was more than a few kilobytes. `notify` hands a D-Bus signal to the connection and +//! returns; nothing here can ask BlueZ whether the radio has caught up, and nothing reports the +//! notification MTU either. Both gaps are worked around rather than solved: the payload is taken +//! from what BlueZ reports on inbound writes (one ATT MTU serves both directions), and the pump +//! pauses every [`NOTIFY_BURST`] chunks. The IO model has the readiness signal this wants and +//! still cannot be used, for the reason above — it serves only the `Acquire*` paths. use std::net::Ipv4Addr; use std::sync::Arc; @@ -34,11 +39,13 @@ use bluer::agent::Agent; // which is how it first presented. use bluer::gatt::local::ReqError as GattError; use bluer::gatt::local::{ - Application, Characteristic, CharacteristicNotify, CharacteristicNotifyMethod, - CharacteristicRead, CharacteristicWrite, CharacteristicWriteMethod, Service, + Application, Characteristic, CharacteristicNotifier, CharacteristicNotify, + CharacteristicNotifyMethod, CharacteristicRead, CharacteristicWrite, CharacteristicWriteMethod, + Service, }; use futures::FutureExt; use std::sync::Mutex as StdMutex; +use std::sync::atomic::{AtomicUsize, Ordering}; use tokio::sync::mpsc; @@ -47,13 +54,46 @@ use crate::link::Link; use crate::session; use crate::upstream::{NameChoice, Sockets}; -/// Notification payload assumed for outbound chunks. +/// Notification payload a session starts with, before any write has reported the negotiated one. /// -/// The write side learns the negotiated MTU (BlueZ reports it per request); the notify side has no -/// way to ask. So chunks are sized for 20 bytes — the payload every BLE link is required to -/// support — which is slower than necessary on a good link and correct on every link. +/// 20 bytes is what every BLE link is required to support, so it is the only safe *first* guess. +/// It used to be the guess for the whole session — the notify side has no way to ask BlueZ, which +/// remains true — and that was wrong in two ways. Tenfold more notifications than the link needed +/// was the visible half; the other half is that a reply above roughly 5 KiB tore the session down +/// (see [`NOTIFY_BURST`]). The write side does learn the real MTU, BlueZ reports it on every +/// inbound write, and both directions share one ATT MTU — so the floor now lasts until the +/// client's first write, which is always `system.authenticate`. const FLOOR_MTU: usize = 20; +/// How many notifications to queue before pausing to let the radio drain. +/// +/// **This is the only backpressure the callback model offers, and it has to exist.** +/// `CharacteristicNotifier::notify` emits a D-Bus `PropertiesChanged` signal and returns as soon +/// as the signal is queued on the connection — it does not wait for BlueZ, let alone for the +/// controller — so an unpaced pump queues a whole reply in microseconds. Measured on the board: +/// a ~5 KiB reply at the 20-byte floor (≈265 notifications) got through in 1.8 s, and a ~7 KiB one +/// (≈350) killed the notification session 150 ms in, leaving the client waiting out its idle +/// timeout against a robot that had already torn down the session. `bluer`'s IO model has a real +/// readiness signal for this and cannot be used here — it serves only the `Acquire*` fd paths, +/// which a CoreBluetooth central never drives (see this module's header). +/// +/// Sixteen chunks is what a connection interval can plausibly carry, so a small reply — an +/// authentication answer is four chunks — never pauses at all. +const NOTIFY_BURST: usize = 16; + +/// How long to pause between bursts. Roughly one connection interval. +const NOTIFY_PAUSE: Duration = Duration::from_millis(20); + +/// How many times to re-send a chunk the D-Bus connection would not take, and how long to wait +/// between attempts. +/// +/// A `notify` that fails while the session is *not* stopped is a queue that is momentarily full, +/// which is recoverable and was being treated as the central having left — the session was torn +/// down mid-reply and the client learned nothing at all. Only an exhausted budget, or a genuinely +/// stopped session, ends the session now. +const NOTIFY_RETRIES: u32 = 5; +const NOTIFY_RETRY_BACKOFF: Duration = Duration::from_millis(20); + /// How often to advertise, and this is the difference between a robot that is found and one that is /// not. /// @@ -313,6 +353,13 @@ async fn serve_on_an_adapter( let for_write = current.clone(); let for_notify = current.clone(); + // The negotiated payload, written by the write callback and read by the session when it sizes + // a reply. An atomic rather than a channel or the mutex above, because the write callback may + // not await and may not block: a yield point there lets two chunks swap places. See + // [`crate::link::Link::mtu`]. + let mtu = Arc::new(AtomicUsize::new(FLOOR_MTU)); + let write_mtu = mtu.clone(); + // The notify callback below takes ownership of `sockets` for the sessions it spawns, and the // reconcile loop at the end outlives it. let for_reconcile = sockets.clone(); @@ -364,6 +411,15 @@ async fn serve_on_an_adapter( String::from_utf8_lossy(&value[..value.len().min(8)]).to_string(); let sender = for_write.lock().expect("write slot poisoned").clone(); + // What BlueZ says this link negotiated, minus the three bytes of ATT + // header a notification cannot use. Stored on every write because it is + // free to do so and a central may renegotiate; logged only when it moves, + // because the value that matters is the one a reply gets chunked for and + // that number had never appeared in the journal at all. + let payload = usize::from(req.mtu).saturating_sub(3).max(FLOOR_MTU); + let previous = write_mtu.swap(payload, Ordering::Relaxed); + let learned = (previous != payload).then_some(payload); + let result = match sender { None => { // Nowhere to send an answer, so accepting the request would be a @@ -394,6 +450,12 @@ async fn serve_on_an_adapter( }; async move { + if let Some(payload) = learned { + tracing::info!( + payload, + "negotiated notification payload; replies are sized for this" + ); + } // Eight bytes of the chunk, so a reordering is visible in the journal // rather than inferred from a parse error three layers up. Truncated // because a request may carry a wifi passphrase. @@ -416,12 +478,17 @@ async fn serve_on_an_adapter( method: CharacteristicNotifyMethod::Fun(Box::new(move |mut notifier| { let slot = for_notify.clone(); let sockets = sockets.clone(); + let mtu = mtu.clone(); async move { tokio::spawn(async move { // A fresh session, so nothing from a previous central can leak // into this one. + // Back to the floor for a new central: the previous one's MTU is + // not this one's, and the first write will report the real value + // before any reply is chunked. + mtu.store(FLOOR_MTU, Ordering::Relaxed); let (link, inbound, mut outbound) = - Link::pair(FLOOR_MTU, "central"); + Link::pair_sharing_mtu(mtu.clone(), "central"); let mine = inbound.clone(); { let mut slot = slot.lock().expect("write slot poisoned"); @@ -438,6 +505,11 @@ async fn serve_on_an_adapter( let session = tokio::spawn(session::run(link, sockets)); tracing::info!("central subscribed"); + // Notifications queued since the last pause. See `NOTIFY_BURST` + // for why a pump with no readiness signal has to pace itself. + let mut queued = 0usize; + let mut gone = false; + loop { tokio::select! { // Biased so a central that has gone away is noticed before @@ -448,16 +520,22 @@ async fn serve_on_an_adapter( // when a notify fails — which needs a reply to send, so a // client that disconnects while idle would hold the slot // until the next request arrives for nobody. - () = notifier.stopped() => break, + () = notifier.stopped() => { + gone = true; + break; + } chunk = outbound.recv() => match chunk { None => break, Some(chunk) => { - if let Err(e) = notifier.notify(chunk).await { - tracing::debug!( - error = %e, "notify failed; central gone" - ); + if !notify_chunk(&mut notifier, chunk).await { + gone = true; break; } + queued += 1; + if queued >= NOTIFY_BURST { + queued = 0; + tokio::time::sleep(NOTIFY_PAUSE).await; + } } }, } @@ -474,7 +552,20 @@ async fn serve_on_an_adapter( // its reassembly buffer and its upstream connections. slot.take(); session.abort(); - tracing::info!("central unsubscribed; session discarded"); + // Which of the two it was, because they need different + // next moves and the old line said "unsubscribed" for + // both — including for a reply this pump could not + // deliver, which is a robot problem wearing a client's + // clothes. + if gone { + tracing::info!( + "the central is gone; session discarded" + ); + } else { + tracing::info!( + "the outbound queue closed; session discarded" + ); + } } else { tracing::debug!( "a newer session holds the slot; leaving it alone" @@ -531,6 +622,46 @@ async fn serve_on_an_adapter( Ok(()) } +/// Send one chunk, retrying a queue that is momentarily full. `false` means give up on the session. +/// +/// **The distinction this function exists to draw**: `notify` returns the same error for "the +/// central unsubscribed" and "the D-Bus connection would not take this signal", and the pump used +/// to read both as the central having left. So a reply too big for one burst tore down the session +/// mid-line, and the client — still connected, as far as CoreBluetooth was concerned — waited out +/// its idle timeout with no error to report and half an answer in its reassembler. That was +/// diagnosed from a board, not from the journal, because the only line it left was a `debug` one +/// blaming the central. +/// +/// `is_stopped` tells them apart: it is closed only when the notification session really has +/// ended. Anything else is retried, and the chunk is cloned because `notify` consumes it and a +/// dropped chunk corrupts the line rather than failing it. +async fn notify_chunk(notifier: &mut CharacteristicNotifier, chunk: Vec) -> bool { + for attempt in 1..=NOTIFY_RETRIES { + match notifier.notify(chunk.clone()).await { + Ok(()) => return true, + Err(_) if notifier.is_stopped() => { + tracing::debug!("the notification session ended mid-reply"); + return false; + } + Err(e) => { + tracing::warn!( + attempt, + error = %e, + "the notification queue would not take a chunk; retrying" + ); + tokio::time::sleep(NOTIFY_RETRY_BACKOFF).await; + } + } + } + // A warning rather than a silence, which is the whole point: this ends a session, and + // whoever is holding the client deserves to find out why from the robot's own journal. + tracing::warn!( + retries = NOTIFY_RETRIES, + "gave up on a notification chunk; ending the session rather than sending a corrupt line" + ); + false +} + /// What the advertisement says about the robot: what it is called, and where it is on the network. /// /// One struct rather than two arguments threaded through the reconcile loop, so that "has anything diff --git a/btd/src/framing.rs b/btd/src/framing.rs index f48474d1..fe305f38 100644 --- a/btd/src/framing.rs +++ b/btd/src/framing.rs @@ -21,20 +21,50 @@ /// and it is reachable by anyone in radio range. Generous next to any real request — the /// largest is an `update.apply` with a long ref — and far below `updaterd`'s own 1 MiB line /// limit, because nothing that big has any business arriving over BLE. +/// +/// **This bounds what a peer may send, not what the robot may answer.** The two are not the +/// same size and were one constant until a reply outgrew it: see [`MAX_REPLY_LINE`]. pub const MAX_LINE: usize = 8 * 1024; +/// Longest line a *client* will reassemble from the robot. +/// +/// The other direction, and it needs its own number. [`MAX_LINE`] is a security bound — the +/// peer it limits is anyone in radio range, so it is deliberately tight — whereas this limits +/// the robot a client chose to connect to and already trusts. Reusing the tight one here made +/// every reply over 8 KiB unreadable, which quietly took `update.show` (kilobytes for an +/// ordinary run) and `system.logs` (a screenful of journal) out of reach of `duckctl` while +/// `btd` served both correctly. +/// +/// 64 KiB, and the real limit on a reply is smaller still and lives elsewhere: `proto`'s +/// `MAX_LOG_BYTES` keeps the journal tail well inside this, because at a typical ATT MTU +/// 64 KiB is a minute on the air and no answer should take that long. +pub const MAX_REPLY_LINE: usize = 64 * 1024; + /// Reassembles inbound chunks into whole lines. -#[derive(Debug, Default)] +#[derive(Debug)] pub struct Reassembler { buf: Vec, + /// What "too long" means for this direction. See [`MAX_LINE`] and [`MAX_REPLY_LINE`]. + limit: usize, +} + +/// The robot's side: [`MAX_LINE`], the tight bound, because the default must be the safe one. +impl Default for Reassembler { + fn default() -> Self { + Self { + buf: Vec::new(), + limit: MAX_LINE, + } + } } /// Why a peer's bytes were rejected. Both cases mean "drop the connection", but they are /// logged differently: one is a client that cannot frame, the other may be an attack. #[derive(Debug, PartialEq, Eq)] pub enum FramingError { - /// No newline within [`MAX_LINE`]. - LineTooLong, + /// No newline within this reassembler's limit, which the variant carries because the two + /// directions have different ones and the message must say which was hit. + LineTooLong { limit: usize }, /// Not valid UTF-8, so it cannot be JSON either. NotUtf8, } @@ -42,7 +72,7 @@ pub enum FramingError { impl std::fmt::Display for FramingError { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self { - Self::LineTooLong => write!(f, "no newline within {MAX_LINE} bytes"), + Self::LineTooLong { limit } => write!(f, "no newline within {limit} bytes"), Self::NotUtf8 => write!(f, "not valid UTF-8"), } } @@ -57,17 +87,28 @@ impl Reassembler { Self::default() } + /// A reassembler for the answers coming *back*, bounded by [`MAX_REPLY_LINE`]. + /// + /// Named for the direction rather than taking a number, so that a client reading replies + /// cannot pick the wrong bound and `btd` cannot accidentally be given the loose one. + pub fn for_replies() -> Self { + Self { + buf: Vec::new(), + limit: MAX_REPLY_LINE, + } + } + /// Feed one chunk; get back every complete line it completed. /// /// Returns a `Vec` because one write can legitimately carry several short lines — a /// client that batches `hello` and `update.status` into one 40-byte write is being /// efficient, not wrong. pub fn push(&mut self, chunk: &[u8]) -> Result, FramingError> { - if self.buf.len() + chunk.len() > MAX_LINE { + if self.buf.len() + chunk.len() > self.limit { // Clear rather than keep the partial line: whatever follows is unparseable // anyway, and holding it would let a peer pin the memory. self.buf.clear(); - return Err(FramingError::LineTooLong); + return Err(FramingError::LineTooLong { limit: self.limit }); } self.buf.extend_from_slice(chunk); @@ -165,11 +206,35 @@ mod tests { fn a_line_without_a_newline_is_refused_at_the_cap() { let mut r = Reassembler::new(); let big = vec![b'x'; MAX_LINE + 1]; - assert_eq!(r.push(&big), Err(FramingError::LineTooLong)); + assert_eq!( + r.push(&big), + Err(FramingError::LineTooLong { limit: MAX_LINE }) + ); // And the buffer was released, so the peer cannot pin memory by retrying. assert_eq!(r.pending(), 0); } + /// A reply larger than the request cap must reassemble on the client side. + /// + /// The regression this pins is one constant serving both directions: with a single 8 KiB + /// cap, a `system.logs` answer arrived intact from the robot and died in `duckctl` as + /// "no newline within 8192 bytes", which reads like a broken robot and is not one. + #[test] + fn a_reply_bigger_than_the_request_cap_is_accepted_by_a_client() { + let reply = format!("{{\"result\":\"{}\"}}\n", "x".repeat(32 * 1024)); + + let mut robot_side = Reassembler::new(); + assert!(matches!( + robot_side.push(reply.as_bytes()), + Err(FramingError::LineTooLong { limit: MAX_LINE }) + )); + + let mut client_side = Reassembler::for_replies(); + let lines = client_side.push(reply.as_bytes()).expect("under the cap"); + assert_eq!(lines.len(), 1); + assert_eq!(lines[0].len(), reply.len() - 1); + } + #[test] fn invalid_utf8_is_refused() { let mut r = Reassembler::new(); diff --git a/btd/src/link.rs b/btd/src/link.rs index 83b9fb65..c4228931 100644 --- a/btd/src/link.rs +++ b/btd/src/link.rs @@ -11,6 +11,9 @@ //! goes away. What happens between those two channels is [`crate::session`], and it never //! learns whether a radio is involved. +use std::sync::Arc; +use std::sync::atomic::{AtomicUsize, Ordering}; + use tokio::sync::mpsc; /// How many chunks may queue in either direction. @@ -38,19 +41,37 @@ pub struct Link { pub outbound: mpsc::Sender>, /// Usable notification payload — `ATT_MTU - 3` — as negotiated for this connection. /// - /// Read once per session rather than per message: a central may renegotiate, but not - /// mid-line, and re-reading it would let a line be chunked two different ways. - pub mtu: usize, + /// **A shared cell rather than a number, because the two halves of the link learn it at + /// different times.** BlueZ reports the negotiated MTU on every inbound write and offers the + /// notify side no way to ask, so a session begins knowing only the 20-byte floor and learns + /// the real value from the first write a client makes — which is always `system.authenticate`, + /// before any reply worth chunking exists. Sizing replies for the floor for the whole session + /// was a tenfold cost in notifications, and above about 5 KiB it stopped being merely slow: + /// see the pacing in `bluez`. + /// + /// Read once per *line*, never per chunk, which is the invariant that matters — a line chunked + /// two different ways cannot be reassembled. + mtu: Arc, /// The central's address, for the log line. Never used for authorization — a BLE address /// is trivially spoofed, and pairing is what authorizes (`architecture.md` §4.2). pub peer: String, } impl Link { - /// A link wired to channels the caller drives. Used by tests and by `--fake`. + /// A link wired to channels the caller drives, with a payload size that never changes. + /// Used by tests and by `--fake`. pub fn pair( mtu: usize, peer: impl Into, + ) -> (Self, mpsc::Sender>, mpsc::Receiver>) { + Self::pair_sharing_mtu(Arc::new(AtomicUsize::new(mtu)), peer) + } + + /// A link whose payload size is written by somebody else — the radio backend, from what + /// BlueZ reports on each inbound write. See [`Link::mtu`]. + pub fn pair_sharing_mtu( + mtu: Arc, + peer: impl Into, ) -> (Self, mpsc::Sender>, mpsc::Receiver>) { let (to_robot, inbound) = mpsc::channel(QUEUE); let (outbound, from_robot) = mpsc::channel(QUEUE); @@ -65,6 +86,11 @@ impl Link { from_robot, ) } + + /// The payload to size the next outbound line for. + pub fn mtu(&self) -> usize { + self.mtu.load(Ordering::Relaxed) + } } /// The queue must be deep enough that a maximal line never needs a blocking send. diff --git a/btd/src/route.rs b/btd/src/route.rs index 3f97f061..1eb67679 100644 --- a/btd/src/route.rs +++ b/btd/src/route.rs @@ -152,6 +152,25 @@ fn permits(call: &proto::Call) -> bool { // cannot report on this way is `btd` itself, which answering at all proves is running. SystemServices => true, + // The tail of one daemon's journal, and the next question after the line above: a unit + // reported as `failed` is a diagnosis nobody can act on without the reason it failed. + // + // Permitted because BLE is where the question is asked. A robot with no network cannot be + // reached by ssh, and that robot — one whose wifi never came up, whose `robotd` died on + // boot — is exactly the one whose journal somebody needs. Refusing here would mean the + // logs are readable over every transport except the one available when things are broken. + // + // Read-only, and bounded on the other side rather than trusted: `configd` picks the unit + // from a fixed list and refuses anything else, so this grants "the tail of a daemon this + // project ships", not `journalctl`. What a phone in the room learns is what that phone + // could already learn by watching the robot fail, in words it can put in a support ticket. + // + // The one thing worth naming as a cost: a journal line can carry more than a status. Ours + // are reviewed for that where it matters — `net.connect`'s passphrase is redacted by a + // hand-written `Debug` with a test pinning it (`proto::NetConnectParams`) — and this + // routing is the second reason that redaction is load-bearing rather than tidy. + SystemLogs(_) => true, + // Rebooting is drastic but recoverable, and it is what an app offers when a robot is // confused — the alternative being "unplug it", which for a walking robot is worse. // Unlike `resetToGolden` it discards nothing. @@ -300,6 +319,9 @@ fn permits(call: &proto::Call) -> bool { // of skills to assume any more, so this is how a phone knows there is a bow to ask for. RobotPolicies => true, + // Static geometry, read-only; the same class of read as the one above. + RobotModel => true, + // Re-reading the slots after something else edited the config. Same blast radius as // `robot.loadPolicy` and the same answer, and a client that can load wants this for the // case where the file changed underneath it. @@ -330,6 +352,10 @@ fn permits(call: &proto::Call) -> bool { // The transport is the gate here, not the credential. PolicyFetch(_) | PolicyInstall(_) => true, + // The detector's set, by the same argument: a read that reaches the network, and an + // install whoever tapped it is standing next to. + DetectorCheck | DetectorInstall(_) => true, + // ── the account, which BLE is the right transport for ──────────────── // // Signing the robot in to a Hugging Face account is what makes it reachable from outside @@ -354,7 +380,7 @@ fn permits(call: &proto::Call) -> bool { // offer, and `robot.init` is its counterpart: standing a robot up moves every joint at once, // which wants the person doing it to be looking at the robot rather than at a screen. Both // are `robotctl` on the robot, deliberately. - RobotInit | RobotRelax => false, + RobotInit | RobotRelax | RobotRebootMotors(_) => false, // `robot.stop` deserves its own line, because refusing it looks wrong. An emergency stop // in the app is exactly what someone reaches for, and §6 does say local should preempt @@ -386,6 +412,8 @@ fn permits(call: &proto::Call) -> bool { // reason to see what the robot sees, it will be through `mediad`'s video path // (`architecture.md` §5.2), where depth belongs next to the frame it annotates. TofStream => false, + // Same as the ToF: the head IMU is tofd's, reached over mediad's video path, not BLE. + HeadImuStream => false, } } @@ -478,6 +506,8 @@ mod tests { // the download, the shape gate at load, the clamps, the fall reflex. proto::method::POLICY_INSTALL, proto::method::POLICY_FETCH, + // Replacing the detector, by the same argument as the policy set. + proto::method::DETECTOR_INSTALL, // Binding the robot to a Hugging Face account, and unbinding it. Provisioning, // like the two below it and for the same reason: a robot out of a box has no // network, so it has no console and no LAN to open one from, and this is the @@ -664,6 +694,8 @@ mod tests { query: "microduck".to_owned(), }), proto::Call::PolicyInstall(proto::PolicyInstallParams::default()), + proto::Call::DetectorCheck, + proto::Call::DetectorInstall(proto::PolicyInstallParams::default()), ] { assert_eq!( upstream_for(&call), diff --git a/btd/src/session.rs b/btd/src/session.rs index 2b22429d..b1bf1e40 100644 --- a/btd/src/session.rs +++ b/btd/src/session.rs @@ -31,7 +31,8 @@ const PIN_ATTEMPTS: u32 = 3; /// Serve one central until it disconnects or breaks framing. pub async fn run(mut link: Link, sockets: Sockets) { let peer = link.peer.clone(); - tracing::info!(peer = %peer, mtu = link.mtu, "session opened"); + tracing::info!(peer = %peer, mtu = link.mtu(), "session opened; mtu is the floor \ + until the first write reports the negotiated one"); let (replies_tx, mut replies) = mpsc::channel::(QUEUE); let config_socket = sockets.config.clone(); @@ -305,7 +306,9 @@ async fn authenticate( /// Chunk one line out to the central. async fn send_line(link: &Link, line: &str) -> Result<(), ()> { - for chunk in framing::chunks(line, link.mtu) { + // Read once here, so one line is chunked one way even if a write reports a new MTU + // meanwhile. See `Link::mtu`. + for chunk in framing::chunks(line, link.mtu()) { if link.outbound.send(chunk).await.is_err() { // The backend dropped its half: the central is gone. return Err(()); @@ -392,6 +395,22 @@ mod tests { } } + /// Like [`read_reply`], and also says how the line was cut up on the way out. + async fn read_reply_in_chunks(from_robot: &mut Receiver>) -> (String, Vec) { + let mut r = Reassembler::new(); + let mut sizes = Vec::new(); + loop { + let chunk = tokio::time::timeout(std::time::Duration::from_secs(2), from_robot.recv()) + .await + .expect("client saw no reply") + .expect("link closed"); + sizes.push(chunk.len()); + if let Some(line) = r.push(&chunk).expect("framing").into_iter().next() { + return (line, sizes); + } + } + } + /// The reply a fake `configd` gives to `system.pairingPin`. fn pin_reply(pin: &str) -> String { serde_json::to_string(&proto::Response::ok( @@ -533,6 +552,71 @@ mod tests { ); } + /// **A reply is sized for the MTU the link has learned, and a session starts knowing only + /// the floor.** + /// + /// This is the mechanism behind `bluez`'s shared MTU cell, tested here because that file needs + /// a radio and this one does not. What it pins is the ordering that makes the trick sound: a + /// central subscribes before it writes, so a session opens at the 20-byte floor and learns the + /// real payload from the first write — which is always `system.authenticate`, before any reply + /// worth chunking exists. + /// + /// Sizing every reply for the floor was not merely slow. Ten times the notifications is ten + /// times the queue depth in BlueZ, and past roughly 5 KiB the notification session was torn + /// down mid-reply — a `system.logs` tail was the first reply big enough to find it. + #[tokio::test] + async fn a_reply_is_chunked_for_the_mtu_the_link_has_learned() { + let dir = tempdir(); + let (_, _) = FakeDaemon::spawn(dir.path(), "configd.sock", vec![pin_reply("424242")]); + let (_, _) = FakeDaemon::spawn(dir.path(), "updaterd.sock", vec![]); + let (_, _) = FakeDaemon::spawn( + dir.path(), + "robotd.sock", + vec![r#"{"jsonrpc":"2.0","id":2,"result":{"healthy":true}}"#.into()], + ); + + let mtu = std::sync::Arc::new(std::sync::atomic::AtomicUsize::new(20)); + let (link, to_robot, mut from_robot) = Link::pair_sharing_mtu(mtu.clone(), "AA:BB"); + tokio::spawn(run( + link, + sockets(dir.path(), "updaterd.sock", "robotd.sock"), + )); + + // At the floor, so the authentication answer goes out in 20-byte pieces. + to_robot + .send( + b"{\"jsonrpc\":\"2.0\",\"id\":0,\"method\":\"system.authenticate\",\"params\":{\"pin\":\"424242\"}}\n" + .to_vec(), + ) + .await + .unwrap(); + let (reply, sizes) = read_reply_in_chunks(&mut from_robot).await; + assert!(reply.contains(r#""authenticated":true"#), "{reply}"); + assert!( + sizes.len() > 1, + "a floor-sized reply arrived whole: {sizes:?}" + ); + assert!( + sizes.iter().all(|&n| n <= 20), + "a chunk exceeded the floor: {sizes:?}" + ); + + // What the write callback does on a real link, once BlueZ has reported the MTU. + mtu.store(182, std::sync::atomic::Ordering::Relaxed); + + to_robot + .send(b"{\"jsonrpc\":\"2.0\",\"id\":2,\"method\":\"robot.health\"}\n".to_vec()) + .await + .unwrap(); + let (reply, sizes) = read_reply_in_chunks(&mut from_robot).await; + assert!(reply.contains(r#""healthy":true"#), "{reply}"); + assert_eq!( + sizes.len(), + 1, + "the reply was still cut up for the floor: {sizes:?}" + ); + } + /// A subscription is a stream of notifications on an open connection, and every one has to /// reach the central. This is the case that would break if replies were correlated to /// requests rather than forwarded as they arrive. diff --git a/btd/src/upstream.rs b/btd/src/upstream.rs index 29f5f0b9..05ee5f46 100644 --- a/btd/src/upstream.rs +++ b/btd/src/upstream.rs @@ -153,30 +153,47 @@ impl Pool { } /// Send one line to `upstream` on `lane`'s connection, connecting first if needed. + /// + /// A daemon that restarted between two requests is ordinary here: `robotd` is the one an + /// update restarts, and `updaterd` restarts itself from a release's postinstall hook. The + /// connection from before the restart is still in the pool, its reader has already seen the + /// socket close, and the first write into it fails. Those bytes never left, so they are written + /// again on a fresh connection rather than reported. Reporting them told a phone that a daemon + /// listening on a fresh socket was not answering, once per lane, after every update. Once and + /// not in a loop: a daemon that is genuinely gone fails the reconnect, and that is the error + /// worth reporting. pub async fn send(&mut self, upstream: Upstream, lane: Lane, line: &str) -> io::Result<()> { let key = (upstream, lane); + let mut bytes = line.as_bytes().to_vec(); + bytes.push(b'\n'); + if !self.conns.contains_key(&key) { let conn = self.open(upstream, lane).await?; self.conns.insert(key, conn); } + match self.write(key, &bytes).await { + Err(e) if peer_is_gone(&e) => { + let conn = self.open(upstream, lane).await?; + self.conns.insert(key, conn); + self.write(key, &bytes).await + } + done => done, + } + } - // Unwrap is sound: just inserted, or the contains_key above held. + /// One bounded write on the connection for `key`. Any failure drops that connection, so + /// nothing keeps writing into a dead socket. This lane's only: the others may be perfectly + /// alive, and a restart that broke one breaks the next write to each of them anyway. + async fn write(&mut self, key: (Upstream, Lane), bytes: &[u8]) -> io::Result<()> { + // Unwrap is sound: the caller inserted it, or `contains_key` held. let conn = self.conns.get_mut(&key).expect("connection present"); - - let mut bytes = line.as_bytes().to_vec(); - bytes.push(b'\n'); - let write = async { - conn.write.write_all(&bytes).await?; + conn.write.write_all(bytes).await?; conn.write.flush().await }; match tokio::time::timeout(WRITE_TIMEOUT, write).await { Ok(Ok(())) => Ok(()), Ok(Err(e)) => { - // A broken pipe here is ordinary — the daemon restarted. Drop the connection - // so the next call reconnects rather than writing into a dead socket forever. - // This lane's connection only: the others may be perfectly alive, and a restart - // that broke one breaks the next write to each of them anyway. self.conns.remove(&key); Err(e) } @@ -233,6 +250,19 @@ impl Pool { } } +/// Whether a write failed because the peer went away, rather than being slow or refusing. Only +/// these are worth one more try, because only these mean the bytes went to a daemon that is no +/// longer there. A timeout is a daemon that is there and stuck, and that is not retried. +fn peer_is_gone(e: &io::Error) -> bool { + matches!( + e.kind(), + io::ErrorKind::BrokenPipe + | io::ErrorKind::ConnectionReset + | io::ErrorKind::ConnectionAborted + | io::ErrorKind::NotConnected + ) +} + #[cfg(test)] mod tests { use super::*; @@ -326,4 +356,64 @@ mod tests { .unwrap_err(); assert!(err.contains("configd refused"), "{err}"); } + /// The daemon restarted between two requests. + /// + /// The comment on the write path says a broken pipe is ordinary, the daemon restarted, and + /// the next call reconnects. It is the call *after* the next: the first write after a restart + /// went into the socket the old daemon closed, failed, and the phone was told the service was + /// not answering while it was listening on a fresh socket the whole time. The pool's own + /// reader had already seen the socket close and told nobody. + /// + /// On the BLE update path this is `updaterd` restarting itself from the release's postinstall + /// hook, and the phone's next `update.status` failing for it. + #[tokio::test] + async fn a_restarted_daemon_gets_the_next_request_not_the_one_after() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("updaterd.sock"); + let sockets = Sockets { + updater: path.clone(), + robot: dir.path().join("robotd.sock"), + config: dir.path().join("configd.sock"), + }; + let (replies, mut forwarded) = mpsc::channel(8); + let mut pool = Pool::new(sockets, replies); + + let before = serve_once( + &path, + proto::Response::ok(Some(proto::Id::Number(1)), &serde_json::json!({})), + ); + pool.send( + Upstream::Updater, + Lane::Prompt, + r#"{"jsonrpc":"2.0","id":1,"method":"update.status"}"#, + ) + .await + .unwrap(); + assert!( + forwarded.recv().await.is_some(), + "the first answer is forwarded" + ); + // The daemon has answered and hung up: it is restarting. + before.await.unwrap(); + + // And it is back, listening on a fresh socket at the same path. + std::fs::remove_file(&path).unwrap(); + let after = serve_once( + &path, + proto::Response::ok(Some(proto::Id::Number(2)), &serde_json::json!({})), + ); + pool.send( + Upstream::Updater, + Lane::Prompt, + r#"{"jsonrpc":"2.0","id":2,"method":"update.status"}"#, + ) + .await + .expect("a daemon that is back must get the request, not a broken pipe"); + let request = after.await.unwrap(); + assert!(request.contains(r#""id":2"#), "{request}"); + assert!( + forwarded.recv().await.is_some(), + "and its answer is forwarded" + ); + } } diff --git a/configd/Cargo.toml b/configd/Cargo.toml index 56a19096..34b125de 100644 --- a/configd/Cargo.toml +++ b/configd/Cargo.toml @@ -15,7 +15,9 @@ description = "Wifi and robot identity — the config service" duck-ipc-proto = { path = "../duck-ipc-proto" } serde.workspace = true serde_json.workspace = true -tokio = { workspace = true, features = ["rt-multi-thread", "macros", "net", "io-util", "time", "sync", "signal"] } +# `process` is for reading the journal: `logs` spawns `journalctl` rather than linking libsystemd +# for one read-only query. See that module for why. +tokio = { workspace = true, features = ["rt-multi-thread", "macros", "net", "io-util", "time", "sync", "signal", "process"] } clap.workspace = true tracing.workspace = true tracing-subscriber = { workspace = true, features = ["env-filter"] } diff --git a/configd/src/bluez.rs b/configd/src/bluez.rs index c30741d9..745faaba 100644 --- a/configd/src/bluez.rs +++ b/configd/src/bluez.rs @@ -6,12 +6,21 @@ //! //! ## The order, and why the state decides rather than the return values //! -//! `connect` **before** `pair`, and `trust` after both. Leading with `Pair()` on an Xbox controller -//! returns `AuthenticationCanceled`; that ordering comes from `microduck_runtime`'s notes and is the -//! one that works on this board. It used to live in a provisioning script's comments and in whoever -//! had done it before; now it is here, once, with the reason attached. +//! **Which order depends on the transport**, and the two are opposite: //! -//! But the order is **tried, not enforced**, because BlueZ's replies do not describe what happened: +//! - an **LE** pad (Xbox): `connect` **before** `pair`, and `trust` after both. Leading with +//! `Pair()` on an Xbox controller returns `AuthenticationCanceled`; that ordering comes from +//! `microduck_runtime`'s notes and is the one that works on this board. +//! - a **BR/EDR** pad (a "Pro Controller" Switch clone, and by the specification a DualShock or a +//! DualSense): `pair` **before** `connect`, then `trust`. See [the transport section](#which-transport-and-what-that-leaves-untested) +//! for what the other order does to it. +//! +//! `Snapshot::is_classic` decides, on whether BlueZ reports a `Class`. The rule used to live in a +//! provisioning script's comments and in whoever had done it before; now it is here, once, with the +//! reason attached. +//! +//! On the LE path the order is **tried, not enforced**, because BlueZ's replies do not describe what +//! happened: //! //! - `Connect()` on a device BlueZ has never bonded with can answer //! `br-connection-profile-unavailable` — there is no profile to connect to *yet*. Refusing there @@ -103,8 +112,8 @@ //! BR/EDR and LE together, and every property `Snapshot` reads is optional partly because the two //! transports present different ones. //! -//! But the pad all of this has been run against is **LE-only**. Its bond stores long-term keys and no -//! `[LinkKey]`, and BlueZ reports no `Class` for it at all: +//! The Xbox pad is **LE-only**. Its bond stores long-term keys and no `[LinkKey]`, and BlueZ reports +//! no `Class` for it at all: //! //! ```text //! # /var/lib/bluetooth///info @@ -113,14 +122,45 @@ //! [PeripheralLongTermKey] //! ``` //! -//! So the BR/EDR half of this file comes from the specification rather than from a radio. That -//! includes the class-of-device branch in [`looks_like_a_gamepad`], which cannot have fired — an LE -//! pad has no class to match — and the `br-connection-profile-unavailable` soft-fail above. +//! The first **BR/EDR** pad arrived on 2026-09-09: a no-name "Pro Controller", a clone of Nintendo's +//! Switch Pro Controller down to the modalias (`usb:v057Ep2009`), which is what makes the kernel's +//! `hid-nintendo` bind it and expose it as "Nintendo Switch Pro Controller" on evdev. BlueZ reports +//! it with `Class: 0x2508` — peripheral, gamepad — and derives `Icon: input-gaming` from that, so +//! [`looks_like_a_gamepad`] recognises it on two signals, and the class-of-device branch has now +//! fired on hardware. Only the HID and PnP UUIDs, no LE at all. +//! +//! What the LE order does to it, from `configd`'s own journal (2026-09-07, an earlier unit of the +//! same pad): `Connect()` on the unbonded pad spends ten seconds and answers +//! `br-connection-create-socket`; the `Pair()` fallback then answers +//! `ConnectionAttemptFailed: Page Timeout`. The pad has left pairing mode — a rejected classic +//! connection is enough to make it stop page-scanning — and every retry repeats the pair. Reproduced +//! by hand, the same order (`bluetoothctl connect`, which bonds as a side effect, then `trust`) goes +//! one step further and ends in the state that is hardest to read: `Paired: yes`, `Connected: yes`, +//! a solid light on the pad, `pad status` saying connected — and **no input device**, so `padd` waits +//! for a pad that BlueZ insists is there. `pair` → `connect` → `trust` works every time, so that is +//! what `bond` does when `Class` is present. A pad already in the broken state is recovered with +//! `pad forget` and a fresh `pad pair`. //! -//! Discovery stays on `auto` regardless, because the pads the heuristic names that are *not* LE — a -//! DualShock, a DualSense — are BR/EDR HID, and filtering to LE would make hardware this claims to -//! recognise unreachable. The thing to know is which way the risk runs: dropping BR/EDR would cost -//! nothing yet observed, and the first classic pad to arrive exercises that path for the first time. +//! **The clone is also two devices at once.** In pairing mode it advertises an LE face, +//! `BLE Controller_280609` at `98:B6:ED:28:06:09` — no class, no appearance, matched by the name +//! heuristic alone — and the BR/EDR face, `Pro Controller` at `98:B6:E9:28:06:09`. The LE face is +//! reported first — and, after the adapter power cycle `pad pair` performs on this board, several +//! seconds first. The first `robotctl pad pair` on this branch (2026-09-09) stopped on it, took the +//! LE order, hung [`BOND_TIMEOUT`] in `Connect()`, and by the time `Pair()` was tried the temporary +//! object was gone: `UnknownObject: Method "Pair" ... doesn't exist`. A two-second grace after the +//! first match did not help; the BR/EDR face was still not in the tree. So `find` now ends the +//! search early only on a match the radio classified — the LE face's name-only match is kept as a +//! fallback but never stops the sweep — and `one_face_per_pad` folds candidates that share their +//! unit octets into the classic one before the ambiguity rule sees them. +//! +//! Discovery stays on `auto`, so BlueZ sweeps both transports and a Pro Controller and an Xbox pad +//! are both found by the same search. +//! +//! Once bonded, a classic pad's reports are the kernel's business and not bluetoothd's: +//! `scripts/setup-board.sh` sets `UserspaceHID=false` in `input.conf`, because the default relays +//! this pad's ~200 packets/s of IMU through bluetoothd and uhid at 16% of a core. Nothing in this +//! file depends on which path is in use; it is noted here because it is the other half of what +//! "supporting a classic pad" turned out to mean. //! //! ## Where this has and has not run //! @@ -134,8 +174,13 @@ //! that re-initiates: the adapter scans as a central for a bonded peripheral while `btd` advertises //! as a peripheral itself. Both roles at once hold on this board's radio. //! -//! What has **not** been exercised on hardware: a pad that bonds over BR/EDR at all, a DualSense, two -//! pads in pairing mode at once, and pairing by explicit address. +//! **And against the Pro Controller clone**, 2026-09-09, on the board above: `pad pair` with the pad +//! in pairing mode found the classic face, bonded it and connected it in eight seconds — `bonded +//! (classic)`, `connected`, `gamepad paired and trusted` — and `padd` drove from it. An Xbox pad +//! paired a minute later through the unchanged LE path. +//! +//! What has **not** been exercised on hardware: a DualSense, two pads in pairing mode at once, and +//! pairing by explicit address. //! //! And one case that cannot be fixed from here: `pad forget` removes only the robot's half of the //! bond. A pad that still holds its half will not pair again until it is put back into pairing mode @@ -149,7 +194,7 @@ use duck_ipc_proto as proto; use zbus::names::OwnedInterfaceName; use zbus::zvariant::{ObjectPath, OwnedObjectPath, OwnedValue}; -use crate::pad::{PadResult, Pads, looks_like_a_gamepad}; +use crate::pad::{Evidence, PadResult, Pads, gamepad_evidence, same_pad}; /// Where our pairing agent lives on the bus. Any path we own will do; this one says whose it is. const AGENT_PATH: &str = "/com/pollenrobotics/configd/pad_agent"; @@ -177,6 +222,17 @@ const BOND_SETTLE: Duration = Duration::from_secs(5); /// How often to re-read `Paired` while waiting for a bond. const BOND_POLL: Duration = Duration::from_millis(200); +/// How long to keep sweeping after the first classified, unbonded pad turns up, for the rest of it. +/// +/// A pad can be two devices — the Pro Controller clones advertise an LE face and a BR/EDR face. +/// The classified one is the one worth having, and this is a short courtesy for the case where the +/// two arrive close together and the other happens to be classified too. It is **not** what +/// protects against the LE face: that face has nothing but a name, and a name-only match never +/// ends the search early at all — see `find`. Measured: after the adapter power cycle `pad pair` +/// performs on this board, the LE face was in BlueZ's tree within two seconds and the BR/EDR face +/// was not, so no grace short enough to be free would have caught it. +const SIBLING_GRACE: Duration = Duration::from_secs(1); + /// How often to re-read the object tree while looking for a pad. /// /// Polling rather than `InterfacesAdded`, which sounds like the right signal and is not: BlueZ emits @@ -382,8 +438,18 @@ impl Snapshot { }) } - fn is_gamepad(&self) -> bool { - looks_like_a_gamepad( + /// Does this pad bond over BR/EDR rather than LE? + /// + /// `Class` is the tell: it is the classic class-of-device, which an LE-only device has no way + /// to present, and BlueZ reports it from the inquiry response before anything else is known + /// about the device. `AddressType` does not separate the two — an Xbox pad's is `public` too. + /// The order `bond` tries depends on this, and the module docs say why. + fn is_classic(&self) -> bool { + self.class.is_some() + } + + fn evidence(&self) -> Option { + gamepad_evidence( &self.name, self.icon.as_deref(), self.class, @@ -391,6 +457,16 @@ impl Snapshot { ) } + fn is_gamepad(&self) -> bool { + self.evidence().is_some() + } + + /// Unbonded, and a pad on the radio's word rather than its name's — the only kind of match the + /// search ends early on. + fn is_fresh_and_classified(&self) -> bool { + !self.paired && self.evidence() == Some(Evidence::Classified) + } + fn as_pad(&self) -> proto::Pad { proto::Pad { mac: self.mac.clone(), @@ -497,9 +573,13 @@ impl BlueZ { /// first, which is the wrong shape: a robot may have several pads bonded, and `padd` drives /// whichever connects. /// - /// So with no address given, the sweep only ends early on a candidate that is **not yet paired**; - /// otherwise it runs to the deadline and reports what it has, which may be the pad already - /// bonded. The cost is that re-running `pad pair` with nothing new in pairing mode takes the whole + /// So with no address given, the sweep only ends early on a candidate that is **not yet paired + /// and classified by the radio** — icon, class or appearance, not its name alone. A name-only + /// match is kept and used if nothing better arrives by the deadline, but it does not stop the + /// search: the Pro Controller clone's LE face is exactly such a match, it is reported seconds + /// before the BR/EDR face that actually pairs, and stopping on it cost every attempt a + /// thirty-second hang and a dead object. Otherwise the sweep runs to the deadline and reports + /// what it has, which may be the pad already bonded. The cost is that re-running `pad pair` with nothing new in pairing mode takes the whole /// window before saying "already paired" — `--timeout` shortens it. /// /// An explicit address ends the sweep as soon as it appears, paired or not: the caller has named @@ -512,6 +592,9 @@ impl BlueZ { /// no way to learn the address that `--mac` needs. So the refusal carries the list. async fn find(&self, mac: Option<&str>, timeout: Duration) -> PadResult { let deadline = tokio::time::Instant::now() + timeout; + // When the first classified, unbonded candidate was seen, so the sweep can run + // [`SIBLING_GRACE`] past it before deciding. + let mut first_fresh: Option = None; loop { let seen = self.devices().await?; let matches: Vec = seen @@ -525,11 +608,19 @@ impl BlueZ { .cloned() .collect(); + let now = tokio::time::Instant::now(); let worth_stopping_for = match mac { Some(_) => !matches.is_empty(), - None => matches.iter().any(|device| !device.paired), + None => { + if matches.iter().any(Snapshot::is_fresh_and_classified) { + let since = *first_fresh.get_or_insert(now); + now - since >= SIBLING_GRACE + } else { + false + } + } }; - if worth_stopping_for || tokio::time::Instant::now() >= deadline { + if worth_stopping_for || now >= deadline { return Ok(Found { matches, seen }); } tokio::time::sleep(DISCOVERY_POLL.min(deadline - tokio::time::Instant::now())).await; @@ -576,9 +667,52 @@ impl BlueZ { .await .map_err(|e| (proto::PadPairFailure::Other, e.to_string()))?; - if !device.paired { - // `Connect()` first, which is the order that works on this board — leading with `Pair()` - // on an Xbox controller returns `AuthenticationCanceled`. + if !device.paired && device.is_classic() { + // A BR/EDR pad: `Pair()` first, then `Connect()`. The other order — the one below, which + // an LE pad needs — does not work on classic HID and leaves things worse than it found + // them. Seen against a "Pro Controller" (a Switch Pro clone, class 0x2508): + // `Connect()` on the unbonded pad spends ten seconds and answers + // `br-connection-create-socket`, and the `Pair()` after it answers + // `ConnectionAttemptFailed: Page Timeout` — the pad has stopped page-scanning by then, + // and the person has to put it back into pairing mode. Done by hand in the same order, + // `bluetoothctl connect` then `trust`, the pad ends up *Paired and Connected* with no + // input device behind it: the light on the pad goes solid, `pad status` says connected, + // and `padd` has nothing to open. `pair` → `connect` → `trust` is the sequence that + // works, so it is the one this takes. + // + // `Pair()` on BR/EDR is synchronous and answers when the bond is done — but it is still + // raced against `Paired`, as below, because that property is the ground truth and the + // reply is only one way to learn it. + let paired = tokio::select! { + outcome = tokio::time::timeout(BOND_TIMEOUT, proxy.pair()) => match outcome { + Ok(Ok(())) => { tracing::info!("bonded (classic)"); true } + Ok(Err(e)) if is_already_paired(&e) => { tracing::info!("already bonded"); true } + Ok(Err(e)) => return Err((proto::PadPairFailure::Rejected, e.to_string())), + Err(_) => false, + }, + bonded = self.wait_until_paired(&device.mac, BOND_TIMEOUT) => bonded, + }; + if !paired { + return Err(( + proto::PadPairFailure::Timeout, + "the pad did not finish pairing".to_owned(), + )); + } + + // Now connect, which is what brings up the HID channel and hands the pad to the kernel + // driver — `hid-nintendo`, for the clone above — so an input device appears. Soft: a + // bonded and trusted pad reconnects by itself, so a refusal here is not worth failing a + // pairing that succeeded. Logged at warn rather than info, though, because a classic + // pad that bonded and will not connect is the "connected light, no input" state from + // the other order, and worth a line in the journal. + match tokio::time::timeout(BOND_TIMEOUT, proxy.connect()).await { + Ok(Ok(())) => tracing::info!("connected"), + Ok(Err(e)) => tracing::warn!(error = %e, "bonded but not connected yet"), + Err(_) => tracing::warn!("bonded; connect did not answer in time"), + } + } else if !device.paired { + // An LE pad. `Connect()` first, which is the order that works on this board — leading + // with `Pair()` on an Xbox controller returns `AuthenticationCanceled`. // // But **soft-failed**, deliberately. A device BlueZ has never bonded with has no known // profile to connect to, so `Connect()` can answer @@ -661,6 +795,38 @@ impl BlueZ { } } +/// Collapse a pad that presents as two devices into the one worth bonding. +/// +/// Candidates that share their unit octets — [`same_pad`] — are one pad seen over two transports, +/// and the classic face is the one to keep: it is the one that carries HID to the kernel and the one +/// `bond` knows the order for. Among faces of the same kind the lowest address wins, for the same +/// determinism the caller applies to bonded pads. Candidates that share nothing pass through, so a +/// genuine second pad in pairing mode is still refused as ambiguous. +fn one_face_per_pad(fresh: Vec<&Snapshot>) -> Vec<&Snapshot> { + let mut kept: Vec<&Snapshot> = Vec::new(); + for candidate in fresh { + match kept.iter_mut().find(|k| same_pad(&k.mac, &candidate.mac)) { + Some(face) => { + let better = match (candidate.is_classic(), face.is_classic()) { + (true, false) => true, + (false, true) => false, + _ => candidate.mac < face.mac, + }; + if better { + tracing::info!( + kept = %candidate.mac, + dropped = %face.mac, + "one pad, two faces: keeping the classic one" + ); + *face = candidate; + } + } + None => kept.push(candidate), + } + } + kept +} + /// Did BlueZ refuse this because the bond already exists? fn is_already_paired(error: &zbus::Error) -> bool { matches!(error, zbus::Error::MethodError(name, _, _) @@ -770,6 +936,7 @@ impl Pads for BlueZ { // Without this, adding a second pad in a room where the first is in range would be // refused as ambiguous forever. let fresh: Vec<&Snapshot> = several.iter().filter(|d| !d.paired).collect(); + let fresh = one_face_per_pad(fresh); match fresh.as_slice() { [only] => (*only).clone(), // Nothing new: report the bonded one, and let `bond` re-assert `Trusted`. This is @@ -908,3 +1075,48 @@ impl Pads for BlueZ { Ok(proto::PadForgetResult { removed: true }) } } + +#[cfg(test)] +mod tests { + use super::*; + + fn face(mac: &str, name: &str, class: Option) -> Snapshot { + Snapshot { + path: OwnedObjectPath::try_from(format!( + "/org/bluez/hci0/dev_{}", + mac.replace(':', "_") + )) + .unwrap(), + mac: mac.to_owned(), + name: name.to_owned(), + icon: None, + class, + appearance: None, + paired: false, + trusted: false, + connected: false, + } + } + + /// The Pro Controller clone as the radio reports it: an LE face first, the classic one after. + /// One pad comes out, and it is the classic one whichever order they arrived in. + #[test] + fn a_pad_with_two_faces_is_one_candidate_and_the_classic_face_wins() { + let le = face("98:B6:ED:28:06:09", "BLE Controller_280609", None); + let classic = face("98:B6:E9:28:06:09", "Pro Controller", Some(0x2508)); + + for order in [vec![&le, &classic], vec![&classic, &le]] { + let kept = one_face_per_pad(order); + assert_eq!(kept.len(), 1); + assert_eq!(kept[0].mac, classic.mac); + } + } + + /// Two different pads stay two candidates — the ambiguity refusal downstream depends on it. + #[test] + fn two_pads_stay_two_candidates() { + let a = face("98:B6:E9:28:06:09", "Pro Controller", Some(0x2508)); + let b = face("78:86:2E:BB:13:28", "Xbox Wireless Controller", None); + assert_eq!(one_face_per_pad(vec![&a, &b]).len(), 2); + } +} diff --git a/configd/src/lib.rs b/configd/src/lib.rs index 0bb00341..3272bb58 100644 --- a/configd/src/lib.rs +++ b/configd/src/lib.rs @@ -27,6 +27,7 @@ #[cfg(target_os = "linux")] pub mod bluez; pub mod identity; +pub mod logs; pub mod net; #[cfg(target_os = "linux")] pub mod nm; diff --git a/configd/src/logs.rs b/configd/src/logs.rs new file mode 100644 index 00000000..96a5ed9c --- /dev/null +++ b/configd/src/logs.rs @@ -0,0 +1,407 @@ +//! The tail of one daemon's journal, for a client that has no shell on the robot. +//! +//! [`units`](crate::units) answers "is `robotd` running"; this answers the question that follows a +//! `failed` — *why*. Until it existed the only answer was `journalctl` over ssh, which needs a +//! network the robot may not have and an address a phone cannot reach. The robot whose journal +//! somebody actually needs is the one whose wifi never came up. +//! +//! **`configd` reads it because reading the journal needs privilege**, the same reason +//! [`units`](crate::units) is here: it runs as root (see `systemd/configd.service`), and +//! `robotctl` does not. On the robot itself nobody should use this — `journalctl` is right there, +//! with `-g`, `--since` and a pager. This is for the transports that have none of that. +//! +//! ## Why it shells out to `journalctl` +//! +//! Reading the journal in-process means linking `libsystemd`, which means vendored C in a +//! cross-compiled build for one read-only query. `journalctl` is on every board this ships to, +//! it is already how the updater talks to systemd (`updater::engine` spawns `systemctl`), and +//! the sandbox in this service's unit permits it: `ProtectSystem=strict` leaves `/usr/bin` +//! readable, and root can read the journal without any capability. +//! +//! ## Why it takes a unit name and not arguments +//! +//! This call is reachable from a phone in radio range (`btd::route`). `journalctl` arguments are +//! not a language to hand such a peer — `-D` reads another journal directory, `_PID=` matches +//! anything on the box — so the unit is chosen from a fixed list and everything else is refused +//! by name. What that costs is the searching a person with a shell would do. What it buys is that +//! the worst any client can ask for is the tail of a daemon this project ships. + +use duck_ipc_proto as proto; + +/// Units that are not part of a daemon release but whose journals answer *our* failures. +/// +/// Both earn their place by being the thing that is broken when the robot cannot be reached at +/// all: BlueZ, when no phone can see the robot, and NetworkManager, when wifi never came up. +/// Neither is ours, which is exactly why their logs are otherwise unreachable — nothing in +/// `system.services` reports on them either. +pub const ALSO_LOGGABLE: [&str; 2] = ["bluetooth.service", "NetworkManager.service"]; + +/// Every unit whose journal this service will read, in the order a reader wants them. +/// +/// Derived from [`crate::units::MANAGED`] rather than repeated, so a unit added to a release +/// becomes readable here without anyone remembering this file — the mistake that list's own doc +/// records having made once. +pub fn loggable() -> impl Iterator { + crate::units::MANAGED.into_iter().chain(ALSO_LOGGABLE) +} + +/// Longest single line kept, in bytes. +// +// Off Linux there is no `read` to call these, and only the tests do — the same shape +// `units::describe` takes, where the Linux half is the real one and the other half exists so a +// laptop can still build and test the crate. +#[cfg(any(target_os = "linux", test))] +/// +/// journald accepts a stdout line up to its own `LineMax` (48 KiB by default), so one daemon +/// logging a serialised blob could otherwise fill a whole reply with one entry and push out the +/// hundred lines around it — which is the opposite of what a tail is for. Truncated with an +/// ellipsis, so a reader can see that the line went on. +const MAX_LINE_BYTES: usize = 2 * 1024; + +/// Resolve a caller's unit name against [`loggable`]. +/// +/// `robotd` and `robotd.service` both work: a caller typing the daemon's name should not have to +/// know that systemd spells it with a suffix. The answer is the canonical name, so the reply says +/// what was actually read. +pub fn resolve(name: &str) -> Option<&'static str> { + let wanted = name.trim(); + let wanted = wanted.strip_suffix(".service").unwrap_or(wanted); + // Case-insensitive for `NetworkManager` alone, really: nobody types that capitalisation from + // memory, and refusing `networkmanager` would be a puzzle rather than a guard. + loggable().find(|unit| { + unit.strip_suffix(".service") + .unwrap_or(unit) + .eq_ignore_ascii_case(wanted) + }) +} + +/// What to tell a caller that named something else. Names the real list, the way `pad.bind` +/// refuses an unknown skill: a typo should come back as the answer, not as a shrug. +pub fn refusal(name: &str) -> proto::Error { + proto::Error::new( + proto::code::INVALID_PARAMS, + format!( + "no journal here for {name:?}; readable units are {}", + loggable().collect::>().join(", ") + ), + ) +} + +/// Read the tail of a unit's journal. +/// +/// `unit` must have come from [`resolve`] — that is what keeps a caller's string out of +/// `journalctl`'s argument list. +#[cfg(target_os = "linux")] +pub async fn read( + unit: &'static str, + lines: usize, + boot: i32, +) -> Result { + let lines = lines.clamp(1, proto::MAX_LOG_LINES); + + let output = tokio::process::Command::new("journalctl") + .arg("--unit") + .arg(unit) + // `--boot=-1` as one argument rather than `-b -1`: an offset that starts with a dash is + // otherwise ambiguous with an option, and which of the two `journalctl` decides it is has + // changed between versions. + .arg(format!("--boot={boot}")) + .arg("--lines") + .arg(lines.to_string()) + .arg("--no-pager") + // The timestamp is the point — a log line with no time answers nothing — and the + // `robotd[1234]:` prefix carries the PID, which is how a restart shows up in a tail. + // The hostname is the one field that is the same on every line and known already. + .arg("--output=short-iso") + .arg("--no-hostname") + .output() + .await + .map_err(|e| format!("could not run journalctl: {e}"))?; + + if !output.status.success() { + // journalctl's own words, which for the case a caller will actually hit — a boot that is + // not in the journal — are already the right answer ("Data from the specified boot (-1) is + // not available"). Ours would be a worse paraphrase. + let stderr = String::from_utf8_lossy(&output.stderr); + let detail = stderr.lines().next().unwrap_or("no output").trim(); + return Err(format!("journalctl refused: {detail}")); + } + + Ok(assemble(unit, &String::from_utf8_lossy(&output.stdout))) +} + +/// Off the board there is no journal to read, and inventing one would make a laptop look like a +/// robot. An error rather than an empty answer, for the same reason [`crate::units::describe`] +/// reports `Unknown` off Linux: "there is nothing here to read" and "this daemon said nothing" +/// are different answers. +#[cfg(not(target_os = "linux"))] +pub async fn read( + unit: &'static str, + _lines: usize, + _boot: i32, +) -> Result { + Err(format!( + "no systemd journal on this host, so nothing to read for {unit}" + )) +} + +/// The pid a line was written by, if the line is the daemon's own. +/// +/// `short-iso` puts the writer in the second field, as `robotd[3227]:` — or as `systemd[1]:` for +/// the lines systemd writes *about* the unit, which `journalctl -u` returns alongside it. Those +/// are the reason this checks the identifier: `Stopping`, `Stopped` and `Started` all come from +/// pid 1, so a bare "the pid changed" would announce a new process three times per restart and +/// name the wrong one. +#[cfg(any(target_os = "linux", test))] +fn pid_of(line: &str, service: &str) -> Option { + let field = line.split_whitespace().nth(1)?.strip_suffix(':')?; + let (identifier, pid) = field.strip_suffix(']')?.split_once('[')?; + if identifier != service { + return None; + } + pid.parse().ok() +} + +/// Turn `journalctl`'s output into a reply that fits the wire. +/// +/// Split out from [`read`] so the trimming is testable without a journal, which is the half with +/// the decisions in it: **oldest lines go first**, because the newest are the ones somebody asked +/// for, and the budget is measured on the serialised form rather than estimated, since JSON +/// escaping is what makes an estimate wrong. +#[cfg(any(target_os = "linux", test))] +fn assemble(unit: &'static str, stdout: &str) -> proto::LogsResult { + let service = crate::units::service_of(unit); + let mut lines: Vec = Vec::new(); + let mut writer: Option = None; + + for line in stdout.lines().map(str::trim_end).filter(|l| !l.is_empty()) { + if let Some(pid) = pid_of(line, service) { + // The tail a person reads spans restarts without saying so: a release is installed, + // every daemon restarts, and forty lines carry two different builds with nothing + // between them. systemd's own `Started …` line is in there, but it is one line among + // forty identical warnings and it reads as just another one. + // + // `-- … --` is `journalctl`'s own shape for a line it inserted rather than recorded + // (`-- Reboot --`, `-- No entries --`), so it is the form a reader already knows not + // to attribute to the daemon. + if writer.is_some_and(|previous| previous != pid) { + lines.push(format!("-- new {service} process, pid {pid} --")); + } + writer = Some(pid); + } + lines.push(truncate_line(line)); + } + + let mut truncated = false; + while serde_json::to_string(&lines).map_or(0, |json| json.len()) > proto::MAX_LOG_BYTES + && !lines.is_empty() + { + lines.remove(0); + truncated = true; + } + + proto::LogsResult { + unit: unit.to_owned(), + lines, + truncated, + } +} + +/// One line, bounded. Cuts on a character boundary, since the reply has to be valid UTF-8. +#[cfg(any(target_os = "linux", test))] +fn truncate_line(line: &str) -> String { + if line.len() <= MAX_LINE_BYTES { + return line.to_owned(); + } + let mut end = MAX_LINE_BYTES; + while end > 0 && !line.is_char_boundary(end) { + end -= 1; + } + format!("{}…", &line[..end]) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn a_daemon_is_named_with_or_without_the_suffix() { + assert_eq!(resolve("robotd"), Some("robotd.service")); + assert_eq!(resolve("robotd.service"), Some("robotd.service")); + assert_eq!(resolve(" btd "), Some("btd.service")); + } + + /// Nobody types that capitalisation from memory. + #[test] + fn networkmanager_is_matched_whatever_the_case() { + assert_eq!(resolve("networkmanager"), Some("NetworkManager.service")); + assert_eq!(resolve("NetworkManager"), Some("NetworkManager.service")); + } + + /// The guard the BLE route depends on: anything not on the list is refused, including the + /// shapes somebody would try if they were treating this as a `journalctl` command line. + #[test] + fn anything_else_is_refused() { + for name in [ + "sshd", + "", + "robotd.service extra", + "--directory=/run/log/journal", + "_PID=1", + "*", + ] { + assert_eq!(resolve(name), None, "{name} resolved"); + } + } + + /// A unit added to a release becomes readable without touching this file. + #[test] + fn every_managed_unit_is_loggable() { + for unit in crate::units::MANAGED { + assert_eq!(resolve(unit), Some(unit), "{unit} is not loggable"); + } + } + + #[test] + fn the_refusal_names_the_units_it_would_accept() { + let message = refusal("sshd").message; + assert!(message.contains("sshd"), "{message}"); + assert!(message.contains("robotd.service"), "{message}"); + assert!(message.contains("NetworkManager.service"), "{message}"); + } + + #[test] + fn a_tail_comes_back_oldest_first_with_blank_lines_dropped() { + let result = assemble("robotd.service", "one\ntwo\n\nthree\n"); + assert_eq!(result.lines, ["one", "two", "three"]); + assert!(!result.truncated); + } + + /// A tail that spans a restart says so, once, naming the process that took over. + #[test] + fn a_restart_is_marked_where_it_happened() { + let stdout = "\ +2026-09-09T12:27:19+00:00 robotd[965]: old build, still running +2026-09-09T12:27:20+00:00 systemd[1]: Stopping robotd.service - Robot control daemon... +2026-09-09T12:27:20+00:00 systemd[1]: Stopped robotd.service - Robot control daemon. +2026-09-09T12:27:20+00:00 systemd[1]: Starting robotd.service - Robot control daemon... +2026-09-09T12:27:21+00:00 robotd[3227]: control loop running +2026-09-09T12:27:24+00:00 robotd[3227]: bus read failed +"; + let lines = assemble("robotd.service", stdout).lines; + + // systemd's three lines come from pid 1 and must not each announce a new process. + assert_eq!( + lines.iter().filter(|l| l.starts_with("-- ")).count(), + 1, + "{lines:#?}" + ); + // Immediately before the first line of the new process, not where systemd spoke. + let at = lines + .iter() + .position(|l| l.starts_with("-- ")) + .expect("a marker"); + assert_eq!(lines[at], "-- new robotd process, pid 3227 --"); + assert!(lines[at + 1].contains("control loop running"), "{lines:#?}"); + assert!( + lines[at - 1].contains("Starting robotd.service"), + "{lines:#?}" + ); + } + + /// One process for the whole tail is the ordinary case, and it gets no marker at all — + /// including for the very first line, whose pid this has nothing to compare against. + #[test] + fn one_process_is_not_marked() { + let stdout = "\ +2026-09-09T12:27:21+00:00 robotd[3227]: control loop running +2026-09-09T12:27:24+00:00 robotd[3227]: bus read failed +"; + let lines = assemble("robotd.service", stdout).lines; + assert_eq!(lines.len(), 2, "{lines:#?}"); + assert!(!lines.iter().any(|l| l.starts_with("-- ")), "{lines:#?}"); + } + + /// A daemon that has restarted twice in one tail is marked twice. + #[test] + fn every_restart_in_the_tail_is_marked() { + let stdout = "\ +2026-09-09T12:00:00+00:00 btd[1]: first +2026-09-09T12:00:01+00:00 btd[2]: second +2026-09-09T12:00:02+00:00 btd[3]: third +"; + let lines = assemble("btd.service", stdout).lines; + assert_eq!( + lines + .iter() + .filter(|l| l.starts_with("-- new btd process")) + .count(), + 2, + "{lines:#?}" + ); + } + + /// The unit's *own* lines are what carry the pid. A line from anything else — systemd, or a + /// child process logging under its own name — cannot start a marker or the tail would be full + /// of them. + #[test] + fn only_the_daemons_own_lines_carry_a_pid() { + assert_eq!( + pid_of("2026-09-09T12:27:21+00:00 robotd[3227]: hello", "robotd"), + Some(3227) + ); + for line in [ + "2026-09-09T12:27:20+00:00 systemd[1]: Started robotd.service.", + "2026-09-09T12:27:20+00:00 setsid[4010]: a child of the daemon", + "2026-09-09T12:27:20+00:00 robotd: an identifier with no pid", + "not a journal line at all", + "", + ] { + assert_eq!(pid_of(line, "robotd"), None, "{line}"); + } + } + + /// Over budget, the *newest* lines survive — the ones somebody asked for. + #[test] + fn an_oversized_tail_keeps_the_end_and_says_it_was_cut() { + let stdout: String = (0..4000) + .map(|i| format!("2026-09-09T12:00:00+0200 robotd[1]: line {i}\n")) + .collect(); + + let result = assemble("robotd.service", &stdout); + + assert!(result.truncated); + assert!( + serde_json::to_string(&result.lines).unwrap().len() <= proto::MAX_LOG_BYTES, + "reply is over the budget the transport can carry" + ); + assert!( + result.lines.last().unwrap().ends_with("line 3999"), + "dropped from the wrong end: {:?}", + result.lines.last() + ); + } + + /// One daemon logging a serialised blob must not push out the lines around it. + #[test] + fn a_single_enormous_line_is_cut_rather_than_taking_the_whole_reply() { + let stdout = format!("prefix {}\nafter\n", "x".repeat(60 * 1024)); + let result = assemble("robotd.service", &stdout); + + assert_eq!(result.lines.len(), 2); + assert!(result.lines[0].len() <= MAX_LINE_BYTES + '…'.len_utf8()); + assert!(result.lines[0].ends_with('…')); + assert_eq!(result.lines[1], "after"); + assert!(!result.truncated); + } + + /// Cutting a multi-byte character in half would make the reply invalid UTF-8, and the line + /// that will hit this is a log message with an arrow or an accent in it — which ours have. + #[test] + fn a_line_is_cut_on_a_character_boundary() { + let line = "é".repeat(MAX_LINE_BYTES); + let cut = truncate_line(&line); + assert!(cut.ends_with('…')); + assert!(cut.len() <= MAX_LINE_BYTES + '…'.len_utf8()); + } +} diff --git a/configd/src/main.rs b/configd/src/main.rs index 48cdeba4..1c09a845 100644 --- a/configd/src/main.rs +++ b/configd/src/main.rs @@ -11,7 +11,7 @@ use configd::net::{FakeNet, Net}; use configd::pad::{FakePads, Pads}; use configd::power; use configd::store::Store; -use configd::{pad, units}; +use configd::{logs, pad, units}; use duck_ipc_proto as proto; use tokio::io::{AsyncBufReadExt, AsyncWriteExt, BufReader}; use tokio::net::{UnixListener, UnixStream}; @@ -445,6 +445,27 @@ async fn dispatch( // exactly the person diagnosing a robot. proto::Call::SystemServices => proto::Response::ok(Some(id), &units::all().await), + // Read-only, and not gated behind `may_mutate` for the same reason as the line above: the + // person who needs a daemon's last words is the person diagnosing a robot, and needing + // privilege to read them would put them out of that person's reach. + // + // The unit is resolved against a fixed list *before* it reaches `journalctl`, which is + // what makes this safe to route over BLE — see `logs` for the boundary and `btd::route` + // for the decision to route it. + proto::Call::SystemLogs(params) => match logs::resolve(¶ms.unit) { + None => proto::Response::err(Some(id), logs::refusal(¶ms.unit)), + Some(unit) => match logs::read(unit, params.lines, params.boot).await { + Ok(result) => proto::Response::ok(Some(id), &result), + // INTERNAL rather than INVALID_PARAMS: the request was well-formed and something + // on the board could not answer it. The one exception a caller will actually meet + // — a boot that is not in the journal — carries journalctl's own wording. + Err(e) => proto::Response::err( + Some(id), + proto::Error::new(proto::code::INTERNAL_ERROR, e), + ), + }, + }, + proto::Call::SystemInfo => proto::Response::ok( Some(id), &proto::SystemInfoResult { diff --git a/configd/src/pad.rs b/configd/src/pad.rs index abb058fd..829df0f7 100644 --- a/configd/src/pad.rs +++ b/configd/src/pad.rs @@ -85,8 +85,9 @@ pub fn pair_timeout(requested: Option) -> Duration { /// - **`class`** is the BR/EDR class-of-device: bits 8-12 are the major device class, and `0x05` /// is Peripheral. Bits 6-7 of the minor field distinguish keyboard from pointing device from /// gamepad — `0x01` in bits 2-5 with the keyboard/pointer bits clear is a joystick or gamepad. -/// Present for a classic pad, absent for a BLE-only one — and every pad tried so far has been -/// LE-only, so this arm is from the specification and has never fired on hardware. +/// Present for a classic pad, absent for a BLE-only one. A no-name "Pro Controller" (a Switch +/// Pro clone) presents `0x2508` — peripheral, gamepad — and this is the arm that names it when +/// BlueZ has not yet derived the icon from it. /// - **`appearance`** is the BLE equivalent: category 15 (`0x03C0..=0x03C4`) is HID, and `0x03C4` /// is specifically Gamepad. Many pads never set it, which is why it cannot stand alone. Only the /// gamepad value counts, so an LE pad advertising generic HID falls through to its name. @@ -102,8 +103,29 @@ pub fn looks_like_a_gamepad( class: Option, appearance: Option, ) -> bool { + gamepad_evidence(name, icon, class, appearance).is_some() +} + +/// How a device came to look like a gamepad — and how much that is worth. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Evidence { + /// The radio said so: BlueZ's icon, the class-of-device or the gamepad appearance. Settled. + Classified, + /// Only the name said so. Enough to pair when nothing better turns up, and not enough to stop + /// looking: the Pro Controller clone's LE face is `BLE Controller_280609` with nothing else + /// set, and stopping on it is what cost every pairing thirty seconds and a dead object. + NameOnly, +} + +/// [`looks_like_a_gamepad`], with the strength of the answer. Same four signals, same order. +pub fn gamepad_evidence( + name: &str, + icon: Option<&str>, + class: Option, + appearance: Option, +) -> Option { if icon == Some("input-gaming") { - return true; + return Some(Evidence::Classified); } if let Some(class) = class { @@ -113,7 +135,7 @@ pub fn looks_like_a_gamepad( // 0x03 remote control — the keyboard (0x10) and pointing-device (0x20) bits are the ones // this must not match, and they live above these values rather than overlapping them. if major == 0x05 && matches!(minor & 0x0f, 0x01 | 0x02) { - return true; + return Some(Evidence::Classified); } } @@ -121,7 +143,7 @@ pub fn looks_like_a_gamepad( // 0x03C4 is Gamepad; the rest of category 15 is other HID. Only the gamepad value counts, // because a Bluetooth keyboard is category 15 too and must not be paired as a pad. if appearance == 0x03C4 { - return true; + return Some(Evidence::Classified); } } @@ -137,6 +159,38 @@ pub fn looks_like_a_gamepad( ] .iter() .any(|needle| lower.contains(needle)) + .then_some(Evidence::NameOnly) +} + +/// Are these two Bluetooth addresses two faces of one pad? +/// +/// Some pads are two devices at once. The no-name "Pro Controller" Switch clones advertise an LE +/// personality (`BLE Controller_280609`, for phones) *and* the BR/EDR one that drives a robot, from +/// two addresses that differ only in the vendor half: `98:B6:ED:28:06:09` and `98:B6:E9:28:06:09`. +/// The lower three octets — the part a vendor assigns per unit — are identical, and the LE name +/// even spells them out. +/// +/// So two candidates that agree on those octets are one pad presenting twice, not two pads in +/// pairing mode, and the choice between them is not ambiguous: the classic one is the one that +/// carries HID to the kernel. Refusing would leave that pad unpairable without an address, and +/// picking whichever appeared first is how the LE face got connected to for thirty seconds while +/// the pad waited for a bond that never came. +/// +/// Three octets, not two or four: two is too few to separate unrelated devices from one vendor, +/// and four already reaches into the OUI, which is exactly the half that differs. +pub fn same_pad(a: &str, b: &str) -> bool { + let tail = |mac: &str| -> Option<[u8; 3]> { + let mut parts = mac.rsplit(':'); + let mut tail = [0u8; 3]; + for slot in tail.iter_mut().rev() { + *slot = u8::from_str_radix(parts.next()?, 16).ok()?; + } + Some(tail) + }; + match (tail(a), tail(b)) { + (Some(a), Some(b)) => a == b, + _ => false, + } } /// A set of pads that exists only in memory. @@ -287,14 +341,16 @@ mod tests { } /// A classic pad, identified by class-of-device with no icon and no name — which is what - /// discovery reports before a device is queried. Synthetic in a way the others are not: no pad - /// bonded to this robot has ever presented a class, so this arm is only ever exercised here. + /// discovery reports before a device is queried. #[test] fn a_peripheral_joystick_class_is_a_gamepad() { // Major 0x05 (peripheral), minor 0x01 (joystick): 0x000504. assert!(looks_like_a_gamepad("", None, Some(0x000504), None)); // Minor 0x02, gamepad: 0x000508. assert!(looks_like_a_gamepad("", None, Some(0x000508), None)); + // The class a "Pro Controller" Switch clone actually presents on the board (2026-09-09): + // the same gamepad minor with the limited-discoverable service bit set. + assert!(looks_like_a_gamepad("", None, Some(0x002508), None)); } /// The direction that matters more. A keyboard and a mouse are peripherals too, and pairing @@ -337,6 +393,21 @@ mod tests { assert!(!looks_like_a_gamepad("Pierre's iPhone", None, None, None)); } + /// A name is enough to pair on and not enough to stop looking on; anything the radio classified + /// is both. The clone's two faces, as BlueZ reports them, land on opposite sides. + #[test] + fn a_name_alone_is_weak_evidence() { + assert_eq!( + gamepad_evidence("BLE Controller_280609", None, None, None), + Some(Evidence::NameOnly) + ); + assert_eq!( + gamepad_evidence("Pro Controller", Some("input-gaming"), Some(0x2508), None), + Some(Evidence::Classified) + ); + assert_eq!(gamepad_evidence("Pierre's iPhone", None, None, None), None); + } + /// The whole arc, over the fake: pair the pad that is in pairing mode, see it bonded and /// trusted, forget it. #[tokio::test] @@ -441,6 +512,16 @@ mod tests { assert_eq!(pad.mac, "A4:AE:11:00:22:33"); } + /// The Pro Controller clone's two faces share their unit octets and nothing else about the + /// address; an unrelated device from the same vendor shares the OUI and nothing else. + #[test] + fn two_faces_of_one_pad_share_their_unit_octets() { + assert!(same_pad("98:B6:ED:28:06:09", "98:B6:E9:28:06:09")); + assert!(same_pad("98:b6:ed:28:06:09", "98:B6:E9:28:06:09")); + assert!(!same_pad("98:B6:E9:28:06:09", "98:B6:E9:46:9F:EA")); + assert!(!same_pad("98:B6:E9:28:06:09", "not an address")); + } + /// A caller cannot hold the adapter in discovery for as long as it likes, and zero means "look /// once" rather than "look forever" — a scripted retry needs to be able to ask. #[test] diff --git a/configd/src/units.rs b/configd/src/units.rs index 667a2891..18d3fffb 100644 --- a/configd/src/units.rs +++ b/configd/src/units.rs @@ -93,8 +93,9 @@ pub async fn describe(unit: &str) -> proto::ServiceUnit { } } -/// `btd.service` names the service `btd`, which is what it publishes under. -fn service_of(unit: &str) -> &str { +/// `btd.service` names the service `btd`, which is what it publishes under — and, with a pid +/// after it, what it logs under. [`crate::logs`] reads it for the second reason. +pub fn service_of(unit: &str) -> &str { unit.strip_suffix(".service").unwrap_or(unit) } diff --git a/configd/systemd/configd.service b/configd/systemd/configd.service index 92683950..8ec1452a 100644 --- a/configd/systemd/configd.service +++ b/configd/systemd/configd.service @@ -71,6 +71,12 @@ RestartSec=2s # - CAP_SYS_BOOT is NOT granted, because logind performs the reboot; configd only asks. A # capability here would let it call reboot(2) directly, which is precisely the unclean # shutdown this service exists to avoid. +# +# `system.logs` spawns `journalctl` (see `configd::logs`) and needs nothing added here, which is +# worth stating so nobody loosens this file for it: /usr stays executable under +# ProtectSystem=strict, and the journal files are 0640 root:systemd-journal — so uid 0 reads them +# as their owner, with the capability bounding set still empty. If a future kernel or journald +# changes that, the fix is a SupplementaryGroups=systemd-journal, not a capability. NoNewPrivileges=yes ProtectSystem=strict ProtectHome=yes diff --git a/deploy/robotd.toml b/deploy/robotd.toml index 160b6d75..9021a7be 100644 --- a/deploy/robotd.toml +++ b/deploy/robotd.toml @@ -1,7 +1,10 @@ # robotd — per-robot parameters. # -# Installed to /etc/robot/robotd.toml. Read once at startup, not watched: changing -# anything here needs `systemctl restart robotd`. Live reload is deferred +# Installed to /etc/robot/robotd.toml. Mostly read once at startup and not watched, so +# mostly a change needs a restart — of the daemon that reads the section, which is not +# always robotd: [media] and [duck_detector] are mediad's and [head_imu] is tofd's. The two +# exceptions: padd re-reads [pad_imu_head_control] a second after the file changes, and robotd +# re-reads [policy] when asked. `robotctl configure` knows which is which and offers it # (docs/design/robotd-design.md §4.2). # # It lives here, and not under /opt/robot/daemon/releases//, on purpose: this is @@ -112,8 +115,9 @@ mode = "walk" # "observation width is 51, expected 61".) # # Walk-mode defaults, spelled out, all under /opt/robot/policies/current/: -# walk = "alpha_walking.onnx" the velstand gait -# stand = "alpha_stand.onnx" standing + body pose +# walk = "velstand.onnx" walks on a twist, stands still at zero command +# stand = "none" velstand stands on its own; "alpha_stand.onnx" is +# the old standing + body-pose network, still in the set # sitstand = "alpha_sitstand.onnx" sit <-> stand, posture flag # ground_pick = "alpha_ground_pick.onnx" A-button pick # kick_left = "ball_kick_left.onnx" @@ -166,8 +170,8 @@ mode = "walk" # Scale actions with battery voltage: effective scale = action_scale * (nominal / measured # EMA, clamped 6.0..9.5 V). The servos' effective kP tracks their supply, so this holds the -# robot's response steady as the pack sags. Off by default, as in the prototype. -# voltage_adapt = false +# robot's response steady as the pack sags. On by default. +# voltage_adapt = true # nominal_voltage = 7.4 [safety] @@ -207,11 +211,11 @@ mode = "walk" # selects the standing network, and that is the stand-up. # # The only thing the daemon does about a fall. With it off, a fall changes nothing: the -# policy keeps driving and the humans stay in charge, which is the prototype's behaviour -# and the default here. ON by default since it was validated on a robot: the whole point is -# that the fleet lands soft, and a mode every board has to opt into individually is a mode -# most boards do not have. Set it to false on a robot that needs the old behaviour. -# limp_fall = true +# policy keeps driving and the humans stay in charge, which is the prototype's behaviour. +# OFF by default: the hand-back at the end is to the standing network, and the default +# configuration (velstand as `walk`, `stand` unset) loads none. Set it to true on a robot +# that runs a standing policy. +# limp_fall = false # When a fall counts as started. This is the whole feature, and both ways of getting it # wrong are real: too early and the robot limps out of leans it would have walked off — @@ -292,8 +296,16 @@ mode = "walk" # The ToF theremin: a hand's distance in front of the beak becomes a note, and the mouth # opens with the pitch. Picked up with `robotctl theremin` (or `robot.theremin`) — this # switch is only whether it *can* be, so a duck with a known-bad ToF can refuse outright. +# +# **Off by default.** It is a party trick, and a duck nobody asked to play one has no +# business holding a subscription to the depth stream; `robotctl theremin` on a duck that has +# not opted in says so and names this key. Turning it on does not start anything by itself. +# +# Nothing else changes with it: `tofd` runs and polls the sensor either way, so +# `robotctl monitor`'s ToF grid works the same whichever way this is set. +# # Needs [audio] on: a theremin with no voice is a mouth opening for no reason. -# enabled = true +# enabled = false # tofd's depth stream. # socket = "/run/tofd/tof.sock" @@ -324,20 +336,26 @@ mode = "walk" # withdrew keeps sounding. # hold_ms = 250 -[detect] +[duck_detector] # Finding other Microducks in the head camera. Read by `mediad`, not `robotd` — the frames are on # `mediad`'s pipeline and perception belongs next to the sensor — but the switch lives here because # this is the file `robotctl configure` edits and a robot has one place for its settings. # -# **Off by default.** The model ships in the release (models/duck_detect.rknn for the NPU, -# models/duck_detect.onnx for the CPU), and a robot nobody has asked to look for ducks should not -# pay ~60 ms of work every half second for it. +# **Off by default.** A robot nobody has asked to look for ducks should not pay ~60 ms of work +# every half second for it. # enabled = true # Which model, and therefore which processor: a `.rknn` runs on the NPU, an `.onnx` on the CPU. -# Unset tries the release's own, NPU first — and falls through to the CPU on a board whose NPU is +# Unset means the installed set — /opt/robot/detector/current/duck_detect.rknn for the NPU, +# duck_detect.onnx for the CPU — NPU first, falling through to the CPU on a board whose NPU is # switched off in its device tree, which is how Armbian ships the Radxa Zero 3. `sudo sh # /opt/robot/daemon/current/scripts/setup-npu.sh` turns it on, and needs a reboot. +# +# That set comes from the Hub (pollen-robotics/microduck-duck-detector, trained in +# pollen-robotics/duck_detector), the way the official policy set does: the release's postinstall +# hook seeds it once, `robotctl duck-detector check` asks whether a newer one exists, and +# `sudo robotctl duck-detector update` installs it and restarts mediad onto it. A retrain is a tag +# there, not a daemon release. This key is for a model of your own. # model = "/home/microduck/my_detector.rknn" # Looks per second. **2 is a thermal limit, not a preference**: flat out this reaches 95 °C on a @@ -350,6 +368,24 @@ mode = "walk" # coordinates. So this is a yes/no threshold rather than a dial. # threshold = 0.35 +[head_imu] +# The BMI088 on the head module, read by tofd and served as `head_imu.stream` — gyro, +# acceleration and a Madgwick orientation, for the mapping work. +# +# **Off by default, and this is the whole section.** Reading it at 100 Hz costs ~3.5-4.5% of a +# core on this SoC, and a bench that takes the loop apart says none of that is fixable: the +# wakeups are 0.7 points of it, the fusion 0.3, and the rest is the two I²C transactions a +# sample takes — which is what a gyro and an accelerometer sample *is*. Nothing subscribes to +# the stream yet, so every duck was paying it from boot for nobody. +# +# Turn it on when something reads it. `tofd --imu-hz 50` is the other lever: the cost is linear +# in the rate. Depth is unaffected either way — tofd ranges the ToF regardless, so +# `robotctl monitor`'s grid does not depend on this. +# +# A subscriber to `head_imu.stream` while this is off gets the reason, naming this key, rather +# than the silence a board with no sensor fitted gives. +# enabled = false + [chorale] # Several ducks singing one piece in four parts, found over Bluetooth with no shared clock: the # conducting duck's beat counter is the timebase, and the parts are worked out from each duck's own @@ -372,10 +408,16 @@ mode = "walk" # only supported way to change one was a systemd drop-in, and nobody reaches for a drop-in to # answer "why is the video soft?". -# Stream the head camera. Off streams a test pattern instead, which is what a board with no -# camera wants — the pipeline still starts, so the WebRTC *control* channel still exists. It is -# bundled with the video track, so a pipeline that will not start costs both. -# camera = true +# Where the video comes from: "camera" or "test". +# +# "test" streams a test pattern, which is what a board with no camera wants — the pipeline still +# starts, so the WebRTC *control* channel still exists. It is bundled with the video track, so a +# pipeline that will not start costs both. A name rather than a boolean because "camera = false" +# does not mean "no video", it means "synthetic video", and nothing in the old key said so. +# +# The pattern ignores `quality` and runs at 256x144@5: it exists to make the session reachable, +# and drawing a 720p30 one in software cost five times what the camera it stands in for does. +# source = "camera" # Frame size and rate, as one name: 1080p30, 720p30, 720p15 or 360p30. # @@ -414,3 +456,22 @@ mode = "walk" # # gcc is webrtcsink's own default, so this line changes nothing; it is here to be found. # congestion_control = "gcc" + +# ── Controller-IMU head control ───────────────────────────────────────────────────────────── +# +# Read by `padd`, not by `robotd`. Some pads carry an inertial unit — the "Pro Controller" Switch +# clones do, an Xbox pad does not. With this on and such a pad connected, Y stops meaning "the +# sticks pose the head" and means "the pad's tilt poses the head": the sticks keep driving, and +# turning the pad in your hands turns the robot's head. Y again holds the head where it is, still +# driving. A third press hands the head back to the pad from where the pad is *now* — its yaw is +# a gyro's word alone and drifts, and re-centring on every re-entry is how you beat that without +# a magnetometer. Off, or on a pad with no IMU, Y is what it always was. +# +# Not [head_imu], which is the IMU in the robot's head. This section was [imu_head] once; a file +# still saying that loads, and `robotctl configure` renames it on its next save. +# [pad_imu_head_control] +# enabled = false + +# Head radians per pad radian. 1 follows the pad exactly; more turns a small wrist movement into +# a large head movement. The head's own travel limit still applies. +# gain = 1.0 diff --git a/docs/README.md b/docs/README.md index 867f71de..7b654d14 100644 --- a/docs/README.md +++ b/docs/README.md @@ -3,6 +3,10 @@ The [README](../README.md) is the front door — what a microduck is, and where to go. If you have one in front of you and want to drive it, start at the [cheat sheet](robot/cheatsheet.md). +[`faq.md`](faq.md) is the other front door: task-shaped questions from somebody building +*against* a duck rather than changing it — running a model too heavy for the board, getting the +camera into their own program, why a Space cannot connect. + It is also where a **publisher** starts: [`policy-manifest.md`](policy-manifest.md) is the contract for a `manifest.json` beside a microduck `.onnx`, and it owns every field. The design docs give the reasoning and point at it. @@ -15,6 +19,7 @@ docs give the reasoning and point at it. | [`pair-a-gamepad.md`](robot/pair-a-gamepad.md) | Once per pad: pairing mode, `pad pair`, and what to do when it will not bond. | | [`cheatsheet-dev.md`](robot/cheatsheet-dev.md) | The commands that need a dev board: branch builds, candidates, dev pushes. | | [`dev-push.md`](robot/dev-push.md) | Build on your machine and install on the board over ssh, with no CI run. | +| [`simulation.md`](robot/simulation.md) | The simulated duck: `scripts/duck-sim`, the real daemons against a MuJoCo body, one duck or several in containers. | | [`duckctl.md`](robot/duckctl.md) | Every `duckctl` command — the robot from a laptop, over Bluetooth. | | [`install-dev.md`](robot/install-dev.md) | Setting up a board for development, from nothing. | | [`install-by-hand.md`](robot/install-by-hand.md) | The same install as separate commands, for testing one step at a time. | @@ -44,6 +49,7 @@ own the mechanism is the bug. | [`webrtc-console.md`](design/webrtc-console.md) | The WebRTC client: serving it from the robot, finding the robot, and what the page should be. | | [`remote-access-design.md`](design/remote-access-design.md) | Reaching a duck from outside the LAN: the Hugging Face account, the device flow, and the bridge to a rendezvous service. | | [`boot-recovery-net.md`](design/boot-recovery-net.md) | Falling back to golden when the release that booted cannot start its daemons. | +| [`simulation.md`](design/simulation.md) | The twin: where the seam between daemon and body is, the body protocol, the fake radio, the containers, and what it is and is not a twin of. | ## `project/` — you are running the project @@ -59,6 +65,7 @@ Dated records rather than reference. They describe a moment, and go stale on pur | [`media-bringup.md`](project/media-bringup.md) | What a Radxa Zero 3W does about video: the VPU, what MPP needs, and the two plugins that have to be built. | | [`pad-minimal-pairing.md`](project/pad-minimal-pairing.md) | The smallest board configuration a gamepad will bond under, found by taking one away at a time. | | [`idle-cpu.md`](project/idle-cpu.md) | What the daemons do when nobody is asking them to: four things that stopped, two that were measured and left alone, and what still wants a board. | +| [`tof-on-demand.md`](project/tof-on-demand.md) | `tofd`'s idle 5% is nine parts head IMU to one part depth, and the IMU has no consumer. Why the laser and the unit were left alone and the IMU got a switch. | ## `ideas/` — not designed yet diff --git a/docs/design/app-path-design.md b/docs/design/app-path-design.md index 361b8df8..6fa304f0 100644 --- a/docs/design/app-path-design.md +++ b/docs/design/app-path-design.md @@ -386,6 +386,40 @@ the tidier model and needs `btd` to know when a call ended, which needs it to pa never does (§3), and that property is what keeps the routed subset a transport rather than a second implementation of the API. +### 3.6 A reply bigger than one burst, and the MTU the notify side cannot ask · **measured** + +`bluer`'s callback model — the one that works here, for the reason in `btd/src/bluez.rs`'s header — +gives the notify side neither of the two things a sender needs. It cannot ask what the link +negotiated, and `notify` returns as soon as a D-Bus signal is queued, so nothing tells the pump when +the radio has caught up. Both gaps were papered over by sizing every reply for the 20-byte floor and +firing chunks as fast as the loop would spin. + +Measured against the board from a Mac, asking `system.logs` for a journal tail: + +| reply | notifications at the floor | what happened | +|---|---|---| +| ~5 KiB (24 lines) | ≈265 | delivered, in 1.83 s | +| ~7 KiB (32 lines) | ≈350 | the notification session was torn down 150 ms in | + +The second row is the one that cost a board session to find. `btd` logged one INFO line saying the +central had unsubscribed — it had not; CoreBluetooth still had the link up — and the client sat +waiting out a 60-second idle budget with half an answer in its reassembler and no error to report. +The pump was reading `notify`'s single error as "the central left", when a full queue and a departed +central are the same `Err` from `bluer` and only `is_stopped()` tells them apart. + +Three changes, none of them a real solution, because the model has no readiness signal to offer: + +- **The payload comes from the write side.** BlueZ reports the negotiated MTU on every inbound + write, one ATT MTU serves both directions, and a central always writes before there is a reply to + send — `system.authenticate` is the first write of every session. So a session opens at the floor + and is sized properly from its first answer onward, roughly a tenfold cut in notifications. +- **The pump pauses every 16 chunks**, about a connection interval. A small reply never pauses. +- **A refused chunk is retried** rather than read as a disconnect, and giving up now logs a warning + saying so. + +What would solve it properly is the IO model's `sendable()`, and that stays out of reach: it serves +only the `Acquire*` fd paths, which a CoreBluetooth central does not drive. + ## 5. Pairing: just-works, and a PIN the transport checks A six-digit PIN, stored by `configd`, checked by `btd` before it serves anything. **Not** by the diff --git a/docs/design/architecture.md b/docs/design/architecture.md index 1b2c1e1b..568590ef 100644 --- a/docs/design/architecture.md +++ b/docs/design/architecture.md @@ -84,7 +84,7 @@ counter ([`updater-design.md`](updater-design.md)). | `updaterd` | releases: verify, install, swap, health-gate, roll back | `/run/updaterd.sock` | GitHub releases, `systemctl`, `robotd` | | `btd` | nothing — BLE transport for a subset of the API | a BLE GATT service | `robotd`, `configd`, `updaterd` — not `padd` or `tofd`, whose streams a radio this narrow cannot carry | | `padd` | nothing — gamepad transport; serves a raw input tap | `/run/padd/pad.sock` (`pad.input` only) | `/run/robotd.sock` | -| `mediad` | the camera and audio pipeline; nothing of the robot — WebRTC transport and the remote front door (§5.2) | TCP: the console on `:8080`, signalling on `:8443` — no unix socket of its own | `robotd`, `configd`, `updaterd` | +| `mediad` | the camera and audio pipeline; nothing of the robot — WebRTC transport and the remote front door (§5.2) | TCP: the console and PNG `GET /frame` on `:8080`, signalling on `:8443`; and one unix socket of its own, `/run/mediad/media.sock`, serving `media.frame` to a local recorder or perception process. A raw frame is ~1.8 MiB, so it is deliberately not carried on the WebRTC control channel | `robotd`, `configd`, `updaterd` | | `tofd` | the head's ToF sensor: an 8×8 depth matrix it publishes and nobody else reads | `/run/tofd/tof.sock` (`tof.stream`) | the HAT's I²C bus | | `robotctl` | nothing — the CLI, and the tool that must work on a broken robot | — | every socket above | diff --git a/docs/design/policy-channel-design.md b/docs/design/policy-channel-design.md index 34f686c9..8f71fbb2 100644 --- a/docs/design/policy-channel-design.md +++ b/docs/design/policy-channel-design.md @@ -273,16 +273,24 @@ updater design). Fetching rather than copying is the point: it is the arrangement `setup-board.sh` already uses for ONNX Runtime and `setup-gstreamer.sh` for the plugins, which are the other two things a board -needs and a release has no business carrying. The pin lives in `[workspace.metadata.policies]` +needs and a release has no business carrying. The duck detector followed the same road out of the +release — `scripts/seed-detector.sh`, `/opt/robot/detector/current`, `robotctl duck-detector +check/update`, a pin in `[workspace.metadata.detector]` — with a fixed file list in place of the +manifest, since its two files have fixed names (`docs/project/npu-bringup.md`). The pin lives in `[workspace.metadata.policies]` and as literals in the script, with a test asserting they agree — `setup-gstreamer.sh`'s trap, because a script that runs from inside a release cannot read the manifest. -**The pin is a floor, not a ceiling**, and that distinction is load-bearing. It ships inside the -daemon release, so bumping it *does* need a daemon release — an earlier draft of this section -claimed otherwise and was simply wrong. What the pin decides is what a *freshly provisioned* board -installs. Moving past it is `robotctl policy update` (§9.1), which is the thing that makes a -retrained gait reach a robot without a daemon release, and therefore the thing that makes this -whole channel worth having. +**The pin is a minimum, not a ceiling**, and that distinction is load-bearing. It ships inside +the daemon release, so bumping it *does* need a daemon release — an earlier draft of this section +claimed otherwise and was simply wrong. The pin decides two things: what a *freshly provisioned* +board installs, and the oldest official set this daemon runs with. A board whose set is from our +repo and below the pin is moved up to it by the post-install hook — the daemon's slot defaults +name files, and a default that names a file only a newer set carries (v5's `velstand.onnx`) would +otherwise leave a board that updated the daemon alone unable to load its gait, unhealthy, and +rolled back. A set past the pin, from another repo, or without a `.source` record is left alone. +Moving past it is `robotctl policy update` (§9.1), which is the thing that makes a retrained gait +reach a robot without a daemon release, and therefore the thing that makes this whole channel +worth having. Three things it does not do. It does not re-download a set it already has, so an update whose pin is unchanged touches no network — which matters because the post-install hook runs under a @@ -657,7 +665,8 @@ dereferencing the symlinks that repository uses to give stable names to particul | in the set | upstream | role | | --- | --- | --- | -| `alpha_walking.onnx` | `BEST_alpha_walking_rough.onnx` | walking / velstand | +| `alpha_walking.onnx` | `BEST_alpha_walking_rough.onnx` | walking / velstand (default `walk` until set v5) | +| `velstand.onnx` | `pollen-robotics/microduck_rl` `velstand_best.onnx` (2026-09-14) | walking + standing at zero command; default `walk` from set v5, `stand` unset | | `alpha_stand.onnx` | `BEST_alpha_stand_body_control.onnx` | standing + body-pose | | `alpha_sitstand.onnx` | `BEST_alpha_sitstand.onnx` | sit ↔ stand (posture flag) | | `alpha_ground_pick.onnx` | `alpha_ground_pick.onnx` | ground pick (phase command) | @@ -703,7 +712,7 @@ its meaning for the things that genuinely are models and not control policies, s | Seeding never overwrites a set it did not install | The handover needs no flag: the first real install ends it (§9) | | The set is downloaded, not shipped | Same as ONNX Runtime and the plugins; bumping the pin ships a gait (§9) | | A failed fetch keeps the set already installed | A half-published revision must not downgrade a working gait (§9) | -| The pin is a floor; `policy update` moves past it | Otherwise a gait still needs a daemon release, which is the thing this channel is for (§9.1) | +| The pin is a minimum; `policy update` moves past it | A default that names a newer set's file must not roll the daemon back on boards behind it; a gait still needs no daemon release (§9.1) | | A set records the repo it came from | One writer, one copy, nothing to configure twice or drift (§9.1) | | Reload is a third thing, not reset-all | They look identical from outside and conflating them discards every override (§9.1) | | One-shot skills are config, not code | Kicks and roulade were the same arm with different numbers; a community one is a fifth set (§10) | diff --git a/docs/design/remote-access-design.md b/docs/design/remote-access-design.md index d1008352..ab78dfed 100644 --- a/docs/design/remote-access-design.md +++ b/docs/design/remote-access-design.md @@ -51,6 +51,13 @@ service too was made the other way for this reason, and §3.1 is where it costs ## 2. The account is an OAuth device flow against Hugging Face +**The flow lives in [`hf-robot-account`](https://github.com/pollen-robotics/hf-robot-account)**, +its own repository, because nothing in it is about a duck: any robot with no browser signs in the +same way. What stays in `updater/src/account.rs` is what is a fact about *this* robot — the token +path, the `robot` group, the mapping onto `proto`, and which JSON-RPC code each refusal deserves. +Everything below about the flow itself describes that crate; everything about where the credential +lands and who may read it describes this repository. + ### 2.1 Why the device grant, which is also where `reachy_mini` ended up `reachy_mini` has **both**. It started with authorization code + PKCE, pointing the redirect URI @@ -190,8 +197,8 @@ drops its answer. That is also what makes `logout` able to promise what it says. `huggingface_hub` ships a **first-party public device-code client**, `DEVICE_CODE_OAUTH_CLIENT_ID` = `26be6b09-91c5-47da-9861-d2d2bb7a7e36`, which is what `hf auth login` uses. It is public — no secret, so nothing needs baking into a release beyond a public identifier — and it needs no OAuth -app registered anywhere. `updater::account::CLIENT_ID` is that constant, and it is the whole of -what this decision came to. +app registered anywhere. `hf_robot_account::HUGGINGFACE_CLIENT_ID` is that constant, and it is the +whole of what this decision came to. Two alternatives, recorded because the first one looks obvious and is blocked: @@ -223,7 +230,7 @@ What the account actually needs is `openid profile`, plus `read-repos` for one r mechanism (`policy-channel-design.md` §7). **So the narrow version is a Pollen-owned public device-code client with -`openid profile read-repos`** — one constant in `account.rs` and one click by somebody with HF org +`openid profile read-repos`** — one `Config::client_id` and one click by somebody with HF org admin. It is not blocking: the flow works today and a scope change is a re-login. It is worth doing before a duck goes home with anybody, because the failure mode is asymmetric — a robot that has been able to write all along cannot be un-done, while a robot that needs a wider scope later @@ -301,8 +308,11 @@ Three things make that acceptable rather than merely permitted: - **It is revocable** — `account.logout` from anywhere, and revoking the grant on Hugging Face, which no robot-side gate could offer. - Worth being exact about the first one, because it is weaker than it sounds: **`logout` deletes - the robot's copy and revokes nothing.** The robot stops being a producer, which is the effect + Exact about the first one, because it has two halves and only one of them is `updaterd`'s. + `logout` deletes the credential, and the relay notices within one heartbeat — ten seconds — and + drops its connection, which on this service evicts the producer at once. So a robot signed out + stops being listed and stops being reachable. What it does **not** do is revoke anything: + **`logout` deletes the robot's copy and tells Hugging Face nothing.** The robot stops being a producer, which is the effect somebody signing out is after — but the access token it held stays valid at Hugging Face until it expires, up to thirty days, for anything that already read the file. The credential is `0640 root:robot`, so "anything" means root or `mediad` on that board; a stolen board is the @@ -339,7 +349,7 @@ A device-code token comes back as `expires_in: 2591999` — thirty days — with and refreshing **rotates** it: the answer carries a *new* refresh token and the old one is spent. So the store is two strings plus a clock, and there are three consequences worth naming. -**A robot that is simply left on must renew itself.** `updater::account::maintain` wakes every six +**A robot that is simply left on must renew itself.** `hf_robot_account::maintain` wakes every six hours and refreshes anything with under a week left — three-quarters of the way through the token's life, leaving a week of retries for a board whose network is marginal. It is spawned unconditionally, unlike the update scheduler, because a robot with update checks switched off @@ -391,6 +401,16 @@ is not, in three ways: | auth | none (§4 of `remote-webrtc.md`) | `Authorization: Bearer ` | | ids | its own `peerId`, its own `sessionId` | different ones, per hop | | our role | `listener` | `producer` | +| where a reply arrives | on the socket, always | **`sessionStarted` and `list` in the `POST` response body**; everything else on the stream | + +That last row cost an afternoon and belongs in a table rather than in somebody's memory. On a +WebSocket every answer comes back on the socket, so a client naturally treats a send as +fire-and-forget. Here `handle_start_session` *returns* `{"type":"sessionStarted","sessionId":…}` to +the caller of `POST /send` — the producer is notified over SSE, the consumer is answered in the +response — and a page that discarded that body got the robot's offer for a session whose id it had +never been told, then failed on a null peer connection. `list` is answered the same way, and is +*also* pushed over the stream after the welcome, which is why the listing half looked healthy while +the session half was broken. So the bridge keeps a session table both ways and rewrites `sessionId` on every `peer` message. That is where `reachy_mini`'s relay has needed most of its scar tissue (§3.4), and it is the honest @@ -416,6 +436,15 @@ into a cadence slower than our own eviction. `reachy_mini`'s ladder has a middle publishes no `lease_seconds`**, so that rung is unreachable here; it is not worth reproducing a negotiation step for a field nothing sends. +**`POST /send` before `GET /events` is a 400, so the order is not a preference.** The peer does +not exist until the stream does — identity comes from the bearer token, and the token is bound to +a peer by the `/events` connection. §3.4 wanted registration before anything reported the robot +reachable for a different reason; the service enforces the same order for its own. + +**The rate limit is 1200 requests per 60 s per peer**, sliding. A 10 s heartbeat and a 30 s poll +are nine requests a minute, so it constrains nothing here — it is a ceiling to know about before +somebody shortens an interval to "be safe". + **The SSE side has its own keepalive to size against.** After 30 s with nothing to deliver the server emits an `event: ping`, whose only job is to stop the HTTP/2 proxy in front of the Space from killing an idle connection. A read timeout on our side therefore has to be comfortably more @@ -439,6 +468,23 @@ Four, each cheap to build in now and expensive to rediscover: yet know the robot exists. - **Backoff with jitter, capped.** 5 s growing to 60 s, plus ~10%. A fleet reconnecting in lockstep after a service restart is a self-inflicted outage. +- **A credential that changed under the connection.** The token is read once per connection, so + neither a `logout` nor a `login --force` onto another account reaches a running relay by itself + — it would go on refreshing the lease with a credential its owner deleted. Re-read on the + heartbeat tick, which makes the lag one cadence, and dropping the stream is the deregistration: + a clean disconnect evicts the peer immediately, and the sweep is only for sockets that never + report closing. There is also an explicit `roles: []` withdraw, which is for keeping the channel + open to re-register later and is not what this wants. +- **A 401 is not a case for backoff.** No number of retries fixes a token the service refuses; a + login does. So that path waits on the token file at the same 30 s cadence as a robot nobody has + signed in, rather than posting a doomed request every five seconds into somebody else's Space. + +Two more from their source, about what the server does rather than what to do about it. **Only +producers carrying `meta.hardware_id` are swept**, so one without it is never evicted — a crashed +daemon would haunt its owner's robot list until the Space restarted. That turns §3.7's "should" +into a "must". And **a second `/events` on the same token supersedes the connection without +evicting the peer**, so a daemon restarting while its previous socket is half-open reconnects +cleanly — which is what makes a 60 s read timeout safe rather than merely brave. What we do **not** copy is their `RobotAppLock`: it arbitrates a local *app* against a remote session, and a duck has no app. `remote-webrtc.md` §9 owns the equivalent question here (a pad and a peer both @@ -480,19 +526,28 @@ fills it in arbitrarily gets subtly wrong behaviour rather than a clear failure: So a duck **must** put something stable there, and the obvious candidate already exists: `producer.rs` reads the SoC serial for the local `meta`, which is exactly "stable per physical robot across reinstalls and renames". Leaving the key out means a robot that reconnects with a - fresh token is listed twice; putting the *name* there means renaming a robot forks its identity. + fresh token is listed twice — and, since the sweep keys on this field, never evicted at all; + putting the *name* there means renaming a robot forks its identity. + + Where there is no serial to read — a developer's laptop, a board whose device tree has none — it + falls back to `/etc/machine-id`, stable per *install* rather than per robot. Weaker and still + correct for the purpose: one machine is listed once, and the producer stays sweepable. `sounds` + already makes exactly this substitution for exactly this reason. - **`name`** is what the listing shows a person, and the consumer's `name` is what the server reports back as `activeApp` to the owner's other devices. `transport` (`"wifi"` / `"usb"`) is a mini-ism a duck can leave alone; a `kind` of `microduck` is what lets one client list both families without opening a session. -**And a hazard that belongs in the provisioning path, not here: peers are keyed by token.** -`get_or_create_peer` is a `token -> peer_id` map, and a second SSE connection on the same token -supersedes the first. Two robots sharing one token therefore take turns being reachable, and -neither is broken in a way that looks like a bug. Each duck runs its own device flow, so each gets -its own token — *unless* an image is cloned with `/etc/robot/hf-token` in it, which is exactly what -this project's flashing path does with everything else in `/etc/robot`. Whatever produces a golden -image has to exclude that file, and this is the note that says why. +**And one protocol fact with a consequence: peers are keyed by token.** `get_or_create_peer` is +a `token -> peer_id` map, and a second connection on the same token supersedes the first. Two +things sharing a token therefore take turns being reachable, and neither looks broken. Each duck +runs its own device flow, so no two robots share one — images are built from scratch rather than +cloned, so there is no path by which a credential is copied onto a second board. + + Where it does bite is a **consumer**: a cloud backend that authenticates with the *robot's* token + supersedes the robot's own peer and takes it off the listing. A Space consuming a duck needs its + own token on the same account, or the visitor's. §5's client uses the visitor's, which is why it + never meets this. ## 4. The rendezvous is the one `reachy_mini` uses — **decided** @@ -525,47 +580,381 @@ The one thing that does not transfer is its **lock model** — `RobotAppLock`, l remote session, which the mini's relay gates incoming sessions on. A duck has no app, so §3.4 takes the reconnect behaviour and leaves that part. -## 5. The client, and where it is served — **open** - -The console is `include_str!`'d into `mediad` and served by the robot (`webrtc-console.md` §1), which -works because the client is on the LAN. **A remote client cannot fetch a page from a robot it cannot -reach**, so remote needs the page hosted off-robot *and* a second signalling transport in it. - -Three ways: - -- **The same file, published by CI to GitHub Pages**, with the transport chosen by the URL it is given. - One page, two hosts, still no build step — the constraint `webrtc-console.md` §8 defends survives. - The service stays a rendezvous and nothing about the client lives in it. -- **The service serves the page.** One host and one deploy, at the cost of putting the client inside a - service we may not own; a page in a private repo is a page nobody here can edit. -- **No client yet.** Prove login and the relay with the service's own dashboard, `/api/robot-status` - and `duckctl` — which needs no page at all, and is the whole of slices 1 and 2 in §8. - -**Recommendation: the third, then the first.** The proof that the token path and the lease work needs -no UI, and deferring the hosting decision by a slice costs nothing. - -One thing to know before that page is written, and it is settled rather than a preference: -`EventSource` **cannot set headers**, so a browser speaking the SSE wire must either put the token -in the query string or read the stream with `fetch` and split SSE by hand. `reachy-mini-js` does -the former. **The server is removing it** — `_resolve_hf_token` accepts `?token=` only as a -transitional fallback, logs a deprecation warning per client IP, and says in its own docstring that -the query form goes once the known clients ship the header. A bearer token in a query string is -also a bearer token in the Space's access log and in every proxy in between. - -So the page is `fetch` plus a few lines of line-splitting, not one browser API — and it should be -written that way first rather than written twice. - -## 6. NAT: decide the STUN server, defer TURN - -`webrtcsink` defaults its `stun-server` to a public Google address. LAN sessions need none, so nothing -has exercised it — and the moment remote works, **a duck's reachability quietly depends on a third -party we do not run**. Set the property rather than inherit it, for the same reason -`remote-webrtc.md` §0 sets `congestion-control` to the value that is already the default: the day -upstream changes it should not be the day every robot's connectivity changes with it. - -TURN is what makes symmetric NAT and CGNAT work at all, and it relays the *whole* session's media at -somebody's expense. Not in the first slice, and `remote-webrtc.md` §11 is right that the decision -belongs with whoever runs the rendezvous rather than with the daemon. +## 5. The client is a Space with a Hugging Face sign-in — **decided** + +The console is `include_str!`'d into `mediad` and served by the robot (`webrtc-console.md` §1), +which works because the client is on the LAN. **A remote client cannot fetch a page from a robot it +cannot reach**, so remote needs the page hosted off-robot *and* a second signalling transport in +it. + +**Decided: a Space in the `pollen-robotics` org with `hf_oauth: true`** — `microduck-console` — +serving the same page the robot serves, with the transport chosen by how it was opened. And the +deciding argument is not hosting, it is the **token**. + +A remote consumer authenticates to the rendezvous exactly as the robot does — a bearer token the +service resolves through `whoami-v2`, reading `name` out of the answer. It performs no token-type +check and no scope check, so a browser's OAuth token is accepted on the same footing as the +robot's device-flow one. Which makes the whole question "where does a page get an HF token", and a +Space answers it: `hf_oauth: true` in the README creates the OAuth app and registers a redirect URI +targeting the Space, and `@huggingface/hub`'s `oauthLoginUrl` / `oauthHandleRedirectIfPresent` do +the rest of the flow **client-side**, PKCE, with no secret in the page. + +### 5.0 It is a Docker Space, and that cost an afternoon + +There are two documented ways for a Space to hand its page that client id, and the tidy one did +not work. A **static** Space is supposed to inject `window.huggingface.variables.OAUTH_CLIENT_ID` +— `huggingface.js` reads exactly that (`oauth-login-url.ts`), `reachy_mini_website` reads exactly +that, and HF ships `huggingfacejs/client-side-oauth` as the example. On `microduck-console` it +never appeared: not after a metadata change, not after the Space went public, not after a +delete-and-recreate, and not after the page was given a real `` for an injector to +work on — a page that had been a bare doctype for its whole life until then. Hugging Face's own +API reported our Space and a working one as indistinguishable: both `sdk: static`, both public, +both `hf_oauth: true`, both `RUNNING`. + +So the console is a **Docker** Space: `hf_oauth: true` puts `OAUTH_CLIENT_ID` in the container's +environment — the documented path for anything that is not static, and the one the `grabette-*` +Spaces use — and eight lines of `sh` substitute it into the page as it is served. A page that +needs no server has one, for that reason and no other. `OAUTH_CLIENT_SECRET` is in the same +environment and never goes near the page. + +Two things that came out of chasing it are worth keeping. The page **logs its own build stamp** +(revision plus a hash of the page), because a static host caches and a browser caches harder, and +an hour went into a fix that was never being loaded. And every step of the sign-in **races a +timeout**: a spent `?code=` left in the address bar leaves `oauthHandleRedirectIfPresent` pending +forever, and everything a page would say about that is on the far side of the `await`. + +GitHub Pages, which an earlier draft of this section preferred, loses on the same point: the page +would need an OAuth app registered by hand and its own redirect URI, maintained by us, to arrive +where a line of README metadata arrives. It stays possible — the page takes a client id from +`?client_id=` as well, which is how one is tried before it is written down — and this is the same +reasoning §2.3 used for taking Hugging Face's own device-code client rather than registering one. + +**We ask for no scopes beyond the defaults.** `openid profile` is always included and is all this +needs — the token's whole job is proving an identity to the rendezvous, which is the argument §2.4 +makes about the robot's own credential, and it would be a poor look to fix it on the robot while +handing a browser `write-repos`. + +**Not a route in `reachy_mini_central`.** One host and one deploy, at the cost of putting duck UI +inside the service the mini fleet depends on. §4's "we maintain it, so a duck-shaped need is a pull +request" cuts both ways: the reverse of that is that our page becomes their operational risk. + +**The page's source lives in this repository**, next to the one the robot serves, because it has to +track two things that live here — the signalling protocol and this project's own method names — and +a copy in a Space repo would drift from both. It **is** the one the robot serves: +`mediad/webclient/index.html`, one file, with the transport decided by whether its +`{{SIGNALLING_PORT}}` token was substituted. `web.rs` substitutes it and a test asserts the served +page has none left; the Space copy keeps it, which is how the page knows no robot served it. + +The Space is a deploy target — `pollen-robotics/microduck-console` — and +`scripts/publish-console.sh` is what puts a page there, along with the `Dockerfile` and +`entrypoint.sh` that serve it. It substitutes the API version from `duck-ipc-proto`, stamps the +build, and refuses to publish a page that has lost either the port token (which would make the +Space's copy try to open a WebSocket) or the client-id token (which would leave it unable to sign +anybody in). By hand while there is one Space; by CI when that stops being true. + +One thing settled rather than preferred, and it survives from the earlier draft: `EventSource` +**cannot set headers**, so a browser speaking the SSE wire must either put the token in the query +string or read the stream with `fetch` and split SSE by hand. `reachy-mini-js` does the former and +**the server is removing it** — `_resolve_hf_token` accepts `?token=` as a transitional fallback, +logs a deprecation per client IP, and says in its own docstring that the query form goes once the +known clients ship the header. A bearer token in a query string is a bearer token in the Space's +access log and in every proxy between. So the page is `fetch` plus a few lines of line-splitting, +written that way once. + +What the page is, then, is the mirror of `mediad::relay`: SSE in, `POST /send` out, gst signalling +envelopes with per-hop ids, and an opaque SDP/ICE payload — the same translation in the other +direction, which is why §3.2's table is worth reading before writing it. + +### 5.1 A duck in a mini's client, which is a conversation rather than a commit + +Putting ducks into a rendezvous whose other clients are `reachy_mini_mobile_app` and its desktop +counterpart means a duck can appear in somebody's mini app — and be driven as a mini, with method +names this project does not serve. Video would arrive; nothing else would, and the failure would +read as a broken robot. + +`meta.kind` is there so a client can tell the families apart (§3.7), and this page filters on it. +The mini's clients cannot be made to, from here: that is a conversation with whoever owns them, and +it is the direction §4 did not consider — not "what does a duck need from the service" but "what +does a duck arriving in the service do to its existing clients". + +**A non-browser consumer works today, and its one snag is the channel label.** +`ReachyCentralConsumer` — their aiortc client — connects to a duck through the rendezvous and +decodes frames with nothing added on either side: `pip install "reachy_mini[central-consumer]"`, +`robot_name="olducky"`, and `latest_frame()` returns `(720, 1280, 3) uint8`. Verified. What it says +on the way past is `ignoring unexpected data channel: 'control'`, because it looks for the label +the mini's daemon opens and ours is `control` (`remote-webrtc.md` §5). Nothing for a perception +consumer, which wants pixels — and the first thing to fix for one that wants to *drive* a duck, +where the label is the smaller half of the problem and the method names are the larger. §5.2 is +both halves, on this side. + +**Two of their clients select on `meta.name` and neither reads `kind`.** The host shell's picker +lists whatever is online, and `ReachyCentralConsumer` matches `robot_name` against `meta.name` with +a **fallback**: one visible producer for the token is used whatever it is called. So a cloud backend +written for a mini, on an account whose only online robot is a duck, picks the duck and drives it +with method names this project does not serve. That is worth telling them before somebody meets it, +and it is a two-line change on their side — `kind` is already on the wire. + +### 5.2 A consumer that drives a duck, and the one line of theirs it has to get past + +`spaces/policy-shop` is the second consumer in this repository and the first that *sends* +anything: sign in, list the account's ducks, and put a policy from the Hub onto one in a click — +`policy.fetch`, `robot.setSkill`, `robot.policies`, `robot.do`, which is `robotctl policy add`'s +own order over a datachannel instead of over a unix socket. + +The whole cost of "drive" over "watch" is a label. `ReachyCentralConsumer` handles +`pc.on("datachannel")` with `if channel.label != "data": ignoring unexpected data channel`, and +`mediad` opens `control` (`remote-webrtc.md` §5) — so a consumer that wants pixels needs nothing +and a consumer that wants to send a call gets no channel at all. `RTCPeerConnection` is a pyee +emitter, so `pc.on` *appends* rather than replaces: a subclass overriding `_build_pc` registers a +second listener, theirs still runs and still warns about a label it does not know, and ours takes +the channel it dropped. Nothing in their package is rewritten, which is what makes it survive the +version that fixes their side — the day their handler takes `control`, ours stops being the first +to claim it and the shim becomes a deletion. + +Their `send_command` is reused rather than reimplemented, and that is not laziness: +`RTCDataChannel.send` is not thread-safe, they already marshal every send onto the loop that owns +the peer connection, and a Gradio callback runs in whichever worker thread the request landed in. + +**Two refusals, and which side owns them is most of what writing this settled.** `policy.fetch` +checks the claims that are about the robot — `obs_len`, `action_len`, `model_api`, `robot.model` — +and it checks them *before* the download, so a client should not repeat them and should show what +the robot said. `robot.setSkill` checks nothing about the command encoding: a phase policy +installed as a one-shot is **accepted**, and the robot then feeds a constant to a network trained +on a phase, which is plausible movement and wrong movement. That rule lives in `robotctl`, and +`robotctl` is not in the path of a click — so `catalogue.refusal` is the same rule written a second +time, and any client that grows this button needs it a third. + +**A `401` from the rendezvous is the token and nothing else, and proving that took reading their +`app.py` after guessing wrong.** The first version of this page listed robots with `GET +/api/robot-status`, got a `401`, and concluded the endpoint must require an established peer — +plausible, because `POST /send` does (§3.1's note) and because the robot only ever polls it while +holding a stream. It does not: the route is `Depends(_resolve_hf_token)` and then +`validate_hf_token`, which is one `whoami-v2` call with no scope check, no token-type check and no +peer requirement, filtered to `p.username == username`. Its own docstring says it exists for +exactly this — "a passive status indicator without consuming a session slot". + +Which makes it the *better* call than the console's `list`, for a reason §3.7 already stated: peers +are keyed by token, so the `/events` stream a browser opens to list would supersede the one a +session is riding on. A page listing that way cannot refresh its list without dropping its own +session. `/api/robot-status` opens nothing. + +The `401` was **Gradio's mocked sign-in**. Outside a Space — `SPACE_ID` unset — +`gr.LoginButton` behaves, the profile is real, and `_get_mocked_oauth_info` sets `access_token` to +the literal string `mock-oauth-token-for-local-dev`. A service that resolves tokens through +`whoami-v2` refuses that, correctly, and the symptom is a page saying "sign in again" beside a +console listing the same duck. So a local run has to prefer `HF_TOKEN` or what `hf auth login` +stored, and the mock is recognised by value rather than by an `SPACE_ID` check — a real token that +arrives outside a Space is still a real token. + +**And the general lesson, which is why the page now logs everything to two places at once.** Four +layers meet in one button — a token, a rendezvous, a candidate pair, a robot's own refusal — and +all four fail as "nothing happened". Every HTTP status, every signalling frame, every JSON-RPC +line and every refusal is logged, with the token's *source* named and the token never written +down; `DUCK_LOG=DEBUG` adds the streaming notifications and the per-candidate ICE lines. The panel +is on the page as well as the terminal because a Space has logs nobody has open and a browser has +no stderr. Each layer is also runnable alone — `uv run rendezvous.py`, `uv run catalogue.py`, +`uv run lan.py` — which is what turns "it does not work" into a line number without a +conversation. + +`IntentResult` is the other thing a client gets wrong once: `robot.setSkill`, `robot.do`, +`robot.init` and `robot.relax` answer `accepted: false` with a reason rather than a JSON-RPC +error, deliberately — safety refusing to run a policy on a fallen robot is not a broken call. A +page that only catches errors reports every one of those as a success and leaves a motionless +robot unexplained. + +**And the transport is what this leans on hardest from a Space.** The control channel is SCTP over +the same candidate pair as the media, so a relay that is not there takes the click with it: from a +data centre the session negotiates and may then carry nothing. §6's endpoint answers now, so the +ordinary case is covered — but a metered dependency is still a dependency, which is why the status +line names the stage it reached rather than saying "connecting…", and why the page has a second +way in. + +**`lan.py` is that second way, and it is a transport rather than a second design.** The robot is +already serving `webrtcsink`'s signalling server at `ws://:8443` — the one the console +talks to — and it carries the same gst envelopes the rendezvous carries over SSE and `POST /send`. +So one hop is swapped and nothing above it changes: the same `control` channel, the same JSON-RPC, +the same buttons, and the page holds either consumer without asking which. §3.2's table is the +whole of the difference, and swapping in the direction of the LAN removes rather than adds — no +account, no lease, no rendezvous, no relay, host candidates on both sides. + +Which turns out to be worth more than a workaround for a dead DNS record, and it is the argument +for keeping it after §6 is fixed: **it separates the transport from everything else.** A click +that works on the LAN and not through the rendezvous has told you which layer to look at, and +that answer was previously a guess. + +Its one cost is that it is hand-written where the rendezvous half was inherited: a dozen envelope +shapes read off `net/webrtc/protocol` and the console page, and getting one wrong produces silence +rather than an error. So `uv run lan.py` stands up a producer on loopback that speaks the same +protocol and drives a real session against it — welcome, list, `startSession`, an offer answered, +DTLS, SCTP, the channel, a call matched to its reply. Two aiortc peers on `127.0.0.1` are not a +duck; they are the same protocol, which is the part that fails quietly. + +### 5.3 Frames out of the robot, which is what §6 was blocking + +The goal §5.2's Space was a step towards is a Space **processing this camera on Hugging Face +hardware**, and that is the one thing the control lane cannot carry: pixels are what a media path +is for. Pulling them means WebRTC, WebRTC across two NATs means a relay candidate, and §6 says +there is not one. `vision-demo` had "signalling worked and media did not" as its documented +expected outcome for exactly this reason. + +**So the robot dials the Space and pushes.** `media.stream {url}` — answered by `mediad` itself, +like `media.video`, because the pipeline is `mediad`'s and no service owns it — tells the robot a +`wss://` to connect to; it opens it outward and sends frames. An outbound WebSocket is the one +thing that always works, and the robot is already proving it every second it is reachable at all. +No relay, no ICE, and **the rendezvous carries an instruction rather than payload**, which is the +property that makes this scale where relaying pixels through a service the mini fleet depends on +would not. + + Space ──media.stream {url}──► rendezvous ──► robot + robot ═══════ H.264, outbound wss, direct ═══════► Space + +**H.264 rather than JPEG, and it was JPEG first.** The board has a hardware encoder, so the encode +costs the VPU rather than a core, and prediction is worth an order of magnitude of bytes — 0.5 KB +an access unit against JPEG's 6.5 KB a frame on synthetic content, less on real footage and the +same direction. Two things had to be built to make it safe, and both are the kind that fail +invisibly: + +- **A receiver that joins mid-stream can decode nothing until a keyframe**, and a Space restarts on + every push, so reconnecting is the common case rather than the exception. `h264parse + config-interval=-1` repeats SPS and PPS in front of every keyframe, and opening the valve sends + an upstream `force-key-unit` so the first thing a receiver gets is decodable. +- **Dropping the oldest and keeping the newest is right for JPEG and wrong here.** A predicted + frame whose reference was dropped decodes to garbage that looks like a broken camera rather than + a broken transport. So a gap abandons the stream to the next keyframe, in two places: the + branch's queue and the sender's channel. + +JPEG stays reachable on `media.stream {"encoding": "jpeg"}`, because every frame being independent +is worth having for a receiver that reconnects constantly. + +**The branch is valved, not conditional.** `webrtcsink` owns the video track's encoder and is +handed raw video on purpose — pre-encoded input puts the encoder out of reach of its congestion +control, which this pipeline tried and reverted — so there is nothing to tap and this is a *second* +encoder off the same raw tee. On a board where the encoder is the budget that has to cost nothing +when nobody is streaming, so the branch is built once behind a `valve drop=true` and opened by a +property write. Adding and removing elements on a live pipeline was the alternative, and +`pipeline.rs`'s history with a `videoflip` is why nobody should reach for that here. + +What this does **not** do is give a browser a picture of a robot, carry audio, or close a teleop +loop. Those want WebRTC and §6 is still what they need; the frame stream is for the case where the +consumer is a program. + +## 6. NAT: STUN on both ends, and the robot offers the relay — **decided** + +`stun.l.google.com:19302`, which is `webrtcsink`'s own default and now also what the console asks +for when it is remote. That second half was missing and mattered: a page offering only its +`192.168.…` addresses to a robot on another network negotiates a session perfectly and carries +nothing, because there is no candidate pair that can work. + +**TURN is not optional and it is not symmetric.** Between a robot behind a home router and a +consumer behind whatever a cloud provider gives a container, srflx-to-srflx needs both NATs to +allow a hole to be punched — often they do, and often enough they do not. A relay always works, at +the cost of somebody's bandwidth, which is why ICE tries it last. + +**One relay candidate is enough, and it is not always the robot's.** A connection needs *one*, not +two, and `aiortc`'s STUN client works where its TURN client does not — so a Python consumer cannot +be the side that relays, and the robot has to be. `reachy_mini`'s #1182 established that +arrangement and `mediad::turn` is the same one. + +**What that argument left out is whether the two ends can address each other at all**, and an +iPhone on a mobile network is the case where they cannot. It has no IPv4 socket: it reaches a +*hostname* through DNS64/NAT64, the STUN server reports an IPv4 reflexive address back, and the +phone gathers a candidate saying so. But an ICE candidate is a bare literal, and the robot's relay +candidate is a bare IPv4 literal on a board with no global IPv6 at all — which that phone cannot +send a packet to. Measured on olducky: six sessions, `offering relay candidates relays=5` every +time, `Ice connection state … failed` every time, about eight seconds apart. + +So **the console offers a relay of its own** (`refreshRelays` in `mediad/webclient/index.html`), +and only its own allocation can bridge this: `turn.cloudflare.com` is a name, so it resolves over +IPv6, and the relayed address Cloudflare hands back is IPv4, which the robot can reach. Confirmed +from the phone before it was written — the same credentials in a Trickle ICE page gathered a +`relay` candidate with an IPv4 address over 4G, where the robot's own candidates paired with +nothing. + +The page mints them with **the visitor's** token, not the robot's, which is the right way round +twice over: a robot's allowance should go on being watched rather than on watching, and a browser +signed in with `hf_oauth` already holds a token of its own. A LAN session asks for none — there +are host candidates on both sides and nothing would use a relay. + +The credentials are Cloudflare's, minted per account by a proxy Hugging Face hosts and +authenticated with **the same token the relay signs in with** — so a robot that belongs to +somebody can offer a relay and one that belongs to nobody cannot, which is the same line §2 draws +everywhere else. They are short-lived: a task refreshes at half of a 600 s lifetime, and retries +in thirty seconds after a *transient* failure only. A robot nobody +has signed in has nothing to retry for, and a warning every thirty seconds for the life of the +daemon is how a log stops being read. + +**Fetching them must never be in the way.** The only caller is GStreamer's `consumer-added` +handler, where the SDP offer for that consumer is not generated until the handler returns, so an +HTTP request there would delay every connection — including the LAN ones that will never use a +relay — by however long the proxy takes to answer. `Relays::uris` therefore reads a cache, never +blocks (a `try_read` that yields nothing rather than waiting) and never fails. An empty answer is +the ordinary state for the first few seconds after boot and forever on a robot with no account, +and it means host and srflx only, which is all anything on the same network needs. + +**The proxy is the Space, and the name in front of it was the dead part.** `turn.fastrtc.org` — +what `fastrtc`'s own code points at and what `reachy_mini` #1182 copied into this arrangement — is +a dangling delegation, not an outage: the `.org` registry names four Route53 nameservers for the +zone, the registration is healthy and locked until 2027, and all four nameservers answer `REFUSED` +for the zone they are authoritative for, which is what Route53 says when the hosted zone behind +them is gone. `fastrtc/turn-service`, the Space that alias pointed at, never stopped answering. +So `DEFAULT_TURN_ENDPOINT` addresses it directly, at +`https://fastrtc-turn-service.hf.space/credentials`, and the vanity record is out of the path. + +**That also closes a token-exfiltration route, which is the half that mattered more than the +outage.** A dangling Route53 delegation is a known takeover shape — create hosted zones until AWS +assigns you one of the four delegated nameservers, and one is enough, because a resolver needs +only one authoritative answer. Whoever landed it would serve records for the name, pass DNS +validation for a certificate on it, and be handed the account token every signed-in robot sends +as a bearer header every five minutes. That token is the robot's whole credential; §2.4 is about +how broad its scopes are. + +**The endpoint is checked before the token can reach it.** `turn::parse_endpoint` is a `clap` +`value_parser` on `--turn-url`: `https` unless the host is loopback, no userinfo, no query and no +fragment. A wrong value stops the daemon at argument parsing rather than becoming a warning every +thirty seconds. Redirects need no separate guard — `reqwest` strips `Authorization` when a +redirect crosses scheme, host or port (`src/redirect.rs`, `remove_sensitive_headers`), so the +token cannot be walked to a third-party origin. + +**The allowance is the robot owner's, and it is finite** — 10 GB a month on a free Hugging Face +account. A relayed video session is roughly a gigabyte an hour, so a robot driven hard over a +relay can spend it, after which the proxy answers with no relay servers and `turn.rs` logs "the +TURN proxy offered no relay servers" at info level. That is indistinguishable from a robot that +was never offered one, which is a legibility gap worth closing when somebody hits it. It is also +the argument `stream.rs` makes for sending frames outbound rather than through a relay. + +**Something now notices when it dies, which is the reason this went unnoticed for three months.** +The endpoint was already dead when #1182 shipped it, and the only symptom was a warning in a log +and a candidate type nobody counted. `.github/workflows/turn-endpoint.yml` runs daily: one +authenticated `GET` against `DEFAULT_TURN_ENDPOINT`, asserting 200 and at least one `turn:`/ +`turns:` entry. Every other check in the repository passes regardless, because every one of them +pairs two peers on one network, which never looks at a relay. + +Three things about its shape are deliberate. It reads the URL **out of `turn.rs` with `sed`** +rather than keeping a copy, because a check holding its own endpoint tests whatever it was last +told and can drift from what the daemon compiles in — and a green check on a URL no robot uses is +worse than no check, since it reads as proof. It is **not** on `pull_request`: the failure being +guarded against is "nobody touched this for months", which a PR trigger cannot see, and a third +party's outage must never block unrelated work. And a **missing `HF_TOKEN` secret fails** rather +than skipping, because a check that quietly skips itself into permanent silence is the exact +failure mode it exists to end. + +Two things this deliberately does not do: + +- **Our own proxy.** This endpoint is a small service holding a Cloudflare Calls key and minting + short-lived credentials for a caller with a valid HF token, and running one ourselves would end + the dependency on a dormant project's Space — `--turn-url` is already the seam it plugs into. + Worth doing, and not worth blocking relay coverage on: `*.hf.space` is `{owner}-{space}`, so it + breaks if `fastrtc` renames or removes the Space, and there is no CNAME layer left to repoint. + That is the residual risk, named rather than closed. +- **A Cloudflare key on the robot**, using `TURN_KEY_ID` and `TURN_KEY_API_TOKEN` directly. Fastest + and worst: a long-lived API token on every board, which is the shape of mistake §2.4 exists to + stop making. + +Nothing about a relay is fatal. `add-turn-server` is checked for existence before it is emitted — +a panic in a C closure aborts the process rather than unwinding, which `pipeline.rs` learned once +already — a refused URI is a line in the journal, and a robot that cannot offer a relay is +reachable from most places rather than none. **And a TURN URI carries a password**, so only the +host half is ever logged. ## 7. Authorisation, restated now that there is an account @@ -601,29 +990,51 @@ Five slices, and the first two are independently useful and need no client: three calls, three transports, two CLIs, and a token that renews itself. Verifiable on its own, which is what made it the first slice: it prints the Hugging Face username. 2. **The relay, registering only** — producer registration, the negotiated heartbeat, reconnect and - backoff, the split-brain poll. `mediad`. Verifiable with no client at all: the service's dashboard - counts a producer and `/api/robot-status` lists the duck. -3. **Session translation** — a remote consumer gets video and the `control` channel. The first slice - that needs something to connect *with*. -4. **The client, hosted.** §5. -5. **STUN decided; TURN if a real network needs it.** §6. + backoff, the split-brain poll. `mediad`. **Done**: `mediad::relay`, a task with no GStreamer in + it, inert until `/etc/robot/hf-token` exists and picking it up without a restart when it does. + Verifiable with no client at all: the service's dashboard counts a producer and + `/api/robot-status` lists the duck. +3. **The client, hosted.** §5. Ahead of session translation rather than after it, which is a change + of order and the reason for it is verification: the service ships no front-end of its own — `GET + /` is a status page counting peers, producers and sessions — and its consumers are the mini's + mobile and desktop apps, which would drive a duck with a mini's method names. So there is + nothing to connect with that we do not write, and translating sessions first would mean + building the half that can only be tested against a fake. +4. **Session translation** — a remote consumer gets video and the `control` channel. **Done**: + `relay::bridge` opens `ws://127.0.0.1:<--port>` as a consumer when a session is asked for, + rewrites `sessionId` per hop, reads no payload, refuses a second session by name, and tells the + service when a session ends however it ends — including the case where there is no producer to + bridge to, which is a robot whose pipeline never reached PLAYING. +5. **STUN decided; TURN offered by the robot.** §6. **Done**: `stun.l.google.com:19302` on both + ends, and `mediad::turn` keeps Cloudflare credentials fresh so every consumer's offer carries a + `relay` candidate — from `fastrtc-turn-service.hf.space`, the Space itself, because the alias + in front of it is a dangling delegation. §6 has the argument; the three things to carry away + are that only the robot needs credentials, that reading them must never block the thread + building an offer, and that no check in this repo would notice if that endpoint died again. ## 9. What is open, and who can close it | | needs | |---|---| | §2.4 the scope breadth | one public device-code client in the `pollen-robotics` HF org with `openid profile read-repos`, created by somebody with org admin. Not blocking — a scope change is a re-login — and it should not ship without it | -| §5 where the client is served | follows the shape of §3, and is the decision that actually couples us to a service | +| a calibration for the camera | `media.video` publishes the module's design figures with `calibrated: false`, which is enough to map a room and not enough for metrology. Measuring one robot and writing `[media.intrinsics]` closes it for that robot; a per-unit calibration in provisioning closes it for the family. §11 of `remote-webrtc.md` | +| everything on the wire should be timestamped at source | `remote-webrtc.md` §11: `abs-capture-time` on the media, checked against what `webrtcsink`, a browser and `aiortc` actually surface; and a monotonic-plus-epoch field on every control-channel notification that describes a moment. Wanted for any consumer that has to relate what the robot saw to what it felt — visual-inertial SLAM is the case that makes it concrete — and it wants its own version bump rather than riding along with a transport | | §2.6 `logout` revokes nothing | whether Hugging Face accepts a revocation for the first-party device-code client, checked rather than assumed. Not blocking — signing out stops the robot being reachable, and a stolen board is answered on hf.co — but it is the difference between "forgotten" and "revoked" | +| §6 the relay check needs a token | `.github/workflows/turn-endpoint.yml` exists and runs daily, and fails until an `HF_TOKEN` secret is set on the repository — a Hugging Face token with no scope beyond sign-in, used only to mint TURN credentials. Failing loudly is deliberate; the alternative is a check that skips itself into silence | +| §6 the relay is somebody else's Space | a credentials proxy of our own, holding the Cloudflare key in one place instead of trusting a dormant project's Space to keep its name. `--turn-url` is the seam. Not blocking — the Space answers — but `*.hf.space` is `{owner}-{space}` and there is no alias left to repoint if it moves | Closed since this page was written: the OAuth client (§2.3 — Hugging Face ships one), whether the token expires (§2.7 — thirty days, with a rotating refresh token), which rendezvous to use (§4 — -the mini's), and whether we can read it (§4 — we maintain it; the "private repo" in an earlier -draft was a wrong-name 401). - -One item this page created rather than closed: **a golden image must not carry -`/etc/robot/hf-token`**, because peers are keyed by token and two robots sharing one take turns -being reachable. §3.7. That belongs to whoever owns the flashing path. +the mini's), whether we can read it (§4 — we maintain it; the "private repo" in an earlier draft +was a wrong-name 401), where the client is served (§5 — a static Space with `hf_oauth`, because +the question was never hosting but how a page gets a token), and which relay endpoint to use (§6 — +the Space itself; the alias in front of it is a dangling delegation). + +One item this page created and closed: **peers are keyed by token**, so two things sharing one +take turns being reachable. Not a provisioning problem — images are built from scratch, not cloned, +so no second board ever receives a copy — but it *is* a constraint on consumers: a cloud backend +must authenticate with its own token, not the robot's, or it takes the robot off the listing by +connecting. §3.7. ## 10. Not doing diff --git a/docs/design/remote-webrtc.md b/docs/design/remote-webrtc.md index 60b6188b..d1b1b1f9 100644 --- a/docs/design/remote-webrtc.md +++ b/docs/design/remote-webrtc.md @@ -518,7 +518,58 @@ The alternative was building `mediad` on an arm64 runner like the plugins in two and leaves nobody able to build `mediad` on a laptop — which for the crate that will need the most iteration against real hardware is the wrong trade. -## 11. Deferred, with reasons +### `media.video` says what the picture is, geometrically + +Width, height, the mount rotation — and the camera's **intrinsics**, which is what a consumer needs +to turn a pixel into a direction. Without them a monocular reconstruction is scale-free and its +angles are wrong; SLAM, visual odometry and "how far away is that" all begin here. + +Four numbers and a flag: `fx`, `fy`, `cx`, `cy`, and `calibrated`. They describe the frame **as it +is sent** — unrotated, because nothing on the robot rotates pixels — so a consumer that applies +`rotate` has to rotate these with it, swapping `cx` with `cy`. The flag is not decoration: `false` +means the IMX219 module's design figures (3.04 mm over a 1.12 µm pitch, principal point assumed +central, no distortion model), which is good to a few percent and enough to map a room; `true` +means a measurement: *this* robot's, written into `[media.intrinsics]`, or — the default, since the camera and lens are one part across the alpha family — the family's solve that `robotd-params` ships. A consumer that needs +metrology can tell that it needs to ask. + +**And the key is absent when the geometry is unknown**, rather than present and wrong. That is a +robot streaming a test pattern, or one where `media-ctl` would not set the sensor mode — in which +case the sensor is in its 3280×2464 boot mode, whose field of view is the whole array rather than +the 1920×1080 crop, and every intrinsic would be off by about 1.7×. `mediad::camera` has the +arithmetic and the mode table, including the fact that reading 720p off the sensor would *narrow* +the view to 27° rather than saving anything. + +## 11. Everything on the wire should carry the time it happened — **wanted** + +Nothing this transport carries is timestamped at source today. A frame arrives when it arrives, a +`robot.state` notification arrives when it arrives, and a consumer that wants to know *when* the +robot saw or felt something has only its own clock to go on — which, over a relay on another +continent, is off by whatever the path cost that second. + +That is fine for driving a robot you are watching, and it is the wrong shape for everything a +remote consumer is interesting for. **SLAM is the case that makes it concrete**: monocular SLAM on +a stream with no capture times can be run, and the moment somebody wants visual-inertial — the IMU +this robot already has, at 50 Hz, on the same control channel — the two series cannot be related +except by guessing. Timestamps applied at the far end measure the network, not the robot. + +Two halves, and they are not the same problem: + +- **Media.** RTP timestamps are relative to a random offset, so they order frames and date none of + them. The mechanism for this is the `abs-capture-time` RTP header extension, which carries a + wall-clock capture time per packet and is what a receiver needs to line video up against + anything else. Whether `webrtcsink` will negotiate it, and what a browser and `aiortc` expose of + it, is the thing to check first — a header extension nothing on the receiving side surfaces buys + nothing. +- **The control channel.** This one is ours and cheap: a monotonic reading, plus the boot epoch + that makes it comparable across processes, on every notification that describes a moment. The + cost is a field per message and an argument about which clock — and the answer has to be the + same one the media path ends up dating frames with, or the two series still cannot be joined. + +Not built, and deliberately not started as part of the remote path: it changes what every +notification looks like, so it wants its own decision and its own version bump rather than riding +along with a transport. `remote-access-design.md` §9 carries it as open. + +## 12. Deferred, with reasons - **A WebSocket surface for server-side programs** (`architecture.md` §5.3). Same JSON-RPC, no media stack, `get_frame` returning a JPEG. It is a few dozen lines once §5's routing exists, and diff --git a/docs/design/robotd-design.md b/docs/design/robotd-design.md index 3cf00dfb..5bfe7489 100644 --- a/docs/design/robotd-design.md +++ b/docs/design/robotd-design.md @@ -43,18 +43,18 @@ the hardware does: the v2 board sits on the Dynamixel bus and serves an on-chip quaternion out of the same register block the servos answer at. One board, one code path, no IMU abstraction. It is listed first in the id vector so it answers before the servo burst. -**One owner at a time, and nothing hard-enforces it.** `serialport` sets `TIOCEXCL`, which +**One owner at a time; tty exclusivity alone does not enforce it.** `serialport` sets `TIOCEXCL`, which turns a second *unprivileged* open into `EBUSY` — but `robotd.service` runs as root, because -motor control needs the character devices, and root is not stopped by that flag. So the -exclusion is arranged rather than enforced, and each other claimant is kept off the port -deliberately: +motor control needs the character devices, and root is not stopped by that flag. The daemon +and standalone `init` therefore share an advisory lock, and other claimants are kept off the +port separately: - **The control loop** owns it for as long as the daemon runs. -- **`robotd init`** opens the port itself, and as root it will succeed *while the daemon is - running* — two writers interleaving packets on one bus, which reads as a hardware fault. So - it wants the daemon stopped, and that is exactly why `robot.init` and `robot.relax` exist as - IPC methods (§3.3): the daemon serves both from inside the loop, so nothing else has to open - the bus at all. `init` is the escape hatch for a robot whose daemon is not running. +- **`robotd init`** opens the port itself, but must first take the daemon's endpoint lock. + It holds that lock through the entire ramp: a running daemon refuses `init`, and an `init` + already moving the robot refuses a daemon startup or another `init`. `robot.init` and + `robot.relax` remain the IPC methods (§3.3) for a running daemon; standalone `init` is the + escape hatch for a robot whose daemon is not running. - **`serial-getty@ttyS2`** — Armbian runs a login console on UART2 by default, and an `agetty` holding the port makes every servo invisible to everything else. `scripts/setup-board.sh` masks the unit; `fuser -v /dev/ttyS2` naming `agetty` is how that was found, and it is still @@ -62,6 +62,23 @@ deliberately: - **The runtime**, at a coarser grain: it drives the same bus, so a board runs the runtime or `robotd` and never both, and the units say so with `Conflicts=` (§5.2). +**Socket ownership.** Both entry points acquire `.lock` (normally +`/run/robotd.sock.lock`) before publishing `/run/robotd/identity.json` or opening the bus. +The daemon also binds its listener before starting the control thread, and keeps the lock +through control-thread shutdown and socket cleanup. A refused duplicate must not overwrite +the owner's PID and build: `robotctl health` and updater startup checks read that identity. +For a listener left by an older daemon without a lock, the daemon's bind path removes only +an actual socket that refuses a bounded connection probe; live listeners and ambiguous paths +are preserved. Standalone `init` only locks and never binds or cleans up the socket. + +**Never unlink the lock file, even at shutdown.** The kernel releases the advisory lock when +its file is closed or the process exits, including `SIGKILL`; the file itself stays in place. +Deleting and recreating it could leave contenders locking two different inodes under the same +name. File existence does not mean the lock is held. The lock is per `--socket`, so callers +using the same physical bus must use the same socket setting; separate endpoints remain useful +for independent fake daemons. Older binaries and other tools do not participate in this lock +and must still be stopped before standalone `init` takes over the bus. + ### 1.2 Who talks to `robotd` ```text @@ -256,6 +273,15 @@ saying so: swapped in arrives at 250, so the check is what removes a whole class of "why is it slow on this robot". `shutdown = 52` is the error mask that latches on overload, overheating and input-voltage faults. +- **A swapped-in servo is adopted, not configured by hand.** A new XL330 answers as ID 1 at + 57 600 baud, and neither is used on this bus. So before the register check, `open_bus` pings + the fifteen expected IDs; if *exactly one* is silent, it looks for ID 1 — first at 1 Mbps, + then by reopening the port at 57 600 — writes it the missing ID and then the bus's baud rate, + reopens at 1 Mbps, runs the same register check on it, and reboots it. The reboot is what + clears the hardware-error alert the flash leaves set, which would otherwise hold torque off + until someone pulled the battery. A complete bus pays fifteen pings for this and nothing + else — the 57 600 probe never runs unless a servo is missing. Two missing servos are left + alone: there is no telling which one a fresh servo replaces, and the journal says so. - The position P gain is written with I and D at **zero**, the runtime's `--ki`/`--kd` defaults. These are RAM registers, so a power cycle restores the servo's factory values, and the factory D is not zero: left in place it damps the servo's internal PID and the robot runs @@ -474,7 +500,8 @@ a robot lying on its side, and the wrong one for softening a landing: gravity pa `fall_gravity_z` held for 200 ms *is* the robot on the floor, and the window worth acting in has closed by then. -So `limp_fall` (on by default since it was validated on a robot) runs a second, separate +So `limp_fall` (off by default: the default velstand gait loads no standing network to hand +back to) runs a second, separate detector — `duck_control::fall` — on the rate rather than the position. Projected gravity rotates with the trunk, so `ġ = −ω × g` is exact and comes straight from the gyro in the same 12-byte IMU block; extrapolating it over ~0.3 s says where gravity is heading. It fires when @@ -751,10 +778,20 @@ single last-writer slot would lose. ### 4.2 Params -A TOML file read at startup, **not watched** — live reload comes later. It lives outside +A TOML file read at startup and, for the most part, **not watched**. It lives outside `releases//` so it survives update *and* rollback, next to the updater's own config at `/etc/robot/robotd.toml`. +Two parts of it are watched, and both are exceptions earned by what a restart would cost rather +than steps towards watching the whole file. `padd` stats the file once a second and re-reads +`[pad]` and `[pad_imu_head_control]` when the mtime moves: a binding is changed from a phone, and restarting +`padd` to apply it would drop the pad session and let `robotd`'s deadman zero a walking robot. +`robotd` re-reads `[policy]` — all of it but `mode` and `enabled` — when asked to, which is how +`robotctl policy add` lands a skill without taking motor control away from a standing robot. +Re-reading `[safety]` or `[control]` under a running loop is a different and much larger promise, +and it is still not made. `robotctl configure` knows which of the three answers a key wants, and +a key that says nothing fails a test in `robotctl`. + Belonging to the board rather than the release is what makes a hand-edited policy path stick: the defaults point inside `releases//`, so an ordinary update keeps a policy alongside the binaries trained against it, and deleting the override goes back to that. The file may be absent @@ -959,3 +996,24 @@ path map now does. §4.4. has to reach the board, so prefer pure-Rust crates on that path. *Unverified on macOS:* the cross-build needs an aarch64 sysroot, which a Mac cannot provide, so `cargo board --bins` fails locally there — build the shipped set with `-p updater -p robotd -p robotctl`, or build on Linux. + +## Mapping telemetry (API v24) + +A mapper on the far end of the video — a laptop today, a server later — needs three things from +the robot that `robot.state` did not carry: a clock shared with `tof.frame`, the IMU beyond its +projected gravity, and where the camera and the ToF sensor are. All three are additive. + +- **`t_ns`** on `robot.state` and `tof.frame` is `CLOCK_MONOTONIC` in nanoseconds (`proto::clock`). + `t` and `at_us` stay: they are each daemon's own elapsed time, and a reader that only has one + stream still wants a number that starts at zero. `mediad`'s `media.video` answer reads + `mono_ns` and `real_ns` at one instant, so RTP timestamps — which RTCP sender reports state in + wall-clock — can be put on the same axis. +- **`imu: {gyro, quat}`** is `ImuData` as the loop read it: the trunk IMU, 50 Hz, nothing above + it (`docs/design/robotd-design.md` §IMU). The head IMU on the prototype HAT is not read by + anything yet; when it is, it streams beside `tof.frame`, not here. +- **`frames: {camera, tof}`** are trunk-frame poses at this tick's *measured* head joints from + `kinematics::head::HeadFk` — the same FK `robot.look` solves against — and **`robot.model`** + answers the static geometry (trunk height, joint order, ToF beam directions, the poses at head + zero). The kinematics stay in one crate; a client asks rather than transcribes. + +Cost: three small structs per published tick, only while someone is subscribed; the FK is ~50 ns. diff --git a/docs/design/simulation.md b/docs/design/simulation.md new file mode 100644 index 00000000..ae8ecf39 --- /dev/null +++ b/docs/design/simulation.md @@ -0,0 +1,290 @@ +# Simulation: the same daemons, a body in MuJoCo + +**Status:** built and in use. `robotd --sim`, `tofd --sim` and `mediad --sim-camera` are here, with +`microduck_rl`'s `duck-body` serving the other half; the containers (§8), the ether (§5) and the +per-duck cameras all run from `scripts/duck-sim`. Measured: the daemon holds `50.0 of 50.0 Hz · 0 +missed` against a MuJoCo body, detects a seated boot from the simulator's own joint angles, and +`robotctl robot init` runs the sitstand policy until the duck is upright and stays there; four ducks +in containers sing a full chorale over the ether. How to use it is +[`docs/robot/simulation.md`](../robot/simulation.md); this page is the design. + +The goal is a duck you develop against exactly as you develop against a robot: the same binaries, +the same units, the same `robotctl`, the same `duckctl open` — with the body in MuJoCo instead of on +the desk. Not a mock, and not a test harness. A twin, with a written-down boundary. + +## 1. What using it looks like + +One command to get a duck. After that it is a robot, and you already know the commands. + +**What works today** (`scripts/duck-sim`, one duck, no container): + +``` +scripts/duck-sim # a window opens, the duck stands up, and it is yours +scripts/duck-sim drive # walk it forward, then stop +scripts/duck-sim ctl health # or anything else robotctl does +scripts/duck-sim log +scripts/duck-sim down +``` + +It builds what is missing, writes the params, finds the ONNX Runtime in the RL repo's venv, starts +both halves and stands the duck up. Three things it exists to stop anyone typing, because none is +guessable: policies live at `/opt/robot/daemon/current` on a robot and in this repo on a laptop; +`ort` dlopens a libonnxruntime a laptop does not have; and a unix socket path is capped at ~108 +bytes, so the state directory has to be short. + +**Where it goes**, once the containers are in: + +``` +duck-sim up 4 # four ducks, sharing one MuJoCo window +duck-sim shell duck-a # you are on duck-a +``` + +Inside, nothing is special: + +``` +duck-a # robotctl health +duck-a # robotctl configure +duck-a # robotctl chorale +duck-a # journalctl -u robotd -f +``` + +And from your own shell, exactly as with a robot on the desk: + +``` +duckctl open duck-a +scripts/dev-push.sh microduck@duck-a +``` + +Four rules the harness holds itself to, because a development tool that needs a runbook does not get +used: + +**No setup step anybody has to remember.** `duck-sim up` on a machine that has never run it builds +the rootfs, fetches what is missing and says so — once. Whatever it genuinely cannot do for you, it +prints as the exact line to paste, in the style the rest of `scripts/` already uses. + +**Defaults are the common case.** No count means one duck. No scene means the apartment. No +`--cameras` means no cameras, because most sessions do not need them and they are what costs. + +**Nothing you cannot stop.** Every duck runs as a systemd unit, so `duck-sim down` is a +`systemctl stop` — not a key combination. That rule was bought with a container that had to be +killed from a second terminal because `Ctrl-]` is `AltGr + )` on a French keyboard. + +**A duck is addressable by name.** `duck-a`, `duck-b`, and each gets its own machine-id — so +`robotctl quack` sounds different on each one, since a duck's voice is generated from its serial. +Four ducks in a chorale should not be four copies of one voice. + +## 2. Where the seam is + +`duck_control::io::RobotIo` — six methods, and the only place a simulator is allowed to exist: + +```rust +fn read(&mut self) -> Result; // joints and IMU, one transaction +fn write(&mut self, targets: &JointTargets) -> Result<()>; +fn set_gain(&mut self, kp: u16) -> Result<()>; +fn set_torque(&mut self, on: bool) -> Result<()>; +fn slow_sensors(&mut self) -> Result; // volts, per-joint temperature +``` + +Above it, nothing changes: the 50 Hz loop, the ONNX policies, `Safety`, fall detection, odometry, +kinematics, maploc, every IPC call, all of `robotctl` and `duckctl`. Below it there is one thing — +`DynamixelIo` — and the IMU is not separate from it, because on this robot the IMU is a Dynamixel +node read in the same `sync_read` as the fifteen servos. + +`FakeIo` was already a full implementation of this trait, which is why `cargo test` needs no +hardware. `RemoteIo` is the third. + +**Where a sensor's daemon *is* its driver, replace neither.** `tofd --fake` already synthesises +frames at the loop level, and `tof/src/sensor.rs` says in as many words that the off-board `Sensor` +"is not a fake sensor and must never become one". Simulated depth feeds that existing loop. The same +reasoning will apply to anything else whose driver cannot be separated from its hardware. + +## 3. The body protocol + +TCP, newline-delimited JSON, one request and one answer per call. `duck_control::sim` is the +implementation and carries the reasoning; the short version: + +* **TCP** because a unix path is capped at `SUN_LEN` (~108 bytes), and because the simulator must be + reachable from outside whatever the daemons run in — a container on Linux, a Linux VM with MuJoCo + on the host on macOS. +* **JSON** because a tick is ~1 KB, so 50 KB/s, against being able to read a frame with `nc` and + write the other end in twenty lines of Python. A packed struct shared across two repositories in + two languages is how this project has lost days before. +* **`TCP_NODELAY`**, not as a micro-optimisation: Nagle delays a small write up to ~40 ms, twice the + tick, and it would present as a slow simulator. + +Requests are `{"op":"hello"|"read"|"write"|"gain"|"torque"|"slow", …}`. The handshake carries +`protocol` (currently 1) and is checked in both directions, because the two halves live in two +repositories and "your simulator is old" and "your daemon is old" are otherwise the same symptom. + +**The simulator reports in the robot's own units** — radians, rad/s, mA, and the IMU already +resolved into the trunk frame. MuJoCo knows its own model's ordering, scaling and frames; a +translation layer on this side would be a second place for that to live and drift. + +**A dead simulator is one bad tick.** MuJoCo compiles its model, so changing the number of ducks +restarts it, and the ducks must live through that. A broken connection is an error returned to the +caller and a reconnect on the next call — no backoff thread, because the control loop is the retry +timer. + +## 4. Faking the radio costs nothing + +Presence is already an IPC contract on `robotd`'s own socket: `chorale.subscribe`, +`chorale.beacon` (what to advertise) and `chorale.heard` (what was heard, with an *age* rather than +a timestamp). And `btd` is a **client** of robotd, not a server — so a simulated ether impersonates +nothing and steals no socket path. One process holds one connection per duck, collects what each +wants to advertise, and delivers it to the others with an RSSI derived from the distance between +their bodies. + +No change to `robotd`, no protocol work, and three things better than the radio for development: +the age-based synchronisation path is exercised for real; RSSI from ground truth makes range cutoffs +and asymmetric links a knob rather than a staging problem; and `from` is documented as an identity +for de-duplication only, so rotating it on a timer turns the address-rotation bug that cost a day +into a regression test. + +## 5. The radio has to be bad, or the bug hides + +A perfect ether hides the bugs a real one causes, and this is measured rather than argued. Four ducks +in the twin converged on one piece every time — simultaneous starts, staggered starts, no difference +— because every duck was visible to every other instantly and losslessly. The one property that +causes the field bug was the one the simulator did not model. + +`duck-ether --discovery --loss --seed ` makes it a bad radio: a duck takes a while to be +*noticed*, per pair and timed from when it goes on the air, and a fraction of deliveries is dropped. +Per pair because it is the asymmetry that splits a flock — one delay shared by everybody cannot +produce it. Seeded, because a flaky radio is only useful for debugging if its flakiness repeats. + +**The reproduction**, four ducks, `a` and `b` singing twelve seconds before `c` and `d` join: + +``` +duck-ether --discovery 90 --loss 0.3 --seed 3 +``` + +On main's chorale: + +| duck | part | bar | roster | +|---|---|---|---| +| a | *not singing* | — | 1 in range | +| b | bass | 4 | 3 voices | +| c | alto | 4 | 2 voices | +| d | bass | 2 | 2 voices | + +Which is the field report — "sometimes nothing happens, sometimes two different songs" — with +disagreeing rosters and a duplicated part as well. + +**What it does not yet prove.** With `chorale-election` merged the same scenario still splits (bars +5, 12 and 8: three timelines). But at `--discovery 20 --loss 0.4` *both* converge, so ninety seconds +of discovery is harsher than that branch was written for, and this is not evidence the fix fails on a +robot. The experiment worth running is a sweep — the discovery value at which each version stops +converging — which is now a loop over a number rather than four robots and a room. + +## 6. Architecture follows the host + +Every artifact this project builds is aarch64. The attractive idea was to run the board's own +binaries on an x86 laptop under `qemu-user` — same bytes, perfect provenance. It was measured, twice, +and the two results point opposite ways. + +**The daemon alone is fine.** CI's real aarch64 artifact, emulated on an x86 laptop: + +| | | +|---|---| +| `robotctl health` | `50.0 of 50.0 Hz · 3804 ticks · 0 missed` | +| host CPU | 4.7% of one core, nine ONNX sessions loaded, policy driving | +| policy inference | 0.029 ms against a 20 ms tick (measured natively) | + +Throughput was never the risk — and the slow part of a real tick, the Dynamixel sync-read, is a +local socket here. + +**Under systemd it is not.** Booted in `systemd-nspawn`, aarch64 systemd 257 comes up in 8.7 s and +then cannot start anything: + +``` +robotd.service: (code=exited, status=226/NAMESPACE) +systemd-journald.service: (code=exited, status=243/CREDENTIALS) +systemd-logind, systemd-tmpfiles, console-getty: the same +``` + +qemu-user 8.2 does not translate the new mount API (`fsopen`, `move_mount`, `open_tree`) that +systemd 257 uses for per-unit namespaces and credentials. Per-unit hardening is the entire reason to +boot a container rather than run seven processes in a terminal, so losing it loses the point. Newer +qemu may fix it; in Debian 13 and Ubuntu 25.04 the static packages are transitional and the real +binaries are dynamically linked, so it is a build-from-source question rather than an apt line. + +So the twin runs **the host's architecture**: + +* **x86 host** — an amd64 container and a native build. Real units, real hardening, real journal; + everything except CI's exact bytes. +* **arm64 host (Apple Silicon)** — an arm64 container running the robot's own signed artifact, + natively. Full provenance, no emulation, identical commands. + +The consequence for the update path is small and worth stating plainly: `robotctl update apply` runs +end to end — preflight, signature, artifact hash, compatibility, health gate, auto-rollback — because +that is how `dev-push.sh` installs. What an x86 twin cannot do is install a *published* release. +`board-test.sh` already covers the real artifact on the real architecture in CI. + +## 7. What it is and is not a twin of + +Identical, because it is the same code on the same path: the control loop, policies, safety, fall +detection, kinematics, odometry, maploc, the whole IPC surface and its clients, the chorale's +election and beat, the systemd units with their real `User=`, groups, `RuntimeDirectory=` and +hardening, and the updater. + +Modelled — the real code path, synthesised input: actuator response (BAM models fitted to the real +XL330s), the IMU, ToF depth, RSSI, the camera image, and release provenance on an x86 host. + +Absent — not exercised at all: the Dynamixel bus driver, the BLE radio, the camera ISP and rkaiq's +3A, the NPU, the hardware encoder and its RGA path, thermals and battery. + +**A useful check on that list:** run a week of real bugs past it. A `videoflip` that cost 22 fps by +breaking the encoder's zero-copy path to the RGA; a 3A engine missing a stream-start event; an +auto-exposure loop that converges once and stops; an INT8 head whose score channel collapses to two +values; a servo bus dropping reads. The twin would have caught **none** of them. That is not a flaw +in the design, it is the boundary of it — and hardware stays the only place the drivers are real. + +## 8. The harness + +One MuJoCo process, one window, N duck bodies in one scene, so ducks share physics and can bump into +each other. `microduck_rl` owns that half: it already has the scenes, the BAM actuator models and +mjlab, and serving a body to a daemon is the mirror of the sim2real it does today. + +Each duck is a `systemd-nspawn` container — no daemon, no image format, a container is a directory — +booted from one Debian 13 Trixie rootfs (`mmdebstrap`, unprivileged, 238 MB, under three minutes) +with an overlay per duck. Run each as a service (`systemd-run --unit=duck-a …`) so `duck-sim down` is +a `systemctl stop`: a duck you cannot quit is not much of a duck. + +`machinectl shell duck-a` is the way in, `duckctl open duck-a` the way to watch, and +`scripts/dev-push.sh microduck@duck-a` the way to install a build. + +Two limits that follow from the physics rather than the plumbing. **Ducks do not hot-join** — MuJoCo +compiles its model, so changing the count restarts the simulator, which is why `RemoteIo` reconnects. +And **camera rendering is the scaling wall**, not the bipeds: N offscreen renders at 30 fps cost far +more than N fifteen-DoF bodies, so cameras should be opt-in per duck. The 45 Hz health gate makes +both a hard edge rather than a soft one — too many ducks and they do not get slow, they go +*unhealthy* and the updater starts rolling releases back. + +## 9. Three ways to pick the wrong model + +Each of these presents as "the duck is on its back", and each cost an hour. + +**`scene_walk.xml` has no collisions.** It includes the model the RL work trains against, whose +actuator default classes carry `contype="0" conaffinity="0"`. The duck sinks through a floor the +scene really does contain — trunk z from 0.120 to -0.105 in a second — and the daemon reports a robot +lying down, correctly. `scene.xml` includes `robot_allcollisions.xml`, and is what a twin wants. + +**`qpos0` is not a pose.** Every joint at zero is a shape this robot is never in; the daemon measured +0.41 rad from its home frame and quite reasonably tried to stand up a robot that was already folded. +The scenes carry `INIT`, `STAND`, `SIT` and `FOLD`, and `STAND` matches +`duck_control::DEFAULT_POSITION` — whose right leg is *mirrored*, not symmetric, which is worth +reading rather than assuming. + +**Torque belongs on at startup.** `robotd` never enables torque when it starts, because a daemon +restarted by an update must leave a standing robot standing — the servos are already holding. A +simulator that starts limp has its duck on the floor before the first read. + +The boot the daemon is actually written for is `--keyframe SIT`: a duck found folded, which it +recognises and stands up with the sitstand policy. + +## 10. Known material to reuse + +`~/MISC/microduck_maploc` (outdated in every other respect) has two things worth taking: a simulated +VL53L5CX in `sim/tof_sensor.py` — 8×8 zones, 45° square FoV, 4 m range, noise that grows with +distance, 15 Hz — and `sim/assets/apartment.xml`, an indoor scene of 83 geoms. A room is worth much +more than a ground plane to maploc, to wander, and to anything that hides. diff --git a/docs/faq.md b/docs/faq.md new file mode 100644 index 00000000..ef026610 --- /dev/null +++ b/docs/faq.md @@ -0,0 +1,83 @@ +# FAQ + +Questions that come up when you want a duck to do something it does not do out of the box. The +design docs say how the machinery works; this says which piece to reach for. + +## I want to run a model that is too heavy for the board + +Run it somewhere else and send it the camera. The board is a Radxa Zero 3 — the NPU takes a small +detector and little more — so anything larger belongs off the robot, and a Hugging Face Space on +paid hardware is the path with the least to build: it already has a GPU option, an account system +the robot shares, and a URL. + +**Use WebRTC.** The robot publishes H.264 over a WebRTC session, your Space consumes it, and the +rendezvous introduces the two so neither needs to know the other's address. `spaces/vision-demo/` +is the worked example. + +``` +your Space ──sign in, list robots──► rendezvous ◄──registers── the duck +your Space ◄═══════════ H.264 over WebRTC, media and control ════════════► the duck +``` + +Three things you get for free by staying on WebRTC, and they are the reason it is the default: +the media is encrypted end to end by DTLS-SRTP, the control channel rides the same session so you +can *drive* as well as watch, and there is a return path — audio, or a second video track — the +day you want one. + +## Does it work from a data centre? My Space is behind a NAT I do not control + +Yes. A robot behind a home router and a container behind a data centre's NAT usually cannot +hole-punch to each other, so the robot offers a **relay candidate** — short-lived Cloudflare +credentials it mints with its own account token — and your consumer uses it without holding any +credentials of its own. `remote-access-design.md` §6 is the mechanism. + +Two caveats worth knowing before you are surprised by them: + +- **A relay costs the robot owner's bandwidth**, metered per Hugging Face account at 10 GB a + month on the free tier. A continuous 720p stream is roughly a gigabyte an hour, so a Space that + watches all day will spend it. ICE only relays when it must — a direct pair is used whenever + one can be found — but "must" is common between a home and a data centre. +- **A browser consumer needs relay credentials of its own** when it has no IPv4 of its own; a + Python one does not. §6 has the case. + +## My consumer is a program and the stream runs all day. Is there something cheaper? + +There is a fallback, and it is a fallback rather than a second design: **`media.stream`**, where +the robot dials *your* WebSocket and pushes frames outbound. + +``` +your Space ──media.stream {url: "wss://…/frames"}──► rendezvous ──► the duck +the duck ══════════ frames, outbound wss, direct ══════════════► your Space +``` + +It costs nobody's relay, because an outbound connection from the robot needs no hole punched. Use +it when all three are true: the consumer is a **program** and not a person, the stream is +**long-running** enough for relay metering to matter, and you need **frames only**. + +What you give up by leaving WebRTC, which is why it is not the default: + +- **No return path at all.** Nothing reaches the robot on this transport — no audio, no second + track, no teleop loop. +- **No control channel.** Driving the robot means a JSON-RPC call over the rendezvous + (`spaces/shared/wire.py`), separately. +- **Encryption is your TLS, not DTLS-SRTP**, and it terminates at your server rather than at the + peer. + +The robot sends one text **hello** describing what is coming — `frames.encoding` is `h264` or +`jpeg` — then one binary message per frame. Branch on the hello rather than sniffing the bytes; +`spaces/vision-demo/receiver.py` is the reference receiver and `decoder_for` is the branch. + +## How does my Space find the robot, and what stops somebody else's reaching it? + +The account. The robot registers with the rendezvous holding its own Hugging Face token, your +Space signs the visitor in with `hf_oauth`, and the service only ever shows an account the robots +it owns. Nothing is configured on the robot and no robot-side gate is involved — +`remote-access-design.md` §7 is the argument for why that is enough. + +`spaces/shared/rendezvous.py` is the listing call: `ducks(token)` gives you what that account can +reach. + +## Can I test without a robot? + +`spaces/vision-demo/fake_duck.py` stands in for one, and `scripts/duck-sim` runs the real daemons +against a MuJoCo body ([`robot/simulation.md`](robot/simulation.md)). diff --git a/docs/policy-manifest.md b/docs/policy-manifest.md index a3e77cb4..321265d0 100644 --- a/docs/policy-manifest.md +++ b/docs/policy-manifest.md @@ -3,7 +3,7 @@ What a `manifest.json` beside a microduck `.onnx` says, and what the robot does with each field. One vocabulary for two shapes: a **single-policy repo** (`/microduck-` on the Hub, one `policy.onnx`, the fields at the top level) and the **official set** -(`pollen-robotics/microduck-policies`, nine files, the same fields once per entry under +(`pollen-robotics/microduck-policies`, ten files, the same fields once per entry under `policies`). One reader understands both, and asking a publisher for something is "add a field", never "adopt our format". @@ -67,6 +67,10 @@ a policy only on a claim that is present and wrong. | `eval` | object | display | free-form: what was checked, in what sim, how it did | | `policies[]` | array | set | one entry per file, with `file` plus any per-policy field above | +Recurrent LSTM exports require `model_api: 2`; existing feed-forward exports remain API 1. +See [recurrent policies](recurrent-policies.md) for the tensor contract, memory lifecycle +and offline rehearsal. + ## A single-policy repo ```json diff --git a/docs/project/idle-cpu.md b/docs/project/idle-cpu.md index d6e36861..12375bc7 100644 --- a/docs/project/idle-cpu.md +++ b/docs/project/idle-cpu.md @@ -4,14 +4,15 @@ A duck sitting on a desk with no pad connected, no viewer, and nothing driving i five daemons. This is what they were each doing in that state, what was changed, and — for the two candidates that were not — the numbers that say why. -Everything here is arithmetic on the code or a measurement on this machine. **Nothing in it has -been measured on a board**, and the last section says what that would take. +Everything here was arithmetic on the code or a measurement on this machine. **Only the test +pattern has been measured on a board**; the last section says what the rest would take. ## What was costing something | | idle before | idle after | |---|---|---| | `mediad` raw branch | 1.84 MB copied 30×/s | copied when a reader asks — ~2×/s | +| `mediad` test pattern | 720p30 drawn by the CPU — 29.4% of a core | 256x144@5 — 0.7% of the pixel rate | | `tofd` frame poll | ~100 I²C reads/s | ~45 | | `padd`, pad connected, sticks centred | 50–100 msg/s | 10 | | `pet-detect`, when enabled | ~400 FFTs/s + 4 forward passes/s | none until the room makes a sound | @@ -19,7 +20,8 @@ been measured on a board**, and the last section says what that would take. ### `mediad` copied every frame The appsink callback copied every buffer off the tee into a slot readers took the latest of. -At 720p30 — the shipped quality, and `media.camera` defaults on — a UYVY frame is 1 280 × 720 × 2 +At 720p30 — the shipped quality, and `media.source` defaults to the camera — a UYVY frame is +1 280 × 720 × 2 = 1.84 MB. The readers are auto-exposure at 2 Hz and the duck detector at 2 Hz when enabled, so twenty-eight of every thirty copies were made for nobody: 55 MB/s of memcpy and a 1.8 MB allocation thirty times a second, from boot. @@ -28,6 +30,30 @@ A reader now asks and the callback answers the *next* frame. Answering with the would be cheaper again and would put the reader's own polling period into the measurement — an exposure loop steering on half-second-old luma hunts rather than settles. +### The test pattern was drawn at the camera's resolution + +A board with `[media] source = "test"` streams `videotestsrc` instead, so that a robot with no +camera still has a session — signalling, negotiation, the datachannel, the control API, all of +which ride the video track. It ran at `[media] quality`, the same rung a camera streams at. + +The difference is where the pixels come from. A camera's frames arrive off the ISP in hardware and +cost `v4l2src` almost nothing to hand on; a test pattern's are drawn by this process, one packed +UYVY frame at a time. Two boards, same model, both idle with nothing connected: + +| source | idle `mediad` | +|---|---| +| `Camera`, rkisp | 6.1% | +| `Test`, 720p30 | **29.4%** | + +Five times the cost of the thing it stands in for, to synthesise 1.84 MB thirty times a second for +a tee whose readers had all said no — `camera.json` on that board read `"frames":23142` against +`"consumers":0`. + +The pattern now runs at `TEST_PATTERN_GEOMETRY` — 256x144 at 5 fps, 16:9 and 8-aligned like every +`Quality` rung, 73 KB a frame. That is 0.7% of the pixel rate, and it is not a compromise: nothing +in what the pattern is *for* wants resolution. `media.video` reports the small geometry, because +that is what the robot is producing. + ### `tofd` polled a 15 Hz sensor 100 times a second `data_ready` every 10 ms across the whole 66 ms between frames: about seven I²C transactions to @@ -35,8 +61,11 @@ find one frame, six of them answered no. The loop now sits out the stretch in wh cannot yet have one and polls through the rest. Frame age is unchanged — still bounded by the 10 ms poll, which is the granularity a frame is noticed at either way. -This runs on every duck with a ToF fitted whether or not anyone uses the theremin, because -`robotd`'s depth reader subscribes at startup rather than when the instrument is picked up. +This runs on every duck with a ToF fitted whether or not anyone uses the theremin: `tofd` +ranges continuously and sends to whoever happens to be subscribed, so no subscriber is the +normal state and none of this poll is conditional on one. `[theremin] enabled` is off by +default and does not change the figure — what it saves is `robotd`'s parked read, which was +never the cost here. ### `padd` re-sent the same three zeros fifty times a second @@ -107,7 +136,12 @@ measurement on a developer's machine. The four changes want confirming where the is: - `mediad`, `tofd` and `padd` CPU before and after, from `dev-push.sh` and `top -H`. The camera - one should be the visible change. + one should be the visible change. What exists so far is `mediad` idle on 0.12.0 — 6.1% with a + camera, 29.4% with the 720p test pattern — which is an after-figure for the raw branch with no + before-figure beside it, and nothing at all for `tofd` or `padd`. +- The test pattern at 256x144@5 on the board that measured 29.4% at 720p30. The arithmetic says + 0.7% of the pixel rate; what it actually leaves is `videotestsrc`'s per-frame overhead, which + no longer scales with the picture. - SoC temperature at idle over ten minutes, which is the number the whole exercise is for. The `videoflip` episode took this board to 97 °C and throttled it to 408 MHz, so idle headroom is what decides whether a duck walks well while it is also looking at something. @@ -116,3 +150,10 @@ is: doing it. - That petting still starts as promptly as it did, on a robot in an ordinary room rather than a silent one. + +## What came after + +The head sensors were the loose end here, and a `ps -L` on a board settled them: `tofd`'s idle ~5% +is 4.5% head IMU and 0.5% depth, and the IMU has no consumer in the tree at all. So the poll this +page made cheaper was never the cost, and the thing worth turning off is the sensor nobody asked +for. See [`tof-on-demand.md`](tof-on-demand.md). diff --git a/docs/project/npu-bringup.md b/docs/project/npu-bringup.md index 326f4c8a..a1cb7607 100644 --- a/docs/project/npu-bringup.md +++ b/docs/project/npu-bringup.md @@ -8,6 +8,32 @@ here as a quantised `.rknn`. First model, for reference: `yolo11n` at 320×320, from three sessions, mAP50 0.976 on a held-out session — and 3.9 MB after INT8 quantisation, which kept 2 of 2 detections at 95% box overlap against the float model on the desk. +## Where the model comes from + +**The Hub, the way the policies do.** `duck_detector` publishes every run to +[`pollen-robotics/microduck-duck-detector`](https://huggingface.co/pollen-robotics/microduck-duck-detector) +— `duck_detect.rknn` for the NPU and `duck_detect.onnx` for the CPU fallback, at the repo root under +fixed names, one tag per run. Nothing in this repository carries the weights: `mediad` reads them +from `/opt/robot/detector/current`, and what fills that is + +| | | +|---|---| +| `scripts/seed-detector.sh` | run by the release's postinstall hook; installs the pin in `[workspace.metadata.detector]` on a board that has nothing, and never touches a set it did not install | +| `robotctl duck-detector check` | what is installed against what the repo offers | +| `sudo robotctl duck-detector update [--version ]` | installs a revision and restarts `mediad` onto it | + +It is `seed-policies.sh` and `robotctl policy check/update` with a different root and a fixed file +list, served by the same `updaterd` calls (`detector.check`, `detector.install`), and +`docs/design/policy-channel-design.md` §9 has the reasoning that carries over: the pin is a floor, +nothing partial goes live, a retrain is a tag rather than a daemon release. + +Two things worth knowing. The model repo **shares its name with the dataset repo**; the robot only +ever addresses the model (`…/resolve//…`, `api/models/…`), and the dataset lives under +`datasets/`, so nothing on the robot can land on a frame by accident. And `update`'s "newest" is +decided by **version tags** (`v2` sorts above `v1`; a name like `experimental` never counts), so a +run meant for robots wants a `vN` tag — the first run was tagged `duck-v1`, which is what the pin +names and is fine to install by name, but is not a version `check` can rank. + ## What is here | | | @@ -99,12 +125,16 @@ that as the price of perception, the two should be measured apart. ## What is still missing -**Nothing on the robot can get a frame.** `mediad` has a raw NV12 tee branch that exists precisely -for this — `architecture.md` §5.3 — but no IPC exposes it, which is also why capturing a dataset has -to stop `mediad` to take the camera. Two ways forward, and they are not exclusive: +`mediad` has a raw tee branch that exists precisely for this — `architecture.md` §5.3. Two ways +forward, and they are not exclusive: -- **`media.frame`**: a call that answers with one frame. Useful for far more than perception (a - snapshot in the console, a still for a bug report), and it makes capture stop fighting the daemon. +- **`media.frame`** — **done.** A call that answers with one frame, on `mediad`'s own unix socket + (`/run/mediad/media.sock`), group-readable like the other observation sockets. Useful for far + more than perception (a snapshot in the console, a still for a bug report), and it means capturing + a dataset no longer has to stop `mediad` to take the camera. It answers a JSON-RPC header naming + a byte count, then those bytes: a raw frame is ~1.8 MiB, which is not something to base64 into a + control reply. It asks the tee for the *next* frame rather than taking a cached one, so a reader + cannot be handed the frame a stopped camera stopped on. - **The detector inside `mediad`**: subscribe to the raw branch, run the model at a few Hz, and publish detections on the state stream. This is where it ends up — perception next to the sensor, deriving features rather than shipping pixels — and it is what a behaviour would consume. @@ -112,3 +142,41 @@ to stop `mediad` to take the camera. Two ways forward, and they are not exclusiv Once detections exist as state, the behaviours in `docs/ideas/autonomous_behavior.md` that currently key on Bluetooth ("a duck is *nearby*") can key on sight ("a duck is *there*"): approaching, following, facing, and a chorale where the ducks look at each other while they sing. + +### Taking a snapshot + +On the robot, `robotctl frame --output frame.uyvy` saves one fresh packed UYVY frame and prints +its JSON metadata to stderr: width, height, bytes, capture timestamp, and `rotate` — degrees +clockwise the camera is mounted from upright, the same number `media.video` tells a WebRTC peer. +Use those dimensions when converting it, and apply that turn, for example `ffmpeg -f rawvideo +-pixel_format uyvy422 -video_size 1280x720 -i frame.uyvy -frames:v 1 -vf transpose=1 frame.png` +for a 1280×720 capture off a 90° mount; do not assume either number after changing the camera +mode or the mount. The file is written only after the full response arrives. + +**The pixels are the ones the sensor delivered, and `rotate` is reported rather than applied** — +the pipeline stopped turning frames because `videoflip` cost the encoder its zero-copy path and +the board 22 fps, so every consumer turns for itself. Omit the `-vf` above and the picture is +sideways with nothing in it to say why, which is exactly what `rotate` exists to prevent. It is +`0` when `--flip-in-pipeline` already turned them. + +From a browser on the robot's LAN, open `http://:8080/frame`, or save it with +`curl --fail http://:8080/frame -o frame.png`. This returns an **upright** PNG, with +`Cache-Control: no-store`. That route is the exception to the paragraph above, because a PNG has +nowhere to carry an angle: a quarter-turn mount swaps its width and height against the capture +geometry, and the cost is one rotation per request rather than one per frame. A stopped or unavailable camera returns HTTP 503, never the +last good picture. PNG preserves the RGB conversion without JPEG compression; it is not a +byte-for-byte replacement for the raw UYVY data. This has the same LAN access boundary as the +existing console and camera stream; there is no additional authentication on this route. + +The local endpoint supports `hello`, then `media.frame` on the same connection. Its socket is +configurable with `mediad --frame-socket ` and `robotctl --media-socket frame`. +Failure to claim the socket aborts startup; an existing file or live listener is left intact. +There are at most 16 local connections, each limited to five seconds, and at most four HTTP +snapshot jobs. HTTP capture has a three-second deadline and response metadata is capped at +4 KiB. Invalid geometry and payloads larger than 16 MiB are refused by both clients. + +`media.frame` intentionally has no `Call`/service-lane route: its JSON header is followed by a +binary tail. Neither the WebRTC control datachannel nor `duckctl`'s current BLE transport +carries snapshots. Use the local socket or the console's HTTP route; remote video transport is +separate. A future shared socket-group helper can replace the existing duplicated ownership +code without coupling this feature to a multi-daemon refactor. diff --git a/docs/project/tof-on-demand.md b/docs/project/tof-on-demand.md new file mode 100644 index 00000000..806d5c81 --- /dev/null +++ b/docs/project/tof-on-demand.md @@ -0,0 +1,182 @@ +# The head sensors when nobody is looking + +`tofd` ranged a laser and read an IMU from boot to shutdown on every duck with the head module +fitted, whether or not one process was subscribed to either stream. This started as "start the +unit when it is needed and stop it after" and ended somewhere much smaller, by measuring at each +step: **the depth stream is not what costs anything, the sensor that does has no consumer at all, +and nothing inside its loop was worth optimising.** So the laser and the unit were left alone and +the head IMU got a switch, off by default. + +It follows [`idle-cpu.md`](idle-cpu.md), which took `tofd`'s poll from ~100 I²C reads a second to +~45 and left the question of *why it polls at all* open. + +## What it costs, measured + +On olducky (Radxa Zero 3, RK3566), an idle board with nothing subscribed to either stream: + +```text +$ ps -L -o tid,comm,pcpu -p "$(pidof tofd)" + TID COMMAND %CPU + 12209 tofd 0.0 + 12212 tof-sensor 0.5 + 12213 head-imu 4.5 +``` + +Percentages are of one core, so the ~5% `top` shows for `tofd` is 1.25% of this SoC — and it is +**nine parts head IMU to one part depth**. The socket-serving runtime, which has nothing to serve, +costs nothing. + +That split is the whole of this document. Two things follow from it. + +**The IMU number is the two reads a sample takes.** Not the wakeups, and not the fusion. The +crate's `bench_imu` runs the loop five ways, 10 s each at 100 Hz, on this board with `tofd` +stopped: + +```text + sleep only (0 reads) cpu 0.69 % + filter only (0 reads) cpu 1.00 % + update (2 reads) cpu 3.35 % + update_all (2 reads) cpu 3.86 % + three-reads (3 reads) cpu 4.20 % +``` + +Being woken a hundred times a second is 0.69 points; the Madgwick update adds 0.31; the two I²C +transactions are the remaining ~2.4–2.9. A gyro sample and an accelerometer sample *are* twelve +bytes at two addresses, so there is no version of a full sample that costs fewer than two +transactions — which is why nothing in this loop is worth optimising. + +Two caveats on those numbers, since they are the basis for a decision. `update` and `update_all` +do identical work (one delegates to the other) and came out 0.51 points apart, so the noise floor +is around half a point — CPU frequency scaling on an A55, most likely. And an earlier run of the +same three read modes, with `tofd` still holding the bus, compressed them to 4.44/4.46/4.53 and +made the third read look free. Neither changes the shape: the reads dominate, the floor is small, +the third read is worth somewhere between 0.3 and 0.9 points. + +**Nothing subscribes to it.** `head_imu.stream` has no consumer in this tree: `btd` declines to +proxy it (`btd/src/route.rs:412`), the updater's degraded IPC declines it +(`updater/src/ipc.rs:819`), and no daemon subscribes. It was added for the mapping work, which has +not arrived. So the largest recurring cost in this daemon is a sensor read for nobody. + +Worth saying plainly what this is *not*: the walk policy's IMU is a different chip on a different +bus — the `imu_to_dxl` v2 board on the Dynamixel bus, read in the same 50 Hz `sync_read` as the +fifteen servos (`duck-control/src/bus.rs`, `duck-control/src/model.rs:76`). Nothing here touches +it, and nothing here can cost the policy an IMU sample. + +## What was decided + +**`[head_imu] enabled`, default off.** Nothing in the loop paid off, so the switch is the answer: +a stream nothing subscribes to should not cost ~4% of a core from boot. It lives in +`robotd.toml`, where `mediad` already reads `[media]`, so `robotctl configure` writes it and +offers the `tofd` restart; `tofd --imu` reads the chip for one session without touching the file, +and a subscriber while it is off gets a reason naming the key rather than the silence an unfitted +sensor gives. Turn it on when the mapping work reads the stream. + +**The redundant third read went anyway**, as bmi088-rs#1 and a `tofd` that takes the sample from +one call. It is worth 0.3–0.9 points, which is inside the noise of the table above, so it landed +for the other reasons: the published `accel` becomes the sample the quaternion was computed from +rather than one read ~200 µs later, and a read can fail in one place instead of two. + +**The rate is the remaining lever, and it is linear** — `--imu-hz 50` halves the read cost. Left +as a flag rather than a config key: the first real consumer decides what rate it needs, and +guessing now would be a number nobody chose. + +**If a consumer ever needs 100 Hz cheaply, the answer is the FIFO.** Both chips have one, so ten +samples could arrive in one transaction instead of twenty — the only change that attacks the term +that actually dominates. It costs frame parsing, watermark configuration and up to 100 ms of +latency, which is why it is not being built for a consumer that does not exist. + +## The gating that was designed instead, and is no longer needed + +Kept because the reasoning is what led to the switch, and because step 1 is a bug regardless. + +### 1. Make "somebody wants this" true — a bug either way + +`accept` subscribes a connection to **both** channels before it has read a byte of the request +(`tof/src/main.rs:612`), so `receiver_count()` counts connections, not interest: a client that +asked for `head_imu.stream` holds a depth receiver, and one that connects and says nothing holds +both. + +Move the `subscribe()` calls inside the matched arms of `subscriber()`, so a receiver exists only +where the method is known, and hand the function the two `Sender`s instead of two `Receiver`s. This +is worth doing on its own — a connection that has asked for nothing should not read as wanting +depth — and it is the prerequisite for anything below. + +### 2. The IMU thread could open on the first subscriber and close after the last + +There is nothing expensive to preserve: `open_imu` is a handful of register writes, no firmware and +no probe. So the thread could wait on an IMU lease, open, read at `imu_hz` while somebody is +listening, and close when they go — the same 4% saved, without an operator having to know about a +switch. + +**Not built, because the switch got there first and is a tenth of the machinery.** This becomes +the better answer the moment there *is* a consumer: an opted-in duck would otherwise pay the 4% +whenever nothing happens to be reading. Two details would decide whether it is done right: + +Two details decide whether it is done right: + +- **The subscribe answer must not lie.** `ImuStatus` starts at `unavailable: "no reading yet"`, + and `head_imu.stream`'s answer is written before the first sample. Cold, that answer would read + as "no BMI088 fitted", which is the one thing it must not say on a board that has one. So the + handler would signal the thread and wait for the open to resolve — bounded, a couple of hundred + milliseconds — before answering `found` or `lost`. (The switch has the same problem and solves + it the same way: `ImuStatus::off` is its own sentence, naming the key.) +- **A cold `quat` is a converging `quat`.** The Madgwick fusion settles over about a second + (`tof/src/imu.rs:43`), so a subscriber gets orientation that is still moving where today it gets + one converged since boot. `gyro` and `accel` are raw and unaffected. Document it on + `method::HEAD_IMU_STREAM`; the first real consumer can say whether it needs a warm filter, which + is easy to add then and pointless to guess at now. + +### 3. Do not gate the ranging, and do not touch the unit + +Both were the plan before the measurement, and both are now closed: + +- **Ranging on demand buys 0.5% of one core.** That is what the `tof-sensor` thread costs to hold a + laser open, poll it and publish fifteen frames a second, after the poll work in `idle-cpu.md`. + Stopping and starting ranging is cheap in itself — `start(hz)`/`stop()` are one transaction each + (`tof/src/sensor.rs:293`, `:181`) — but it means a state machine in the one process that owns a + shared I²C bus, a resume path that has to be right, and a hardware assumption nobody has tested + (that stop-then-start does not re-upload firmware). Half a percent does not buy that. +- **`robotd`'s theremin can keep subscribing at startup.** Its depth reader connects once and holds + (`robotd/src/theremin.rs:237`), which under a ranging gate would have pinned the laser on for + exactly the ducks that play notes. With no ranging gate there is nothing to pin, and the reason it + connects early — so the first arming window waits only for frames — stands unchallenged. + +The one argument left for gating the laser is the VCSEL's own power, which is not CPU and does not +show up in `ps`. If somebody wants that closed, the number is in the VL53L8CX datasheet; at +milliamps it stays closed and this bullet can go. + +## Why not start and stop the unit + +Kept because it is the answer to the question that started this, and because the reason is a durable +fact about the daemon rather than a measurement. + +**The bring-up is seconds, and it is per process.** `Sensor::open` probes the device ID and then +uploads the ULD firmware — ~90 KB over I²C, "a few seconds at 400 kHz" (`tof/src/sensor.rs:226`) — +once per process, before ranging. A `monitor` that started the unit on `t` would show an empty grid +for seconds, and `robotctl theremin` could not begin arming until the upload finished. + +**Nothing owns the "off".** Two clients can want depth at once, so stopping on exit needs a +reference count that survives a client being `SIGKILL`ed, and systemd has none for manually started +units. A `monitor` killed with Ctrl-\\ would leave the sensor ranging forever. + +**It needs privilege no client has.** `monitor` runs as an operator in the `robot` group; +`systemctl start` wants root or a polkit rule granting `manage-units` on that unit to that group — a +new install-path artifact that a board provisioned before it would silently not have. + +**One unit, two sensors.** Stopping `tofd` stops the head IMU too, which is the sensor this document +ends up caring about. + +Socket activation would fix the privilege and the leak for free — a `tofd.socket` unit owning +`/run/tofd/tof.sock`, where connecting *is* the start signal and systemd owns the lifecycle. It does +not fix the firmware, which it moves onto the first connection of every session and repeats on each +idle exit. It is the right mechanism for the day the *process* is what we want gone; it is not the +answer to a poll, and after the measurement there is no poll worth answering. + +## What is left for a board + +- `ps -L` on a duck running the switch, showing no `head-imu` thread at all. +- The bench again with the CPU governor pinned and the modes interleaved, if anybody wants the + third read's worth to better than half a point. Nothing now rests on it. +- SoC temperature at idle over ten minutes, before and after. ~4% of a core will not be visible in + it, and that is the honest expectation to write down rather than discover: this was about not + reading a sensor for nobody, not about heat. diff --git a/docs/recurrent-policies.md b/docs/recurrent-policies.md new file mode 100644 index 00000000..b162c3ba --- /dev/null +++ b/docs/recurrent-policies.md @@ -0,0 +1,86 @@ +# Recurrent ONNX policies + +The runtime accepts feed-forward policies and the explicit-state LSTM export produced by +mjlab/rsl_rl. Both consume the existing 61D observation and produce the same 14 raw joint +actions at 50 Hz. Observation normalization must be baked into the exported model. +The training critic is not deployed. This adds no sensors or action filtering. + +## Model contract + +All tensors are float32. Inputs and outputs are matched by name, independently of order. + +| Model | Inputs | Outputs | +| --- | --- | --- | +| Feed-forward | `obs [1, 61]` | One output `[1, 14]` (existing output names remain accepted) | +| LSTM | `obs [1, 61]`, `h_in [layers, 1, hidden]`, `c_in [layers, 1, hidden]` | `actions [1, 14]`, `h_out [layers, 1, hidden]`, `c_out [layers, 1, hidden]` | + +Batch dimensions may be symbolic; inference always uses one robot. Layers and hidden +width must be positive static dimensions, identical across the four state tensors. +Each state is limited to 1,048,576 elements. Unsupported names, ranks, types, widths and +extra tensors fail at load. Warm-up also rejects invalid or non-finite outputs. +GRU and other recurrent export signatures are not implemented. + +Publish recurrent policies with **`model_api: 2`** so older daemons refuse them before +installation. This daemon also accepts existing API 1 feed-forward models. The manifest +schema, observation width and action width are unchanged. + +## Memory lifetime + +Each loaded policy owns preallocated hidden and cell tensors. Every successful inference +copies both new states into those buffers. Failed inference clears memory; non-finite +outputs are rejected before either state is committed. Warm-up state is discarded. + +Memory starts at zero on initial activation, on switching networks (including switching +back), explicit disable, controller reset after a pause or fall recovery, and a chained +skill restarting. Brief sensor dropouts follow the existing resume grace period. +Command changes within an active network retain memory. + +Replacing an inactive slot preserves the active network's memory only when its ONNX file +has the same SHA-256 digest. A changed active model starts fresh, even at the same file path. +Use self-contained ONNX exports for this guarantee: externally referenced weight files +are not part of the ONNX file digest. An explicit controller reset always clears all slots. + +Existing action scaling and optional runtime filters retain their configuration. For a +policy trained without action filtering, disable those filters in its deployment tuning; +adding recurrence does not make a mismatched filter configuration valid. + +## Offline rehearsal and board timing + +The example uses the production Rust inference path and never opens the motor bus: + +```sh +export ORT_DYLIB_PATH=/path/to/libonnxruntime.so +cargo run --release -p duck-control --example policy-rehearsal -- policy.onnx > rehearsal.json +``` + +It warms up the actor, clears state, then measures 1,000 inferences with nominal gravity, +zero commands and previous-action feedback. JSON includes p50/p95/p99/max latency, the +number exceeding 20 ms, and the actions. Run the built example **on the Radxa** to establish +onboard timing; development-host timings do not establish the robot's timing budget. +These timings exclude sensor reads, motor writes and scheduling delays. + +For deterministic multi-step comparison against PyTorch or Python ONNX Runtime, supply a +JSON array of records as the second argument: + +```json +[{"obs": ["replace with exactly 61 numbers"], "reset": true}] +``` + +Each record supplies the complete observation, including previous actions. Set `reset` at +episode boundaries. Compare the returned action sequence to the reference with the same +observations and resets. This checks inference and memory handling, not physical sim2real +transfer. Export trained checkpoints through the training repository's normalized exporter. + +## Tests + +```sh +cargo test -p duck-control +# Requires ONNX Runtime >= 1.23: +cargo test -p duck-control --test recurrent_policy -- --ignored +cargo test -p robotd recurrent_memory_resets_with_controller_feedback -- --ignored +``` + +Tiny checked-in fixtures use an actual ONNX LSTM operator. They exercise memory continuity, +resets, policy switches, fallback selection, warm-up, hot swaps, dynamic batch dimensions, +invalid contracts and non-finite state. Regenerate with +`python duck-control/tests/fixtures/generate.py` in an environment with `onnx` and `numpy`. diff --git a/docs/robot/cheatsheet.md b/docs/robot/cheatsheet.md index d6c3676d..e286380f 100644 --- a/docs/robot/cheatsheet.md +++ b/docs/robot/cheatsheet.md @@ -90,7 +90,11 @@ this is relative motion and it drifts; it answers "did it walk in a circle" and angles between degrees and radians; `t` opens the [ToF matrix](#the-tof-sensor-tofd); `d` toggles the robot view and `[` / `]` orbit it; `p` opens the pad's raw input stream — every evdev report from the gamepad, with the gaps between them, which is the only place a stalled radio is visible -([pair a gamepad](pair-a-gamepad.md#when-it-drops-while-you-are-driving)). Angles are degrees on screen — joints, head and the yaw rate. +([pair a gamepad](pair-a-gamepad.md#when-it-drops-while-you-are-driving)). A pad with an inertial +unit — the Pro Controller clones have one, the Xbox does not — grows that block by a panel: a +wireframe pad that tilts and turns with the one in your hands, its pitch, roll and drifting yaw, +the raw acceleration and rates, and whether the gyro's rest bias has been learned yet (hold it +still half a second). The yellow bar is the pad's front edge. Angles are degrees on screen — joints, head and the yaw rate. Redirected or piped it prints one line per tick instead, so `> run.log` and `| grep FALLEN` behave, and those numbers stay radians whatever the screen is set to. The joint vectors are in `--json`, which carries the whole state, one object per line: @@ -118,7 +122,8 @@ about a robot behaving oddly, and until now answering it meant a full-screen edi An interactive editor over `/etc/robot/robotd.toml`: every key the daemons know, the feature switches first (policy on/off, walk/roller, limp-fall, audio, pet detection, battery shutdown, camera and video quality…), current value against default, one line of doc. SPACE toggles, ENTER types a -value, `u` reverts a key to its default. Values in yellow (marked `•`) are the keys where +value, `u` reverts a key to its default, `ctrl+f` opens a fuzzy search over everything on +screen (the selection follows as you type; ENTER or ESC keeps it there). Values in yellow (marked `•`) are the keys where this robot diverges from the defaults; everything else is the built-in default, and `unset` optionals show what they resolve to `(auto)`. @@ -134,8 +139,10 @@ Three properties worth trusting: - **It cannot write a file robotd refuses to start on.** Every save is validated through the daemon's own loader first, atomically (temp file + rename), and rejected with the reason. -The daemons read the file once at startup, so saving offers a restart — of the ones that read -what you changed: `[media]` is `mediad`, everything else is `robotd`. `sudo`, because the file +Saving offers what the change actually needs, from the daemon that actually reads it: a restart +for most keys (`[media]` and `[duck_detector]` are `mediad`'s, `[head_imu]` is `tofd`'s), a `robotd` +*reload* for `[policy]` — the motors stay powered — and nothing at all for `[pad]` and +`[pad_imu_head_control]`, which `padd` picks up within a second. `sudo`, because the file is root-owned — without it the editor opens read-only and says so on the first write. `--file` points it elsewhere for a bench copy. The shipped `deploy/robotd.toml` stays the reference for *why* each knob exists; this is for flipping them. @@ -147,9 +154,12 @@ sudo robotctl configure ``` Set `media.quality` — `1080p30`, `720p30`, `720p15` or `360p30` — and take the restart it -offers. `media.camera` off streams a test pattern instead, which is what a board with no camera +offers. `media.source` set to `test` streams a test pattern instead, which is what a board with +no camera wants: the WebRTC *control* channel rides on the video track, so a pipeline that cannot start -costs both. `media.bitrate` follows the quality unless you set it; the unit is bits per second. +costs both. The pattern ignores `media.quality` and runs at 256x144@5 — it is there to make the +session exist, and drawing a 720p one costs five times the CPU a real camera does. +`media.bitrate` follows the quality unless you set it; the unit is bits per second. `media.congestion_control` is the other knob in that section, and it is the one that moves CPU: `disabled` drops the bandwidth estimator, which is the largest single consumer in `mediad` (7.6% of @@ -212,6 +222,23 @@ name one — `--version v1` is how to go back. The robot returns to its home pos slot and drives again, and **a slot you loaded yourself is left alone**, because it points somewhere else entirely. +#### A newer duck detector + +The model `mediad` finds other ducks with lives on the Hub the same way +(`pollen-robotics/microduck-duck-detector`) and versions on its own line: + +``` +robotctl duck-detector check +``` + +``` +sudo robotctl duck-detector update +``` + +Same shape as the policy pair — `--version ` names one, and `check` changes nothing. `update` +restarts `mediad`, which drops the console's video for a moment; whether the detector then runs at +all is `[duck_detector] enabled` in `robotctl configure`. + #### Trying your own file No release, no file to edit, no restart: @@ -411,6 +438,7 @@ sudo robotctl robot init ``` sudo robotctl robot relax --yes +sudo robotctl robot reboot-motors # every servo; or `reboot-motors 3 11` for just those. Torque off, then init / Start ``` `init` powers the joints and ramps to the home pose over about two seconds — **it moves every joint**, @@ -429,6 +457,13 @@ corrupt each other's replies: sudo systemctl stop robotd && sudo /opt/robot/daemon/current/bin/robotd init && sudo systemctl start robotd ``` +**Replacing a motor** needs no configuration tool. Fit the new servo straight from the box (ID 1, +57 600 baud), power the servos, and `robotd` — or `robotd init` — finds the one joint that no longer +answers, flashes the new servo as that joint, sets its registers and reboots it. The journal says +`factory-fresh servo on the bus; flashing it as the missing joint` and then `replacement servo +adopted`. One at a time: with two joints missing it cannot tell which the new servo is for, waits, +and says so. + `init` works whether or not the robot has fallen — by default a fall is a *report* (visible in `robotctl monitor`), not a gate, matching the prototype. A board that sets `[safety] fall_limp` or `fall_recover` in `robotd.toml` arms the gate: there a fallen robot goes limp and refuses @@ -467,8 +502,8 @@ mapping is the prototype's, so muscle memory carries over: | --- | --- | | left stick | drive: forward/back and strafe · head: head yaw and pitch · body pose: up and crouch | | right stick | drive: turn · head: neck pitch and head roll · body pose: pitch and roll | -| **Start** | toggle the policy — nothing moves until it is on | -| **Y** / triangle | head mode: sticks pose the head (body holds still) | +| **Start** | first press: torque on and a 2 s ramp to the home pose, then hold. Second press: the policy drives. After that it toggles the policy | +| **Y** / triangle | head mode: sticks pose the head (body holds still). With `[pad_imu_head_control] enabled` and a pad that has an IMU: the pad's tilt poses the head and the sticks keep driving — see below | | **B** / circle | body-pose mode: sticks lean and crouch the standing robot | | **A** / cross | ground pick | | **X** / square | roulade — one forward roll; hold to chain rolls | @@ -476,7 +511,23 @@ mapping is the prototype's, so muscle memory carries over: | **DPad-Down** | sit ↔ stand | | **RT / LT** | mouth (either trigger) — RT also quacks; LT rides the "wheee" while held | | **DPad-Up**, held 3 s | switch drive mode, walk ⇄ roller | -| **Select**, held 2 s | sit down, then power off | +| **DPad-Right** | reboot every servo: the way back from a tripped overload without pulling the battery. Torque off, then Start | +| **Select**, short press | torque off (`robot.relax`) **on release**: the emergency stop. The robot drops, so hold it. Then Start stands it up again | +| **Select**, held 2 s | sit down, torque off, power off — the release afterwards does nothing more | + +**Drive the head with the pad itself.** A Pro Controller carries an IMU, and with + +```bash +sudo robotctl configure # Controller-IMU head control → enabled +``` + +Y changes meaning on such a pad: the first press hands the head to the pad — tilt it and the head +tilts, turn it and the head turns — while the sticks go on driving the body. Press Y again and the +head holds where it is, sticks still driving. Press it a third time and the pad drives the head again +**from wherever the pad is now**: its yaw is a gyro's word alone and drifts, and re-centring on every +re-entry is how you beat the drift without a magnetometer. `gain` in the same section is head +radians per pad radian, 1 by default. On an Xbox pad, or with the switch off, Y is the stick head +mode above. `padd` picks the change up within a second; no restart. There is no stop button: release the sticks and the robot stands, and `robotd`'s deadman stops it if `padd` dies. On a roller robot (`mode = "roller"` in `robotd.toml`) the sticks take the roller @@ -656,6 +707,11 @@ closer is higher — and the mouth opens with the note, wide at the top of the r until Ctrl-C and puts the instrument down on the way out. `--off` puts down one a client left up. +**Off by default** — `[theremin] enabled` in `robotd.toml`, per duck, like the chorale above. +`robotctl configure` is the way to set it, and it offers the `robotd` restart that picks it up; +until then `robotctl theremin` refuses and names the key. Nothing else turns off with it: `tofd` +runs regardless, so the depth grid below works on a duck that has never played a note. + An explicit mode with nothing clever inside it: while it is up, the nearest return inside the playable band is the hand. Point the duck at open space and it is silent; point it at a wall 40 cm away and it plays a steady note. It plays sitting, standing or walking — the mouth is @@ -725,6 +781,19 @@ provisions the bus itself; the ToF step only adds the stable `/dev/i2c-pihat` name. Both sensor generations are supported — a VL53L5CX and a VL53L8CX are interchangeable on the board, and the daemon picks the driver from an ID read. +#### The head IMU (`head_imu.stream`) + +`tofd` also serves the head module's BMI088 — gyro, acceleration and a Madgwick +orientation — and it is **off by default**: `[head_imu] enabled` in `robotd.toml`, +set with `robotctl configure`, which offers the `tofd` restart. Reading it costs +~4% of a core at 100 Hz and nothing subscribes yet, so a duck that is not mapping +was paying that from boot. A subscriber while it is off gets a reason naming the +key, not the silence an unfitted sensor gives. `tofd --imu` reads it for one +session without touching the file, and `--imu-hz` trades rate for cost linearly. + +None of this touches depth: the ToF ranges either way, so the grid above works on +a duck whose IMU has never been switched on. + ### Wifi (`configd`) ``` diff --git a/docs/robot/dev-push.md b/docs/robot/dev-push.md index f9d0b949..7f552b22 100644 --- a/docs/robot/dev-push.md +++ b/docs/robot/dev-push.md @@ -333,7 +333,7 @@ This should not have been necessary, so it is worth reading the journal for why | | | |---|---| | `DUCK_ROBOT` | The robot, by name. Its address is found over Bluetooth and cached. | -| `DUCK_BOARD_USER` | The ssh user on the board, for the name path. Default `radxa`. | +| `DUCK_BOARD_USER` | The ssh user on the board, for the name path. Default `radxa`. `duckctl ssh` and `duckctl scp` read it too. | | `DUCK_PIN` | The robot's pairing PIN, if it is not the factory `000000`. Read by `duckctl`. | | `DUCK_BOARD_CACHE` | Where resolved addresses are cached. Default `~/.cache/duck/boards`. | | `DUCK_BOARD` | The board, by address, instead of an argument. `radxa@192.168.1.42`. | diff --git a/docs/robot/duckctl.md b/docs/robot/duckctl.md index c3d02dbe..0e4cccc2 100644 --- a/docs/robot/duckctl.md +++ b/docs/robot/duckctl.md @@ -74,6 +74,46 @@ ssh radxa@$(duckctl ip) connects to nothing, needs no PIN, and takes about a second — and the answer is not stale: `btd` re-reads the address every five seconds and re-advertises when it moves. +Or skip the substitution: + +```bash +duckctl ssh +``` + +```bash +duckctl ssh -- sudo robotctl pad pair +``` + +`ssh` finds the address the way `ip` does and then becomes `ssh`, so the prompts, the terminal and +the exit status are ssh's own. The account is `--user`, else `DUCK_BOARD_USER` from the environment +— the variable [`dev-push.sh`](dev-push.md) reads, so a laptop set up for pushing is set up for this +— else `radxa`. Words after `--` run on the robot instead of opening a shell. + +Files go the same way: + +```bash +duckctl scp report.md :/tmp/ +``` + +```bash +duckctl scp :/var/log/robotd.log . +``` + +A path starting with `:` is on the robot — `scp`'s own `host:path` with the host left out, since the +host is the part this finds for you. Everything else reaches `scp` as typed, `-r` and the rest of +its flags included, and the progress meter and the exit status are `scp`'s own. The account resolves +the way `ssh`'s does. + +This tool's own flags come first, before the paths: + +```bash +duckctl --name ducky scp -r logs/ :/tmp/ +``` + +A copy with no `:` anywhere in it is refused before the scan, because it is a local-to-local copy +that no robot is party to and nothing in `scp`'s output would say so. A local file that really is +named `:foo` is `./:foo`. + A robot bonded to this machine often stops advertising the service to it, and then `ip` connects and asks `net.status` instead. That is slower and needs the PIN, and it always answers. `--verbose` says which of the two happened. @@ -248,6 +288,50 @@ duckctl --name version The API version, the release, and the git revision it was built from. A `revision` of `null` means the release was built on somebody's laptop rather than by CI. +## Logs + +```bash +duckctl --name logs robotd +``` + +The last 40 lines of that daemon's journal, this boot. For more, and for the boot before this one: + +```bash +duckctl --name logs robotd -n 200 +``` + +```bash +duckctl --name logs btd --boot -1 +``` + +Readable units: `updaterd`, `robotd`, `configd`, `btd`, `padd`, `mediad`, `tofd`, plus +`bluetooth` and `NetworkManager`. The `.service` suffix is optional, and anything else comes back +refused with that list. + +Lines go to stdout and everything else to stderr, so `logs robotd -n 200 | grep -i panic` works. +A long tail is trimmed to what the radio can carry, oldest lines first, with a note saying so. + +A tail that spans a restart says where: + +``` +2026-09-09T12:27:20+00:00 systemd[1]: Starting robotd.service - Robot control daemon... +-- new robotd process, pid 3227 -- +2026-09-09T12:27:21+00:00 robotd[3227]: control loop running joints=15 hz=50.0 driving=true +``` + +Which matters after an update, when forty lines carry two different builds' output. Anything in +`-- … --` comes from the robot rather than the journal. + +There is no `-f`, no `--since` and no search. For those, ssh in: + +```bash +ssh radxa@$(duckctl --name ip) +``` + +```bash +journalctl -u robotd -f +``` + ## Updates Same words as `robotctl update`, so a command learned on the robot works here. Every one of them @@ -549,7 +633,8 @@ minutes with nothing arriving at all — which is why they are the way to run an ## What it prints Replies go to stdout as pretty JSON, and everything else — progress, diagnosis, what the radio -saw — to stderr. So `duckctl ... info > reply.json` keeps the two apart, and a JSON-RPC error +saw — to stderr. `logs` is the exception and prints its lines as lines, since a journal tail in +escaped JSON is unreadable; a refusal from it still prints as JSON. So `duckctl ... info > reply.json` keeps the two apart, and a JSON-RPC error from the robot still exits non-zero. Progress lines start with `·` and are one line each, so `update apply > outcome.json` leaves them on screen and keeps the outcome in the file. diff --git a/docs/robot/pair-a-gamepad.md b/docs/robot/pair-a-gamepad.md index 072a4eb2..9639efd5 100644 --- a/docs/robot/pair-a-gamepad.md +++ b/docs/robot/pair-a-gamepad.md @@ -14,6 +14,12 @@ On an **Xbox** controller this is two presses, and the second is the one that go On a **DualSense**: hold Create and PS together until the light bar flashes. +On a **Pro Controller** — the no-name Switch-style pads, which bluetoothctl lists as +`Pro Controller` — hold the small **Sync** button on the top edge, next to the USB-C port, until the +player lights sweep back and forth. It is a classic Bluetooth (BR/EDR) pad, the Xbox one is LE, and +the robot pairs the two in opposite orders; `pad pair` picks the right one by itself, and +[pairing one by hand](#pairing-a-pro-controller-by-hand) says what the order is if you have to. + ## Pair it ```bash @@ -57,6 +63,21 @@ padd active — driving whatever pad connects Two lines, because they fail separately: a connected pad with a dead driver looks exactly like a working robot ignoring you. +A Pro Controller also carries a six-axis IMU, which the kernel exposes as a second input device +beside the one that drives. `robotctl monitor`, with `p` for the pad block, shows it as a panel +that appears only when the pad has one: a wireframe pad posed like the real one, pitch and roll +from gravity, yaw from the gyro alone (it drifts — nothing on a pad observes heading), and the +gyro's rest bias, which the clone needs learned before the picture stops turning on its own. Set +the pad down for half a second and the panel says `settled`. The stream costs nothing while the +monitor is not open: `padd` reads the IMU node only while somebody is subscribed to the tap — or +while it is steering the head from it, below. + +The same attitude can drive the robot's head. `sudo robotctl configure`, section *Controller-IMU +head control*, `enabled`: Y then hands the head to the pad's tilt while the sticks keep driving, +Y again holds the head, and a third Y re-centres on the pad's current attitude and follows again — +the [cheat sheet](cheatsheet.md#gamepad-configd) has the full cycle. The picture in the monitor and +the head use one filter, so where the drawn pad points is where the head goes. + `paired but NOT trusted` is the state worth knowing. It works now and does not reconnect after a reboot, because approving a reconnection needs an agent and at boot there is none. Re-run `pad pair` to fix it. @@ -129,6 +150,44 @@ to check and how to drop it. Both are workarounds for the aic8800 radio, not properties of the design. They go when the radio does. +### A classic pad and bluetoothd's CPU + +A Pro Controller streams IMU samples in every packet, about 200 packets a second, whether or not +anyone touches it. With BlueZ's default `UserspaceHID=true` each one is relayed by bluetoothd through +uhid, and that cost 16% of a core on graphite with the pad idle. `scripts/setup-board.sh` sets +`UserspaceHID=false` in `/etc/bluetooth/input.conf` on every board, which hands the channel to the +kernel's `hidp` and takes bluetoothd out of the data path — measured 0.0% afterwards, same driver, +same input nodes, same bond. It applies at the next boot. An LE pad such as the Xbox is untouched by +the setting: HID over GATT never went through `input.conf`. A board provisioned before this exists +gets it by re-running the script: + +```bash +sudo sh scripts/setup-board.sh && sudo reboot +``` + +## Pairing a Pro Controller by hand + +Only if `pad pair` is not available. The order matters and is the **reverse** of the Xbox one: + +```bash +bluetoothctl pair 98:B6:E9:28:06:09 +``` + +```bash +bluetoothctl connect 98:B6:E9:28:06:09 +``` + +```bash +bluetoothctl trust 98:B6:E9:28:06:09 +``` + +`connect` first — which bonds as a side effect — ends in the state that is hardest to read: the pad's +light goes solid, `pad status` says `connected`, `bluetoothctl info` says `Paired: yes` and +`Connected: yes`, and **no input device exists**, so `padd` sits at "waiting for padd to open a pad" +and nothing drives. Recover with `sudo robotctl pad forget
`, put the pad back into pairing +mode and pair again. Running `pad pair` never produces this state: it knows the pad is classic from +the class BlueZ reports and pairs before it connects. + ## When pairing fails every time Check `/etc/bluetooth/main.conf` for the `Privacy` setting. It should read `Privacy = device`. diff --git a/docs/robot/simulation.md b/docs/robot/simulation.md new file mode 100644 index 00000000..e0cd43f5 --- /dev/null +++ b/docs/robot/simulation.md @@ -0,0 +1,175 @@ +# The simulated duck + +A duck in MuJoCo, driven by the real daemons. You develop against it exactly as you develop against +a robot on the desk: the same `robotd`, the same policies, the same 50 Hz loop, the same `robotctl`, +the same console. Only the body is different. + +This page is how to use it. [`design/simulation.md`](../design/simulation.md) is what it is and is +not a twin of, and why it is built the way it is. + +## What it is + +`robotd --sim host:port` runs the daemon with `duck_control::sim::RemoteIo` in place of the servo +bus: every tick, joint positions, velocities and the IMU come in over a TCP socket from a MuJoCo +process, and the policy's targets go back out. Everything above that seam — the control loop, the +policy, safety, fall detection, kinematics, odometry, the whole IPC surface — is the code a robot +runs, unchanged and unable to tell. `tofd --sim` gets its 8×8 depth frames from the same simulator; +`mediad --sim-camera` gets a rendered head-camera image, mounted a quarter turn off like the real one. + +The MuJoCo half lives in [`microduck_rl`](https://github.com/pollen-robotics/microduck_rl) as +`duck-body`: one process, one window, N duck bodies in one scene, with the BAM actuator models the +policies were trained against. + +**What it is good for:** anything in the daemons and their clients — IPC, `robotctl`, the console, +the updater, the chorale, policies standing and walking, mapping. **What it cannot tell you:** +anything in a driver. The Dynamixel bus, the BLE radio, the camera ISP, the NPU and the hardware +encoder are absent, not modelled; a bug in one of those is only visible on a robot. + +## What you need + +- This repo, built for your machine (`cargo build` happens on its own). +- A checkout of `microduck_rl` with its venv, at `~/Pollen/microduck_rl` or wherever `DUCK_SIM_RL` + points. It provides `duck-body`, the scenes, and the `libonnxruntime` a laptop otherwise lacks. +- For the container form only: `sudo`, `systemd-nspawn` (package `systemd-container`) and + `mmdebstrap`. The script says which one is missing and prints the line to install it. + +## `up` or `boot`? + +Both drive the same MuJoCo body with the same `robotd`. They differ in what the daemons run *under*. + +| | `scripts/duck-sim` (`up`) | `scripts/duck-sim boot N` | +|---|---|---| +| The daemons run as | plain processes, your user, a pidfile each | systemd services inside a `systemd-nspawn` container per duck, from the real unit files | +| Needs | nothing beyond the repo and `microduck_rl` | `sudo`, a one-time Debian rootfs build | +| Starts in | seconds | a minute the first time, seconds after | +| Identity | your laptop's; several ducks are one process tree | a machine-id, a voice and a socket per duck, and `duck-ether` between them | +| Exercises | the control loop, policies, IPC, `robotctl`, the console | all of that, plus `User=`/groups/`RuntimeDirectory=`/hardening, the updater's apply, health gate, rollback and restart order, `journalctl` | + +Rule of thumb: **`up`** when you are working on the control loop, a policy, IPC or a client. +**`boot`** when you are working on anything that touches systemd, the updater or provisioning, or on +more than one duck talking to another. Two of the bugs this simulator has caught were a daemon whose +user did not exist, so its unit never started — a class `up` cannot see at all. + +## One duck, no container + +```sh +scripts/duck-sim # a MuJoCo window opens, the duck stands up, and it is yours +scripts/duck-sim status # health, and whether it is standing +scripts/duck-sim drive # walk forward for 8 s (args: vx vyaw, default 0.15 0) +scripts/duck-sim ctl health # anything robotctl does, aimed at this duck +scripts/duck-sim monitor # robotctl monitor: joints, IMU, ToF, sticks +scripts/duck-sim log # robotd's log +scripts/duck-sim simlog # the MuJoCo side +scripts/duck-sim realtime # how fast the world is running (see below) +scripts/duck-sim down +``` + +Without an argument the script builds the daemons from the branch you are on, writes a params file +naming the policies in this repo, starts `duck-body`, `tofd --sim` and `robotd --sim` under +`~/.cache/duck-sim`, and enables the standing policy. `ctl` is `robotctl` pointed at that duck's +sockets, which is the only thing that is different from a robot: on a board the sockets are under +`/run`, here they are under the state directory. + +To talk to it from your own tools, the sockets are `~/.cache/duck-sim/duck-a.sock` (robotd; `duck.sock` +is a link to whichever duck `ctl` talks to) and `~/.cache/duck-sim/duck-a-tof.sock`, and the body is +on TCP port 7801. + +## Several ducks, each a machine you log into + +```sh +scripts/duck-sim boot 4 # four ducks in one world, each in its own container (sudo) +scripts/duck-sim shell # you are on duck-a +scripts/duck-sim shell duck-c # or any other +duck-a # robotctl health +duck-a # journalctl -u robotd -f +scripts/duck-sim down +``` + +`boot` runs each duck's daemons under real systemd in a `systemd-nspawn` container, booted from one +Debian 13 rootfs (built once, about three minutes, kept under the state directory) with an overlay +per duck. That buys the part the plain form cannot fake: the real unit files with their `User=`, +groups, `RuntimeDirectory=` and hardening, under a real init — so `robotctl update apply`, the +health gate and the restart order behave as they do on a robot. Ducks are `duck-a`, `duck-b`, ... and +each has its own machine-id and its own voice. + +With more than one duck the script also starts `duck-ether`, a fake radio that carries the chorale's +BLE beacons between containers. It is deliberately a *bad* radio — dropped and delayed beacons — +because a perfect one hides the bugs the real one shows. + +The first time, `boot` builds the rootfs and may ask you to install `mmdebstrap`. Every duck is a +systemd unit, so `down` is a `systemctl stop`, never a key combination. + +## A room to look at, and eyes to look with + +```sh +DUCK_SIM_SCENE=apartment DUCK_SIM_CAMERAS=a scripts/duck-sim boot 2 +``` + +The default world is a bare floor. `apartment` is six rooms in 7×6 m with doorways off centre on +purpose, so a pose is recognisable from the duck's 45° forward view; anything with a slash is a path +to your own scene, and `microduck_rl`'s `scene_*.xml` files are the built-in ones. + +Cameras are opt-in per duck (`a`, `a,c`, or `all`) because a rendered frame costs 12 ms against +0.3 ms to step four ducks' physics: one camera is a third of a core, four is most of one. Each duck +with a camera gets its own `mediad`, and its console is served at `http://127.0.0.1:8080`, `8081`, +... by index, exactly the page a robot serves. + +## Knobs + +Environment variables, all optional: + +| Variable | Default | What it does | +|---|---|---| +| `DUCK_SIM_RL` | `~/Pollen/microduck_rl` | Where `duck-body`, the scenes and the ONNX runtime are. | +| `DUCK_SIM_STATE` | `~/.cache/duck-sim` | Sockets, logs, params, the rootfs and the ducks' overlays. Short on purpose: a unix socket path is capped at about 108 bytes. | +| `DUCK_SIM_DUCKS` | `1` | How many ducks; `boot N` sets it too. | +| `DUCK_SIM_SCENE` | bare floor | A scene name (`apartment`) or a path. | +| `DUCK_SIM_CAMERAS` | none | Which ducks render a camera: `a`, `a,c`, `all`. | +| `DUCK_SIM_DUCK` | `duck-a` | Which duck `ctl` and `monitor` talk to. | +| `DUCK_SIM_KEYFRAME` | `SIT` | Where a duck starts: `SIT` folded on the floor (the standing policy rises from it), `HOME`, `STAND`, `FOLD`. | +| `DUCK_SIM_VIEWER` | `1` | `0` runs MuJoCo headless. | +| `DUCK_SIM_PORT` | `7801` | The first duck's body port; +1 per duck. | +| `DUCK_SIM_FRAME_PORT` | `7901` | The first camera's frame port; +1 per camera. | + +## Things worth knowing + +**Real time matters.** The daemons' loops are wall-clock. A simulator running below 1.0× real time +is not merely slow to watch: the policies are driving a robot that moves less than they expect, and +they cannot balance it. `scripts/duck-sim realtime` reports the factor, and `boot` prints it. Too +many ducks, or too many cameras, and the ducks do not get slow — they go *unhealthy* at the 45 Hz +gate, and in the container form the updater starts rolling releases back. Fewer ducks, fewer +cameras, or a headless viewer are the fixes, in that order. + +**Ducks do not hot-join.** MuJoCo compiles its model, so changing the number of ducks restarts the +simulator. The daemons survive that: `RemoteIo` reconnects on the next tick, and a duck whose body +is briefly gone reports unhealthy rather than dying, the same as a robot with no power on the bus. + +**`--sim` is not `--fake`.** `robotd --fake` is a robot made of nothing: no physics, positions echo +back perfectly, nothing falls over. It is for unit tests and for laptop work that needs no body at +all. `--sim` is the twin. The two flags are mutually exclusive. + +**A duck should not think it is a laptop.** A duck's voice and its chorale identity are derived from +the hardware serial, which several ducks on one machine would share. The script sets +`DUCK_IDENTITY` per duck so they differ; nothing on a robot sets it. + +## By hand, without the script + +Each half takes a `host:port` and can be run alone, which is useful when something is wrong: + +```sh +# the body, from the RL repo's venv +duck-body --ducks 1 --port 7801 --keyframe SIT +# the daemons, from this repo +target/debug/tofd --sim 127.0.0.1:7801 --socket /tmp/d/duck-a-tof.sock +DUCK_RUNTIME_DIR=/tmp/d ORT_DYLIB_PATH= \ + target/debug/robotd --sim 127.0.0.1:7801 --params --socket /tmp/d/duck-a.sock +# a camera, if duck-body was started with --cameras a +printf '[media]\nquality = "360p30"\n' > /tmp/d/mediad.toml +target/debug/mediad --sim-camera 127.0.0.1:7901 --config /tmp/d/mediad.toml \ + --robot-socket /tmp/d/duck-a.sock --tof-socket /tmp/d/duck-a-tof.sock +``` + +The camera's geometry has to match on both sides — `mediad` streams the `[media] quality` rung +(`360p30` is 640×360), and the body must render at the same size, because frames arrive raw with no +handshake and `mediad` refuses one of the wrong size rather than showing a picture nobody can read. +`scripts/duck-sim` is the record of the rest of the arguments; read it before improvising. diff --git a/duck-control/Cargo.toml b/duck-control/Cargo.toml index f0bb20c3..28d8452f 100644 --- a/duck-control/Cargo.toml +++ b/duck-control/Cargo.toml @@ -13,6 +13,9 @@ description = "Robot control core: model, bus, sensing" # both crates, so no binary gains a dependency from this edge. duck-ipc-proto = { path = "../duck-ipc-proto" } serde = { workspace = true } +# The simulator link speaks newline-delimited JSON — see `sim` for why a packed struct shared +# between two repositories in two languages is the worse trade. +serde_json = { workspace = true } thiserror.workspace = true tracing.workspace = true # `load-dynamic` dlopens libonnxruntime at first use instead of linking it. Two @@ -24,6 +27,8 @@ tracing.workspace = true # internally, so a successful probe means its own load will succeed too — see # `policy::ensure_runtime` for why that matters. libloading = "0.8" +# Identify unchanged networks when a different slot is hot-swapped. +sha2 = "0.11.0" ort = { version = "=2.0.0-rc.11", default-features = false, features = ["load-dynamic"] } # 1.6.0 is a floor, not a preference: it de-stuffs protocol 2.0 status packets whose payload @@ -37,3 +42,6 @@ rustypot = "1.6.0" # not have, so leaving it on breaks the aarch64 build outright. rustypot disables it for the # same reason; this crate has to as well, or cargo's feature unification turns it back on. serialport = { version = "4.8", default-features = false } + +[dev-dependencies] +serde_json.workspace = true diff --git a/duck-control/examples/policy-rehearsal.rs b/duck-control/examples/policy-rehearsal.rs new file mode 100644 index 00000000..877cc99d --- /dev/null +++ b/duck-control/examples/policy-rehearsal.rs @@ -0,0 +1,77 @@ +//! Offline inference only: never opens a motor bus. +//! cargo run --release -p duck-control --example policy-rehearsal -- policy.onnx [trace.json] +//! Trace: [{"obs": [61 floats], "reset": false}, ...]. Outputs actions and latency JSON. +use duck_control::{ + obs::Observation, + policy::{Net, Policy, PolicyPaths}, +}; +use std::{error::Error, path::PathBuf, time::Instant}; + +fn main() -> Result<(), Box> { + let mut args = std::env::args_os().skip(1); + let path = PathBuf::from( + args.next() + .ok_or("usage: policy-rehearsal MODEL.onnx [TRACE.json]")?, + ); + let trace: Vec = if let Some(path) = args.next() { + serde_json::from_slice(&std::fs::read(path)?)? + } else { + // Benchmark with nominal gravity, zero commands and previous-action feedback. + vec![serde_json::Value::Null; 1000] + }; + if trace.is_empty() || args.next().is_some() { + return Err("expected a non-empty trace and at most two arguments".into()); + } + let mut policy = Policy::load( + &PolicyPaths { + walk: path, + ..Default::default() + }, + 0.05, + )?; + // Warm CPU/kernel caches, then discard all memory from these synthetic steps. + for _ in 0..50 { + policy.infer(&Observation::zeroed(), Net::Walk)?; + } + policy.reset(); + let mut actions = Vec::with_capacity(trace.len()); + let mut times = Vec::with_capacity(trace.len()); + let mut previous = [0.0; 14]; + for entry in trace { + let mut x = [0.0f32; 61]; + if entry.is_null() { + x[5] = -1.0; + x[34..48].copy_from_slice(&previous); + } else { + let values: Vec = + serde_json::from_value(entry.get("obs").ok_or("trace entry missing obs")?.clone())?; + x = values + .try_into() + .map_err(|_| "trace observations must contain exactly 61 numbers")?; + if entry + .get("reset") + .and_then(|v| v.as_bool()) + .unwrap_or(false) + { + policy.reset(); + } + } + let observation = Observation::from(x); + let start = Instant::now(); + previous = policy.infer(&observation, Net::Walk)?; + times.push(start.elapsed().as_secs_f64() * 1000.0); + actions.push(previous); + } + times.sort_by(f64::total_cmp); + let percentile = |p: f64| times[((times.len() as f64 * p).ceil() as usize).saturating_sub(1)]; + println!( + "{}", + serde_json::json!({ + "steps": times.len(), "latency_ms": {"p50": percentile(0.50), "p95": percentile(0.95), + "p99": percentile(0.99), "max": times[times.len()-1]}, + "over_20_ms": times.iter().filter(|&&t| t > 20.0).count(), "actions": actions, + "scope": "actor inference only; excludes sensor and motor I/O" + }) + ); + Ok(()) +} diff --git a/duck-control/src/bus.rs b/duck-control/src/bus.rs index 00fbca9c..4b76f9c4 100644 --- a/duck-control/src/bus.rs +++ b/duck-control/src/bus.rs @@ -19,7 +19,10 @@ use rustypot::servo::dynamixel::xl330::Xl330Controller; use crate::imu::{IMU_BLOCK_LEN, SflpDecoder}; use crate::io::{ImuStale, IoError, JointTargets, Result, RobotIo, Sensors, SlowSensors}; -use crate::model::{BAUD_RATE, EXPECTED_REGISTERS, IMU_DXL_ID, JOINT_IDS, NUM_JOINTS}; +use crate::model::{ + BAUD_RATE, EXPECTED_REGISTERS, FACTORY_BAUD_RATE, FACTORY_ID, IMU_DXL_ID, JOINT_IDS, + JOINT_NAMES, NUM_JOINTS, +}; /// Start of the contiguous block read every tick: `present_pwm`, `present_current`, /// `present_velocity`, `present_position`. Twelve bytes covers all four, and happens to be @@ -48,6 +51,16 @@ const VOLTS_PER_COUNT: f64 = 0.1; /// costs a bounded hiccup rather than stalling the loop on the serial driver's default. const READ_TIMEOUT: Duration = Duration::from_millis(30); +/// How long a servo is off the bus after a REBOOT before it answers again — "a few hundred +/// milliseconds" per [`RobotIo::reboot`], with margin. Pinging too early would read a servo +/// that is merely still booting as one whose flash failed, and fail an adoption that worked. +const REBOOT_SETTLE: Duration = Duration::from_millis(500); + +/// Pause after each EEPROM write. The servo acknowledges before the cell is necessarily +/// committed, and the writes here happen once per motor swap, so waiting costs nothing and +/// removes the one race the datasheet leaves open. +const EEPROM_SETTLE: Duration = Duration::from_millis(20); + /// Run of consecutive stale reads at which the journal says something. /// /// 25 reads is half a second at 50 Hz — the same span [`SflpDecoder::ready`] waits for before @@ -88,6 +101,9 @@ impl StaleImuTracker { pub struct DynamixelIo { controller: Xl330Controller, + /// Kept so the port can be reopened at the factory baud rate: `rustypot` owns the serial + /// handle outright and offers no way to change its speed in place. + port: String, /// IMU first, then the servos in [`JOINT_IDS`] order — the order blocks come back in. ids: Vec, imu: SflpDecoder, @@ -99,17 +115,7 @@ pub struct DynamixelIo { impl DynamixelIo { pub fn open(port: &str) -> Result { - let serial = serialport::new(port, BAUD_RATE) - .timeout(READ_TIMEOUT) - .open() - .map_err(|e| IoError::Port { - path: port.to_owned(), - source: std::io::Error::other(e), - })?; - - let controller = Xl330Controller::new() - .with_protocol_v2() - .with_serial_port(serial); + let controller = open_controller(port, BAUD_RATE)?; let mut ids = Vec::with_capacity(NUM_JOINTS + 1); ids.push(IMU_DXL_ID); @@ -117,6 +123,7 @@ impl DynamixelIo { Ok(Self { controller, + port: port.to_owned(), ids, imu: SflpDecoder::default(), stale_imu: StaleImuTracker::default(), @@ -132,42 +139,190 @@ impl DynamixelIo { pub fn check_registers(&mut self) -> Result { let mut fixed = 0; for &id in &JOINT_IDS { - for &(name, want) in EXPECTED_REGISTERS { - // rustypot returns a Vec even for a single-id read. An empty one means the - // servo did not answer, which must not be read as "register is fine". - let raw = match name { - "return_delay_time" => self.controller.read_return_delay_time(id), - "baud_rate" => self.controller.read_baud_rate(id), - "pwm_slope" => self.controller.read_pwm_slope(id), - "shutdown" => self.controller.read_shutdown(id), - other => unreachable!("unhandled register {other}"), - } - .map_err(|e| IoError::Bus(format!("read {name} on {id}: {e}")))?; - - let got = *raw.first().ok_or(IoError::ShortRead { - what: "register read", - expected: 1, - got: 0, - })?; - - if got == want { - continue; - } - tracing::warn!(id, register = name, got, want, "correcting motor register"); - match name { - "return_delay_time" => self.controller.write_return_delay_time(id, want), - "baud_rate" => self.controller.write_baud_rate(id, want), - "pwm_slope" => self.controller.write_pwm_slope(id, want), - "shutdown" => self.controller.write_shutdown(id, want), - other => unreachable!("unhandled register {other}"), - } - .map_err(|e| IoError::Bus(format!("write {name} on {id}: {e}")))?; - fixed += 1; + fixed += self.check_registers_of(id)?; + } + Ok(fixed) + } + + /// [`Self::check_registers`] for one servo. + fn check_registers_of(&mut self, id: u8) -> Result { + let mut fixed = 0; + for &(name, want) in EXPECTED_REGISTERS { + // rustypot returns a Vec even for a single-id read. An empty one means the + // servo did not answer, which must not be read as "register is fine". + let raw = match name { + "return_delay_time" => self.controller.read_return_delay_time(id), + "baud_rate" => self.controller.read_baud_rate(id), + "pwm_slope" => self.controller.read_pwm_slope(id), + "shutdown" => self.controller.read_shutdown(id), + other => unreachable!("unhandled register {other}"), + } + .map_err(|e| IoError::Bus(format!("read {name} on {id}: {e}")))?; + + let got = *raw.first().ok_or(IoError::ShortRead { + what: "register read", + expected: 1, + got: 0, + })?; + + if got == want { + continue; } + tracing::warn!(id, register = name, got, want, "correcting motor register"); + match name { + "return_delay_time" => self.controller.write_return_delay_time(id, want), + "baud_rate" => self.controller.write_baud_rate(id, want), + "pwm_slope" => self.controller.write_pwm_slope(id, want), + "shutdown" => self.controller.write_shutdown(id, want), + other => unreachable!("unhandled register {other}"), + } + .map_err(|e| IoError::Bus(format!("write {name} on {id}: {e}")))?; + std::thread::sleep(EEPROM_SETTLE); + fixed += 1; } Ok(fixed) } + /// The expected servo IDs that do not answer a ping, in [`JOINT_IDS`] order. + /// + /// Fifteen pings, each bounded by [`READ_TIMEOUT`], so about half a second when the servos + /// are unpowered and a few milliseconds when they are not. Run once at startup: this is + /// what decides whether [`Self::adopt_replacement`] has anything to do, and it is the only + /// bus traffic the replacement path costs a robot whose servos are all present. + pub fn missing_servos(&mut self) -> Result> { + let mut missing = Vec::new(); + for &id in &JOINT_IDS { + let answered = self + .controller + .ping(id) + .map_err(|e| IoError::Bus(format!("ping {id}: {e}")))?; + if !answered { + missing.push(id); + } + } + Ok(missing) + } + + /// Flash a factory-fresh servo so it takes the place of the one that is missing. + /// + /// A new XL330 answers as ID 1 at 57 600 baud. Neither is used on this bus, so when exactly + /// one expected servo is silent the new one can be found, given the missing ID, switched to + /// the bus's speed, and then handed the same EEPROM check every other servo gets. That is + /// the whole of a motor swap: nobody has to run a configuration tool first. + /// + /// The servo is rebooted at the end, deliberately. A servo flashed this way comes out of it + /// with its hardware-error alert set (the observation behind this whole path), and that + /// alert holds torque off until the servo is power-cycled or rebooted. Rebooting here means + /// the servo that comes out of this is indistinguishable from one that was always there. + /// + /// Returns `Ok(false)` when nothing answers at the factory defaults: the servo is simply + /// missing, or was replaced by one that is not fresh. The bus is back at [`BAUD_RATE`] + /// either way, so the caller can keep waiting on it. + pub fn adopt_replacement(&mut self, id: u8) -> Result { + let name = JOINT_IDS + .iter() + .position(|&j| j == id) + .map(|i| JOINT_NAMES[i]) + .ok_or_else(|| IoError::Bus(format!("{id} is not a joint id")))?; + + // A servo that was already re-flashed to 1 Mbps but kept its ID is the one case where + // reopening the port would lose it; look at this speed first. + let baud = if self.ping_fresh()? { + BAUD_RATE + } else { + self.reopen(FACTORY_BAUD_RATE)?; + if !self.ping_fresh()? { + self.reopen(BAUD_RATE)?; + return Ok(false); + } + FACTORY_BAUD_RATE + }; + tracing::warn!( + id, + joint = name, + found_at_baud = baud, + "factory-fresh servo on the bus; flashing it as the missing joint" + ); + + // ID first, then the baud rate: the servo answers the second write at the old speed + // and switches only afterwards, so both are acknowledged. The other order would need + // a reopen between the two writes for nothing. + self.controller + .write_id(FACTORY_ID, id) + .map_err(|e| IoError::Bus(format!("write id {id} on {FACTORY_ID}: {e}")))?; + std::thread::sleep(EEPROM_SETTLE); + if baud != BAUD_RATE { + let want = EXPECTED_REGISTERS + .iter() + .find(|(n, _)| *n == "baud_rate") + .map(|&(_, v)| v) + .expect("baud_rate is an expected register"); + self.controller + .write_baud_rate(id, want) + .map_err(|e| IoError::Bus(format!("write baud_rate on {id}: {e}")))?; + std::thread::sleep(EEPROM_SETTLE); + self.reopen(BAUD_RATE)?; + } + + // Now an ordinary servo at the right address: the same check the others get pins + // return_delay_time and the rest. + let fixed = self.check_registers_of(id)?; + + RobotIo::reboot(self, id)?; + std::thread::sleep(REBOOT_SETTLE); + let back = self + .controller + .ping(id) + .map_err(|e| IoError::Bus(format!("ping {id} after reboot: {e}")))?; + if !back { + return Err(IoError::Bus(format!( + "servo {id} ({name}) was flashed but did not come back from its reboot" + ))); + } + // The reboot exists to clear this; say so if it did not, because a servo that keeps + // its alert will hold torque off and the symptom — one limp joint — points nowhere. + let hardware_error = self + .controller + .read_hardware_error_status(id) + .map_err(|e| IoError::Bus(format!("read hardware_error_status on {id}: {e}")))? + .first() + .copied() + .unwrap_or(0); + if hardware_error != 0 { + tracing::error!( + id, + joint = name, + hardware_error, + "replacement servo still reports a hardware error after its reboot" + ); + } + tracing::warn!( + id, + joint = name, + registers_fixed = fixed, + "replacement servo adopted" + ); + Ok(true) + } + + /// Does anything answer at the factory ID, at whatever speed the port is open at? + fn ping_fresh(&mut self) -> Result { + self.controller + .ping(FACTORY_ID) + .map_err(|e| IoError::Bus(format!("ping factory id {FACTORY_ID}: {e}"))) + } + + /// Close the port and open it again at `baud`. + /// + /// The old handle has to be gone first: `serialport` opens ttys exclusively, so opening a + /// second handle while the first lives fails with `EBUSY`. Hence the placeholder controller + /// — one with no port, never used — standing in while the real one is dropped. + fn reopen(&mut self, baud: u32) -> Result<()> { + self.controller = Xl330Controller::new(); + self.controller = open_controller(&self.port, baud)?; + Ok(()) + } + /// Present positions only — a lighter read than [`RobotIo::read`], used once at startup /// to adopt the pose the robot is already in. pub fn present_positions(&mut self) -> Result<[f64; NUM_JOINTS]> { @@ -241,6 +396,32 @@ impl DynamixelIo { } } +/// The serial port at `baud`, wrapped in a Protocol 2 controller. +fn open_controller(port: &str, baud: u32) -> Result { + let serial = serialport::new(port, baud) + .timeout(READ_TIMEOUT) + .open() + .map_err(|e| IoError::Port { + path: port.to_owned(), + source: std::io::Error::other(e), + })?; + Ok(Xl330Controller::new() + .with_protocol_v2() + .with_serial_port(serial)) +} + +/// Which servo a factory-fresh one should become, given the IDs that did not answer. +/// +/// Only an unambiguous answer is one: with two servos silent there is no telling which of them +/// the new one replaces, and guessing would flash a leg joint as a neck joint. With none silent +/// there is nothing to adopt — a stray fresh servo on a complete bus is not this code's problem. +pub fn replacement_target(missing: &[u8]) -> Option { + match missing { + [one] => Some(*one), + _ => None, + } +} + impl RobotIo for DynamixelIo { fn read(&mut self) -> Result { let blocks = self @@ -313,6 +494,15 @@ impl RobotIo for DynamixelIo { DynamixelIo::set_torque(self, on) } + fn reboot(&mut self, id: u8) -> Result<()> { + // The status packet is a courtesy the servo may not manage before it resets, so only a + // failure to send is an error here. + self.controller + .reboot(id) + .map(|_| ()) + .map_err(|e| IoError::Bus(format!("reboot {id}: {e}"))) + } + fn set_gain(&mut self, kp: u16) -> Result<()> { // I and D are written too, at zero — the prototype's `--ki`/`--kd` defaults, which // its startup writes to every motor. These are RAM registers, so every power-up @@ -407,6 +597,17 @@ impl RobotIo for DynamixelIo { mod tests { use super::*; + /// One silent servo is the only case a swap can be inferred from. With two silent there is + /// no telling which the fresh servo replaces, and guessing would flash a leg joint as a neck + /// joint; with none silent a stray fresh servo is nobody's replacement. + #[test] + fn a_replacement_is_inferred_only_from_exactly_one_missing_servo() { + assert_eq!(replacement_target(&[]), None); + assert_eq!(replacement_target(&[23]), Some(23)); + assert_eq!(replacement_target(&[23, 31]), None); + assert_eq!(replacement_target(&JOINT_IDS), None); + } + /// The block parsed per servo must cover current, velocity and position without /// overrunning. If `READ_LEN` and the offsets below ever disagree, joints get each /// other's values — which reads as a wiring fault, not a code bug. diff --git a/duck-control/src/io.rs b/duck-control/src/io.rs index 379724ff..82304062 100644 --- a/duck-control/src/io.rs +++ b/duck-control/src/io.rs @@ -134,6 +134,14 @@ pub trait RobotIo { /// never because a process began. fn set_torque(&mut self, on: bool) -> Result<()>; + /// Reboot one servo: the Protocol 2 REBOOT instruction. + /// + /// The way out of a latched hardware error — overload, overheating, electrical shock, the + /// `shutdown` mask — which otherwise holds torque off until the battery is pulled. The servo is + /// off the bus for a few hundred milliseconds and comes back with torque off and its RAM + /// registers, the gains among them, at their EEPROM defaults; the caller owns putting them back. + fn reboot(&mut self, id: u8) -> Result<()>; + /// Supply voltage and case temperatures, in one extra transaction. /// /// Not part of [`Sensors`], and not on the tick's critical path: these registers sit at @@ -191,6 +199,8 @@ pub struct FakeIo { /// How many times torque was written, so a test can tell "brought up once" from "written every /// tick" — the latter being a bus transaction per joint per tick. pub torque_writes: usize, + /// Every servo id rebooted, in order. + pub reboots: Vec, } impl Default for FakeIo { @@ -217,6 +227,7 @@ impl FakeIo { track_targets: true, torque: None, torque_writes: 0, + reboots: Vec::new(), } } @@ -284,6 +295,11 @@ impl RobotIo for FakeIo { Ok(()) } + fn reboot(&mut self, id: u8) -> Result<()> { + self.reboots.push(id); + Ok(()) + } + fn imu_ready(&self) -> bool { self.imu_ready } diff --git a/duck-control/src/lib.rs b/duck-control/src/lib.rs index 14f161b6..ca6f95df 100644 --- a/duck-control/src/lib.rs +++ b/duck-control/src/lib.rs @@ -15,6 +15,8 @@ pub mod model; pub mod obs; pub mod policy; pub mod safety; +/// A robot in MuJoCo, over TCP — the backend `robotd-design.md` §9 deferred. +pub mod sim; pub use imu::ImuData; pub use io::{FakeIo, IoError, JointTargets, RobotIo, Sensors, SlowSensors}; diff --git a/duck-control/src/model.rs b/duck-control/src/model.rs index ef1f144e..9a5fe374 100644 --- a/duck-control/src/model.rs +++ b/duck-control/src/model.rs @@ -79,6 +79,12 @@ pub const IMU_DXL_ID: u8 = 200; pub const BAUD_RATE: u32 = 1_000_000; +/// What a servo answers as out of the box: ID 1 at 57 600 baud. Both are deliberately unused +/// on this bus — no joint is ID 1 and nothing runs at that speed — which is what lets a +/// replacement be told apart from every servo already fitted ([`crate::bus`]). +pub const FACTORY_ID: u8 = 1; +pub const FACTORY_BAUD_RATE: u32 = 57_600; + /// EEPROM registers asserted (and corrected) at startup. /// /// `return_delay_time` is the load-bearing one: the XL330 ships at 250, which is 500 µs of @@ -158,6 +164,16 @@ mod tests { assert!(!JOINT_IDS.contains(&IMU_DXL_ID)); } + /// The replacement path finds a new servo by the ID it ships with. If a joint ever took + /// ID 1, a fresh servo would be indistinguishable from it — and flashing "the missing + /// joint" onto ID 1 would re-address a servo that was never missing. + #[test] + fn factory_defaults_are_unused_on_the_bus() { + assert!(!JOINT_IDS.contains(&FACTORY_ID)); + assert_ne!(IMU_DXL_ID, FACTORY_ID); + assert_ne!(FACTORY_BAUD_RATE, BAUD_RATE); + } + /// `MOUTH_INDEX` is used to skip a slot when mapping 14 policy actions onto 15 joints. /// Pointing it at the wrong joint would shift every action after it by one. #[test] diff --git a/duck-control/src/obs.rs b/duck-control/src/obs.rs index de1fcb84..433416d8 100644 --- a/duck-control/src/obs.rs +++ b/duck-control/src/obs.rs @@ -144,6 +144,13 @@ pub struct Observation { data: [f32; OBS_LEN], } +/// Replay an already assembled observation (for offline policy rehearsal). +impl From<[f32; OBS_LEN]> for Observation { + fn from(data: [f32; OBS_LEN]) -> Self { + Self { data } + } +} + impl Observation { pub fn as_slice(&self) -> &[f32] { &self.data diff --git a/duck-control/src/policy.rs b/duck-control/src/policy.rs index 8d7cf84f..346d76b2 100644 --- a/duck-control/src/policy.rs +++ b/duck-control/src/policy.rs @@ -4,7 +4,8 @@ //! `microduck_runtime` does; the skill networks — sit↔stand, ground pick, the two kicks — //! are selected explicitly by the scheduler in `robotd`, which owns the priority rules. //! Every network shares the one 61-D observation layout, so a skill is a session choice -//! plus a command-block encoding, never a new contract. +//! plus a command-block encoding. LSTM exports additionally pass hidden/cell state, +//! owned by each network; sensor observations and joint actions are unchanged. //! //! **Everything is validated at load, not at inference.** A bundle with the wrong //! observation width, the wrong action count, or a missing ONNX Runtime must fail while the @@ -17,7 +18,9 @@ use std::sync::OnceLock; use ort::session::Session; use ort::session::builder::GraphOptimizationLevel; -use ort::value::{Value, ValueType}; +use ort::tensor::TensorElementType; +use ort::value::{Tensor, Value, ValueType}; +use sha2::{Digest, Sha256}; use crate::obs::{ACTION_LEN, OBS_LEN, Observation}; @@ -34,6 +37,12 @@ const INTRA_THREADS: usize = 1; #[derive(Debug, thiserror::Error)] pub enum PolicyError { + #[error("reading {path}: {source}")] + Read { + path: PathBuf, + #[source] + source: std::io::Error, + }, #[error("loading {path}: {source}")] Load { path: PathBuf, @@ -68,12 +77,14 @@ pub enum PolicyError { impl PolicyError { /// The file this error is about, when it is about one. /// - /// `Load` and `Shape` name a file; a missing runtime or an `ort` panic does not, and + /// `Read`, `Load` and `Shape` name a file; a missing runtime or an `ort` panic does not, and /// blaming whichever policy happened to be loading when the dylib turned out to be absent /// would send an operator to replace a file that is fine. pub fn path(&self) -> Option<&Path> { match self { - PolicyError::Load { path, .. } | PolicyError::Shape { path, .. } => Some(path), + PolicyError::Read { path, .. } + | PolicyError::Load { path, .. } + | PolicyError::Shape { path, .. } => Some(path), PolicyError::Inference(_) | PolicyError::RuntimeMissing { .. } | PolicyError::RuntimePanic { .. } => None, @@ -227,12 +238,13 @@ pub struct PolicyPaths { /// release, so a missing or corrupt file is a broken bundle, and the right outcome is /// "unhealthy, roll it back", not a robot that silently lost its kick. pub struct Policy { - walk: Session, - stand: Option, - sitstand: Option, - ground_pick: Option, - skills: Vec, + walk: Network, + stand: Option, + sitstand: Option, + ground_pick: Option, + skills: Vec, standing_threshold: f64, + active: Option, /// Roller mode and fall-recovery mode reserve the standing network (roller has none; /// fall recovery keeps it for getting up), so command magnitude must never select it. standing_disabled: bool, @@ -255,15 +267,16 @@ impl Policy { // It also proves ONNX Runtime is actually present and usable, which with // `load-dynamic` is not known until something is run. let zero = Observation::zeroed(); - fn open_warm(path: &Path, zero: &Observation) -> Result { - let mut session = open(path)?; - run(&mut session, path, zero)?; - Ok(session) + fn open_warm(path: &Path, zero: &Observation) -> Result { + let mut network = open(path)?; + network.run(zero)?; + network.reset(); + Ok(network) } fn open_opt( path: &Option, zero: &Observation, - ) -> Result, PolicyError> { + ) -> Result, PolicyError> { path.as_deref().map(|p| open_warm(p, zero)).transpose() } @@ -278,6 +291,7 @@ impl Policy { .map(|path| open_warm(path, &zero)) .collect::, _>>()?, standing_threshold, + active: None, standing_disabled: false, }) }) @@ -324,18 +338,92 @@ impl Policy { observation: &Observation, net: Net, ) -> Result<[f32; ACTION_LEN], PolicyError> { - let session = match net { - Net::Walk => None, + // Resolve fallback before comparing: asking for an absent skill must not reset + // the walking network on every tick. + let net = match net { + Net::Stand if self.stand.is_none() => Net::Walk, + Net::SitStand if self.sitstand.is_none() => Net::Walk, + Net::GroundPick if self.ground_pick.is_none() => Net::Walk, + Net::Skill(i) if i >= self.skills.len() => Net::Walk, + net => net, + }; + let changed = self.active != Some(net); + let network = match net { + Net::Walk => &mut self.walk, + Net::Stand => self.stand.as_mut().unwrap(), + Net::SitStand => self.sitstand.as_mut().unwrap(), + Net::GroundPick => self.ground_pick.as_mut().unwrap(), + Net::Skill(i) => &mut self.skills[i], + }; + if changed { + network.reset(); + } + let result = network.run(observation); + // Never carry a failed inference's state into another control tick. + if result.is_err() { + network.reset(); + self.active = None; + } else { + self.active = Some(net); + } + result + } + + /// Preserve the running network when only another slot changed. Compare model + /// bytes, not paths: a reload may replace a file in place, and a seated swap may + /// intentionally replace the active network. Those cases must start fresh. + pub fn carry_over(&mut self, from: &Self) { + self.reset(); + let Some(net) = from.active else { + return; + }; + let target = match net { + Net::Walk => Some(&mut self.walk), Net::Stand => self.stand.as_mut(), Net::SitStand => self.sitstand.as_mut(), Net::GroundPick => self.ground_pick.as_mut(), - Net::Skill(index) => self.skills.get_mut(index), + Net::Skill(i) => self.skills.get_mut(i), }; - let session = match session { - Some(session) => session, - None => &mut self.walk, + let source = match net { + Net::Walk => Some(&from.walk), + Net::Stand => from.stand.as_ref(), + Net::SitStand => from.sitstand.as_ref(), + Net::GroundPick => from.ground_pick.as_ref(), + Net::Skill(i) => from.skills.get(i), }; - run(session, Path::new(""), observation) + if let (Some(target), Some(source)) = (target, source) + && target.digest == source.digest + { + if let (Some(dst), Some(src)) = (&mut target.state, &source.state) { + dst.h + .try_extract_tensor_mut::() + .unwrap() + .1 + .copy_from_slice(src.h.try_extract_tensor::().unwrap().1); + dst.c + .try_extract_tensor_mut::() + .unwrap() + .1 + .copy_from_slice(src.c.try_extract_tensor::().unwrap().1); + } + self.active = Some(net); + } + } + + /// Start a new episode, including when resuming after disable or fall recovery. + /// A network also starts fresh whenever selection switches away and back to it. + pub fn reset(&mut self) { + self.active = None; + self.walk.reset(); + for network in self + .stand + .iter_mut() + .chain(self.sitstand.iter_mut()) + .chain(self.ground_pick.iter_mut()) + .chain(self.skills.iter_mut()) + { + network.reset(); + } } } @@ -364,85 +452,238 @@ pub fn validate(path: &Path) -> Result<(), PolicyError> { catching_ort_panics(|| open(path).map(drop)) } -fn open(path: &Path) -> Result { - let session = Session::builder() - .and_then(|b| b.with_optimization_level(GraphOptimizationLevel::Level3)) - .and_then(|b| b.with_intra_threads(INTRA_THREADS)) - .and_then(|b| b.commit_from_file(path)) - .map_err(|source| PolicyError::Load { - path: path.to_owned(), - source, - })?; +/// The mjlab/rsl_rl LSTM export passes state explicitly. Buffers belong to one +/// session, are allocated at load, and are never shared between policy slots. +struct Network { + session: Session, + state: Option, + action_name: String, + path: PathBuf, + digest: [u8; 32], +} - check_width(path, "observation width", session.inputs(), OBS_LEN)?; - check_width(path, "action count", session.outputs(), ACTION_LEN)?; - Ok(session) +struct LstmState { + h: Tensor, + c: Tensor, } -/// Assert the trailing dimension of a graph's single tensor outlet. -/// -/// The leading dimension is the batch and is usually dynamic (`-1`), so only the last one -/// is checked. That is the one that encodes the contract. -fn check_width( - path: &Path, - what: &'static str, - outlets: &[ort::value::Outlet], - expected: usize, -) -> Result<(), PolicyError> { - let shape = match outlets.first().map(|o| o.dtype()) { - Some(ValueType::Tensor { shape, .. }) => shape, - _ => { - return Err(PolicyError::Shape { - path: path.to_owned(), - what, - expected: expected.to_string(), - got: "not a tensor".into(), - }); +impl Network { + fn reset(&mut self) { + if let Some(state) = &mut self.state { + state.h.try_extract_tensor_mut::().unwrap().1.fill(0.0); + state.c.try_extract_tensor_mut::().unwrap().1.fill(0.0); } - }; + } - let got = shape.iter().last().copied().unwrap_or(-1); - if got != expected as i64 { - return Err(PolicyError::Shape { - path: path.to_owned(), - what, - expected: expected.to_string(), - got: got.to_string(), - }); + fn run(&mut self, observation: &Observation) -> Result<[f32; ACTION_LEN], PolicyError> { + let fail = |e: String| PolicyError::Inference(format!("{}: {e}", self.path.display())); + if !observation.as_slice().iter().all(|v| v.is_finite()) { + return Err(fail("non-finite observation".into())); + } + let input = Value::from_array(([1usize, OBS_LEN], observation.as_slice().to_vec())) + .map_err(|e| fail(format!("building input: {e}")))?; + let outputs = match &self.state { + Some(state) => self.session.run(ort::inputs![ + "obs" => &input, "h_in" => &state.h, "c_in" => &state.c + ]), + None => self.session.run(ort::inputs!["obs" => &input]), + } + .map_err(|e| fail(e.to_string()))?; + let (_, actions) = outputs[self.action_name.as_str()] + .try_extract_tensor::() + .map_err(|e| fail(e.to_string()))?; + if actions.len() != ACTION_LEN || !actions.iter().all(|v| v.is_finite()) { + return Err(fail("expected 14 finite actions".into())); + } + let mut result = [0.0; ACTION_LEN]; + result.copy_from_slice(actions); + if let Some(state) = &mut self.state { + let (hs, h) = outputs["h_out"] + .try_extract_tensor::() + .map_err(|e| fail(e.to_string()))?; + let (cs, c) = outputs["c_out"] + .try_extract_tensor::() + .map_err(|e| fail(e.to_string()))?; + let (expected_h, h_in) = state + .h + .try_extract_tensor_mut::() + .map_err(|e| fail(e.to_string()))?; + let (expected_c, c_in) = state + .c + .try_extract_tensor_mut::() + .map_err(|e| fail(e.to_string()))?; + // Check both before updating either, including dynamic runtime output shapes. + if hs != expected_h || cs != expected_c || !h.iter().chain(c).all(|v| v.is_finite()) { + return Err(fail( + "invalid LSTM output state shape or non-finite state".into(), + )); + } + h_in.copy_from_slice(h); + c_in.copy_from_slice(c); + } + Ok(result) } - Ok(()) } -fn run( - session: &mut Session, +fn shape_error(path: &Path, what: &'static str, expected: &str, got: String) -> PolicyError { + PolicyError::Shape { + path: path.to_owned(), + what, + expected: expected.into(), + got, + } +} + +/// Require an exact rank and float32 type. Batch may be symbolic, but this runtime +/// always supplies batch one; state layers and hidden width must be known at load. +fn tensor_shape(path: &Path, outlet: &ort::value::Outlet) -> Result, PolicyError> { + match outlet.dtype() { + ValueType::Tensor { + ty: TensorElementType::Float32, + shape, + .. + } => Ok(shape.to_vec()), + other => Err(shape_error( + path, + "tensor type", + "float32", + format!("{}: {other:?}", outlet.name()), + )), + } +} + +fn outlet<'a>( path: &Path, - observation: &Observation, -) -> Result<[f32; ACTION_LEN], PolicyError> { - let input = Value::from_array(([1usize, OBS_LEN], observation.as_slice().to_vec())) - .map_err(|e| PolicyError::Inference(format!("{}: building input: {e}", path.display())))?; - - let outputs = session - .run(ort::inputs!["obs" => &input]) - .map_err(|e| PolicyError::Inference(format!("{}: {e}", path.display())))?; - - let value = outputs - .values() - .next() - .ok_or_else(|| PolicyError::Inference(format!("{}: no output", path.display())))?; - let (_, data) = value.try_extract_tensor::().map_err(|e| { - PolicyError::Inference(format!("{}: extracting output: {e}", path.display())) - })?; + outlets: &'a [ort::value::Outlet], + name: &str, +) -> Result<&'a ort::value::Outlet, PolicyError> { + outlets.iter().find(|o| o.name() == name).ok_or_else(|| { + shape_error( + path, + "tensor names", + name, + format!("{:?}", outlets.iter().map(|o| o.name()).collect::>()), + ) + }) +} - if data.len() != ACTION_LEN { - return Err(PolicyError::Inference(format!( - "{}: {} actions, expected {ACTION_LEN}", - path.display(), - data.len() - ))); +fn check_matrix(path: &Path, outlet: &ort::value::Outlet, width: usize) -> Result<(), PolicyError> { + let shape = tensor_shape(path, outlet)?; + if shape.len() != 2 || (shape[0] != 1 && shape[0] != -1) || shape[1] != width as i64 { + return Err(shape_error( + path, + "tensor shape", + &format!("[1 or dynamic, {width}]"), + format!("{}: {shape:?}", outlet.name()), + )); } - let mut actions = [0.0f32; ACTION_LEN]; - actions.copy_from_slice(data); - Ok(actions) + Ok(()) +} + +fn open(path: &Path) -> Result { + let bytes = std::fs::read(path).map_err(|source| PolicyError::Read { + path: path.to_owned(), + source, + })?; + let digest: [u8; 32] = Sha256::digest(&bytes).into(); + let session = Session::builder() + .and_then(|b| b.with_optimization_level(GraphOptimizationLevel::Level3)) + .and_then(|b| b.with_intra_threads(INTRA_THREADS)) + .and_then(|b| b.commit_from_file(path)) + .map_err(|source| PolicyError::Load { + path: path.to_owned(), + source, + })?; + + let inputs = session.inputs(); + let outputs = session.outputs(); + check_matrix(path, outlet(path, inputs, "obs")?, OBS_LEN)?; + let recurrent = match (inputs.len(), outputs.len()) { + (1, 1) => false, + (3, 3) => true, + counts => { + return Err(shape_error( + path, + "input/output contract", + "obs -> actions, or obs/h_in/c_in -> actions/h_out/c_out", + format!("{counts:?} tensors"), + )); + } + }; + // Preserve the existing feed-forward output naming contract (the sole output). + let action = if recurrent { + outlet(path, outputs, "actions")? + } else { + &outputs[0] + }; + check_matrix(path, action, ACTION_LEN)?; + let action_name = action.name().to_owned(); + let state = if recurrent { + let mut shapes = Vec::new(); + for (outlets, name) in [ + (inputs, "h_in"), + (inputs, "c_in"), + (outputs, "h_out"), + (outputs, "c_out"), + ] { + let shape = tensor_shape(path, outlet(path, outlets, name)?)?; + if shape.len() != 3 + || shape[0] <= 0 + || (shape[1] != 1 && shape[1] != -1) + || shape[2] <= 0 + { + return Err(shape_error( + path, + "LSTM state shape", + "[positive layers, 1 or dynamic batch, positive hidden size]", + format!("{name}: {shape:?}"), + )); + } + shapes.push(vec![shape[0], 1, shape[2]]); + } + if shapes.iter().any(|shape| shape != &shapes[0]) { + return Err(shape_error( + path, + "LSTM state shapes", + "matching h/c input and output shapes", + format!("{shapes:?}"), + )); + } + let shape = &shapes[0]; + let count = usize::try_from(shape[0]) + .ok() + .and_then(|n| n.checked_mul(shape[2] as usize)) + .filter(|&n| n <= 1_048_576) + .ok_or_else(|| { + shape_error( + path, + "LSTM state size", + "at most 1048576 elements per state", + format!("{shape:?}"), + ) + })?; + let make = || { + Tensor::from_array((shape.clone(), vec![0.0f32; count])).map_err(|source| { + PolicyError::Load { + path: path.to_owned(), + source, + } + }) + }; + Some(LstmState { + h: make()?, + c: make()?, + }) + } else { + None + }; + Ok(Network { + session, + state, + action_name, + path: path.to_owned(), + digest, + }) } #[cfg(test)] diff --git a/duck-control/src/safety.rs b/duck-control/src/safety.rs index 84247813..77db56d4 100644 --- a/duck-control/src/safety.rs +++ b/duck-control/src/safety.rs @@ -167,6 +167,18 @@ impl Safety { self.io.set_torque(on) } + /// Reboot these servos, and forget the gain cache: a rebooted servo comes back at its EEPROM + /// gains, and a cache that still says the running gain is written would leave it there. The + /// next [`Self::apply`] rewrites the gains on every servo. + pub fn reboot_motors(&mut self, ids: &[u8]) -> Result<(), IoError> { + for &id in ids { + tracing::warn!(id, "rebooting servo"); + self.io.reboot(id)?; + } + self.gain = None; + Ok(()) + } + /// The gain last written to the servos, or `None` before the first write. This is what /// the robot is running at, which is not always what the caller asked for. pub fn gain(&self) -> Option { @@ -288,6 +300,26 @@ impl Safety { #[cfg(test)] mod tests { + /// A reboot forgets the gain cache, so the rebooted servos get their gains back on the next + /// apply instead of running at whatever their EEPROM says. + #[test] + fn rebooting_motors_rewrites_the_gain_on_the_next_apply() { + use crate::io::FakeIo; + use crate::model::DEFAULT_POSITION; + let mut s = Safety::new(FakeIo::at(DEFAULT_POSITION), SafetyConfig::default()); + s.apply(DEFAULT_POSITION, DEFAULT_POSITION, 200).unwrap(); + assert_eq!(s.gain(), Some(200)); + s.reboot_motors(&[3, 11]).unwrap(); + assert_eq!(s.io().reboots, vec![3, 11]); + assert_eq!( + s.gain(), + None, + "the gain cache must be forgotten after a reboot" + ); + s.apply(DEFAULT_POSITION, DEFAULT_POSITION, 200).unwrap(); + assert_eq!(s.gain(), Some(200)); + } + use super::*; use crate::imu::ImuData; use crate::io::FakeIo; diff --git a/duck-control/src/sim.rs b/duck-control/src/sim.rs new file mode 100644 index 00000000..8715d343 --- /dev/null +++ b/duck-control/src/sim.rs @@ -0,0 +1,441 @@ +//! A [`RobotIo`] whose robot is in MuJoCo. +//! +//! **The third backend the design doc named.** `docs/design/robotd-design.md` §9 deferred "the +//! MuJoCo backend and the `RemoteIo` protocol"; this is it. Everything above this trait — the +//! control loop, the policy, `Safety`, fall detection, odometry, kinematics, every IPC call and +//! `robotctl` — runs unchanged and cannot tell the difference. That is the whole point: the seam +//! is the one place a simulator is allowed to exist. +//! +//! ## Why TCP and not a unix socket +//! +//! Two reasons, both learned rather than assumed. A unix path is capped at `SUN_LEN` — about 108 +//! bytes — which a scratch directory blows through immediately. And the simulator has to be +//! reachable from *outside* whatever the daemons run in: a container on Linux, and on macOS a Linux +//! VM with MuJoCo on the host beside it. A port crosses all of those; a socket path does not. +//! +//! ## Why JSON +//! +//! One tick is fifteen joints in and fifteen out — about a kilobyte, so 50 KB/s at the loop's 50 Hz, +//! which is nothing next to being able to read a frame with `nc` and write the other half of it in +//! twenty lines of Python. The alternative is a packed struct shared between two repositories in +//! two languages, which is exactly the shape of thing this project has already lost days to when an +//! offset was wrong and the failure was silent. +//! +//! ## What it does when the simulator goes away +//! +//! It goes away *often*: MuJoCo compiles its model, so changing the number of ducks means restarting +//! it, and the ducks are expected to survive that. So a dead connection is an error returned to the +//! caller and a reconnect on the next call — no backoff thread, because **the control loop is the +//! retry timer**. `robotd` already treats a failed `read` as a tick to skip rather than a reason to +//! exit; this is the same tolerance it has for a missing `tofd`. + +use std::io::{BufRead, BufReader, Write}; +use std::net::{TcpStream, ToSocketAddrs}; +use std::time::Duration; + +use serde::{Deserialize, Serialize}; + +use crate::imu::ImuData; +use crate::io::{IoError, JointTargets, Result, RobotIo, Sensors, SlowSensors}; +use crate::model::NUM_JOINTS; + +/// Bumped when the wire format changes in a way an older peer would misread. +/// +/// Checked in the handshake and reported by *both* numbers when it fails, because the two halves +/// live in two repositories and "the simulator is too old" and "the daemon is too old" are the same +/// symptom otherwise. +pub const PROTOCOL: u32 = 1; + +/// How long to wait for the simulator to answer one request. +/// +/// Comfortably longer than a tick, because a simulator that has just been asked to step a +/// contact-heavy scene can be late without being broken — and comfortably shorter than forever, +/// because a wedged simulator must not wedge the control loop with it. +const TIMEOUT: Duration = Duration::from_millis(200); + +/// One request. `op` is the tag, so a frame is readable in a log without a decoder. +#[derive(Debug, Serialize)] +#[serde(tag = "op")] +enum Request<'a> { + #[serde(rename = "hello")] + Hello { protocol: u32, joints: usize }, + #[serde(rename = "read")] + Read, + #[serde(rename = "write")] + Write { targets: &'a [f64; NUM_JOINTS] }, + #[serde(rename = "gain")] + Gain { kp: u16 }, + #[serde(rename = "torque")] + Torque { on: bool }, + #[serde(rename = "slow")] + Slow, +} + +#[derive(Debug, Deserialize)] +struct Hello { + protocol: u32, +} + +/// What the simulator reports, in exactly the units [`Sensors`] wants — radians, rad/s, mA, and the +/// IMU already resolved into the trunk frame. +/// +/// **The simulator does the conversion, not this.** MuJoCo knows its own model's joint order, +/// scaling and frame; a translation layer here would be a second place for that knowledge to live +/// and drift. What crosses the wire is the robot's own units. +#[derive(Debug, Deserialize)] +struct SensorFrame { + positions: [f64; NUM_JOINTS], + velocities: [f64; NUM_JOINTS], + #[serde(default)] + currents_ma: [f64; NUM_JOINTS], + imu: ImuFrame, +} + +#[derive(Debug, Deserialize)] +struct ImuFrame { + gyro: [f64; 3], + gravity: [f64; 3], + quat: [f64; 4], +} + +#[derive(Debug, Deserialize)] +struct SlowFrame { + volts: f64, + temps_c: [f64; NUM_JOINTS], +} + +/// An acknowledgement, so a write that the simulator refused is not silently a write that worked. +#[derive(Debug, Deserialize)] +struct Ack { + #[serde(default)] + error: Option, +} + +/// A robot in MuJoCo, reached over TCP. +pub struct RemoteIo { + addr: String, + link: Option, + /// Reported once rather than every tick a disconnected loop runs. + complained: bool, +} + +struct Link { + write: TcpStream, + read: BufReader, +} + +impl RemoteIo { + /// Name the simulator. Nothing is connected until the first call — a daemon must start whether + /// or not the simulator is up yet, exactly as it starts without a robot on the bus. + pub fn at(addr: impl Into) -> Self { + Self { + addr: addr.into(), + link: None, + complained: false, + } + } + + fn connect(&mut self) -> Result<&mut Link> { + if self.link.is_none() { + let address = self + .addr + .to_socket_addrs() + .and_then(|mut a| { + a.next() + .ok_or_else(|| std::io::Error::other("resolved to no address")) + }) + .map_err(|source| IoError::Port { + path: self.addr.clone(), + source, + })?; + + // A connect timeout as well as a read one: a port nobody listens on refuses instantly, + // but a host that silently drops packets would otherwise hang the loop. + let stream = + TcpStream::connect_timeout(&address, TIMEOUT).map_err(|source| IoError::Port { + path: self.addr.clone(), + source, + })?; + + // **Nagle would be catastrophic here and silent.** It delays a small write waiting for + // more to send, up to ~40 ms — twice the tick — turning every transaction into a + // missed deadline that looks like a slow simulator. + let _ = stream.set_nodelay(true); + let _ = stream.set_read_timeout(Some(TIMEOUT)); + let _ = stream.set_write_timeout(Some(TIMEOUT)); + + let read = BufReader::new(stream.try_clone().map_err(|source| IoError::Port { + path: self.addr.clone(), + source, + })?); + let mut link = Link { + write: stream, + read, + }; + + let hello: Hello = exchange( + &mut link, + &Request::Hello { + protocol: PROTOCOL, + joints: NUM_JOINTS, + }, + )?; + if hello.protocol != PROTOCOL { + return Err(IoError::Bus(format!( + "the simulator speaks protocol {} and this daemon speaks {PROTOCOL} — one of \ + the two is out of date, and they are in different repositories", + hello.protocol + ))); + } + + tracing::info!(addr = %self.addr, protocol = PROTOCOL, "simulated body"); + self.complained = false; + self.link = Some(link); + } + Ok(self.link.as_mut().expect("just connected")) + } + + /// One request and its answer, dropping the connection if anything about it fails. + /// + /// Dropping on *any* error, not only on a closed socket: a frame that will not parse means the + /// two ends disagree about where a message begins, and there is no way to resynchronise a + /// line protocol except by starting again. + fn call Deserialize<'de>>(&mut self, request: &Request<'_>) -> Result { + let result = self.connect().and_then(|link| exchange(link, request)); + if let Err(error) = &result { + self.link = None; + if !self.complained { + self.complained = true; + tracing::warn!( + addr = %self.addr, %error, + "no simulated body; retrying every tick until it answers" + ); + } + } + result + } +} + +fn exchange Deserialize<'de>>(link: &mut Link, request: &Request<'_>) -> Result { + let mut line = serde_json::to_string(request).map_err(|e| IoError::Bus(e.to_string()))?; + line.push('\n'); + link.write + .write_all(line.as_bytes()) + .map_err(|e| IoError::Bus(format!("sending {}: {e}", tag(request))))?; + + let mut answer = String::new(); + let read = link + .read + .read_line(&mut answer) + .map_err(|e| IoError::Bus(format!("waiting for {}: {e}", tag(request))))?; + if read == 0 { + return Err(IoError::Bus(format!( + "the simulator closed the connection during {}", + tag(request) + ))); + } + serde_json::from_str(&answer).map_err(|e| { + IoError::Bus(format!( + "{} answered with something this cannot read: {e}", + tag(request) + )) + }) +} + +fn tag(request: &Request<'_>) -> &'static str { + match request { + Request::Hello { .. } => "hello", + Request::Read => "read", + Request::Write { .. } => "write", + Request::Gain { .. } => "gain", + Request::Torque { .. } => "torque", + Request::Slow => "slow", + } +} + +fn acked(ack: Ack, what: &str) -> Result<()> { + match ack.error { + None => Ok(()), + Some(why) => Err(IoError::Bus(format!("the simulator refused {what}: {why}"))), + } +} + +impl RobotIo for RemoteIo { + fn read(&mut self) -> Result { + let frame: SensorFrame = self.call(&Request::Read)?; + Ok(Sensors { + positions: frame.positions, + velocities: frame.velocities, + currents_ma: frame.currents_ma, + imu: ImuData { + gyro: frame.imu.gyro, + gravity: frame.imu.gravity, + quat: frame.imu.quat, + }, + }) + } + + fn write(&mut self, targets: &JointTargets) -> Result<()> { + let ack: Ack = self.call(&Request::Write { + targets: &targets.positions, + })?; + acked(ack, "the joint targets") + } + + fn set_gain(&mut self, kp: u16) -> Result<()> { + let ack: Ack = self.call(&Request::Gain { kp })?; + acked(ack, "the position gain") + } + + fn set_torque(&mut self, on: bool) -> Result<()> { + let ack: Ack = self.call(&Request::Torque { on })?; + acked(ack, if on { "torque on" } else { "torque off" }) + } + + /// A simulated servo has no latched hardware error to clear and no firmware to restart, so + /// there is nothing to send: the reboot succeeds at once and the caller restores the gains as + /// it would on a robot. Deliberately not an op on the wire — the simulator would only have to + /// answer it with an ack. + fn reboot(&mut self, id: u8) -> Result<()> { + tracing::debug!(id, "reboot of a simulated servo: nothing to do"); + Ok(()) + } + + fn slow_sensors(&mut self) -> Result { + let frame: SlowFrame = self.call(&Request::Slow)?; + Ok(SlowSensors { + volts: frame.volts, + temps_c: frame.temps_c, + }) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::net::TcpListener; + use std::thread; + + /// A simulator made of canned answers: one script per connection, one answer per request. + /// + /// A real socket rather than a trait behind the socket, because the things worth testing here + /// *are* the socket — a peer that hangs up, a frame that will not parse, a second connection + /// after the first died. + fn simulator(scripts: Vec>) -> (String, thread::JoinHandle>) { + let listener = TcpListener::bind("127.0.0.1:0").expect("a loopback port"); + let addr = listener.local_addr().expect("its address").to_string(); + let handle = thread::spawn(move || { + let mut heard = Vec::new(); + for script in scripts { + let (stream, _) = listener.accept().expect("a connection"); + let mut out = stream.try_clone().expect("a writer"); + let mut lines = BufReader::new(stream); + for reply in script { + let mut line = String::new(); + if lines.read_line(&mut line).expect("a request") == 0 { + break; + } + heard.push(line.trim().to_string()); + out.write_all(reply.as_bytes()).expect("a reply"); + out.write_all(b"\n").expect("a newline"); + } + } + heard + }); + (addr, handle) + } + + const HELLO: &str = r#"{"protocol":1}"#; + // One line, because the protocol is one frame per line and a test fixture that wraps would be + // testing the fixture. + const SENSORS: &str = concat!( + r#"{"positions":[0.1,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"#, + r#""velocities":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0.5],"#, + r#""currents_ma":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"#, + r#""imu":{"gyro":[0,0,0],"gravity":[0,0,-1],"quat":[1,0,0,0]}}"# + ); + const OK: &str = r#"{}"#; + + #[test] + fn a_read_carries_the_sensors_the_loop_expects() { + let (addr, sim) = simulator(vec![vec![HELLO, SENSORS]]); + let mut io = RemoteIo::at(addr); + let sensors = io.read().expect("a sensor frame"); + assert_eq!(sensors.positions[0], 0.1); + assert_eq!(sensors.velocities[NUM_JOINTS - 1], 0.5); + assert_eq!(sensors.imu.gravity, [0.0, 0.0, -1.0]); + + let heard = sim.join().expect("the simulator thread"); + assert!(heard[0].contains(r#""op":"hello""#), "{heard:?}"); + assert!(heard[1].contains(r#""op":"read""#), "{heard:?}"); + } + + #[test] + fn the_targets_cross_in_joint_order() { + let (addr, sim) = simulator(vec![vec![HELLO, OK]]); + let mut io = RemoteIo::at(addr); + let mut positions = [0.0; NUM_JOINTS]; + positions[2] = -0.4579; + io.write(&JointTargets::new(positions)).expect("a write"); + + let heard = sim.join().expect("the simulator thread"); + // Positional, exactly as the wire everywhere else in this project is: the third number is + // the third joint, and nothing names it. + assert!(heard[1].contains("[0.0,0.0,-0.4579,"), "{heard:?}"); + } + + #[test] + fn a_protocol_mismatch_names_both_versions() { + let (addr, _sim) = simulator(vec![vec![r#"{"protocol":99}"#]]); + let mut io = RemoteIo::at(addr); + let error = io + .read() + .expect_err("a mismatch is fatal to the connection"); + let said = error.to_string(); + assert!(said.contains("99") && said.contains("1"), "{said}"); + assert!(said.contains("different repositories"), "{said}"); + } + + #[test] + fn a_refusal_is_an_error_rather_than_a_silent_success() { + let (addr, _sim) = simulator(vec![vec![HELLO, r#"{"error":"torque is disabled"}"#]]); + let mut io = RemoteIo::at(addr); + let error = io + .set_torque(true) + .expect_err("a refusal must not read as done"); + assert!(error.to_string().contains("torque is disabled"), "{error}"); + } + + #[test] + fn a_simulator_that_restarts_is_reconnected_to_on_the_next_tick() { + // The case this exists for: MuJoCo compiles its model, so changing the number of ducks + // restarts it — and the ducks are expected to survive that. The first connection dies + // mid-read; the second is expected to be made without anyone asking. + let (addr, sim) = simulator(vec![vec![HELLO], vec![HELLO, SENSORS]]); + let mut io = RemoteIo::at(addr); + + io.read().expect_err("the simulator hung up"); + let sensors = io.read().expect("reconnected on the next call"); + assert_eq!(sensors.positions[0], 0.1); + + let heard = sim.join().expect("the simulator thread"); + let hellos = heard.iter().filter(|l| l.contains("hello")).count(); + assert_eq!( + hellos, 2, + "the second connection must handshake again: {heard:?}" + ); + } + + #[test] + fn a_frame_that_will_not_parse_drops_the_connection() { + // Not merely an error: a half-read line means the two ends disagree about where a message + // starts, and a line protocol cannot resynchronise except by starting again. + let (addr, sim) = simulator(vec![vec![HELLO, "not json"], vec![HELLO, SENSORS]]); + let mut io = RemoteIo::at(addr); + io.read().expect_err("garbage is an error"); + io.read().expect("a fresh connection, not a wedged one"); + + let heard = sim.join().expect("the simulator thread"); + assert_eq!(heard.iter().filter(|l| l.contains("hello")).count(), 2); + } +} diff --git a/duck-control/tests/fixtures/bad_action_count.onnx b/duck-control/tests/fixtures/bad_action_count.onnx new file mode 100644 index 00000000..2040531d Binary files /dev/null and b/duck-control/tests/fixtures/bad_action_count.onnx differ diff --git a/duck-control/tests/fixtures/bad_batch.onnx b/duck-control/tests/fixtures/bad_batch.onnx new file mode 100644 index 00000000..4cb3d50e Binary files /dev/null and b/duck-control/tests/fixtures/bad_batch.onnx differ diff --git a/duck-control/tests/fixtures/bad_rank.onnx b/duck-control/tests/fixtures/bad_rank.onnx new file mode 100644 index 00000000..f0399159 Binary files /dev/null and b/duck-control/tests/fixtures/bad_rank.onnx differ diff --git a/duck-control/tests/fixtures/bad_state_shape.onnx b/duck-control/tests/fixtures/bad_state_shape.onnx new file mode 100644 index 00000000..245643bc Binary files /dev/null and b/duck-control/tests/fixtures/bad_state_shape.onnx differ diff --git a/duck-control/tests/fixtures/bad_width.onnx b/duck-control/tests/fixtures/bad_width.onnx new file mode 100644 index 00000000..2b24d32f Binary files /dev/null and b/duck-control/tests/fixtures/bad_width.onnx differ diff --git a/duck-control/tests/fixtures/dynamic_batch.onnx b/duck-control/tests/fixtures/dynamic_batch.onnx new file mode 100644 index 00000000..923abe99 Binary files /dev/null and b/duck-control/tests/fixtures/dynamic_batch.onnx differ diff --git a/duck-control/tests/fixtures/dynamic_hidden.onnx b/duck-control/tests/fixtures/dynamic_hidden.onnx new file mode 100644 index 00000000..264daa3d Binary files /dev/null and b/duck-control/tests/fixtures/dynamic_hidden.onnx differ diff --git a/duck-control/tests/fixtures/extra_input.onnx b/duck-control/tests/fixtures/extra_input.onnx new file mode 100644 index 00000000..3b98c2f1 Binary files /dev/null and b/duck-control/tests/fixtures/extra_input.onnx differ diff --git a/duck-control/tests/fixtures/feedforward.onnx b/duck-control/tests/fixtures/feedforward.onnx new file mode 100644 index 00000000..2c97e5ea Binary files /dev/null and b/duck-control/tests/fixtures/feedforward.onnx differ diff --git a/duck-control/tests/fixtures/generate.py b/duck-control/tests/fixtures/generate.py new file mode 100644 index 00000000..1f74e0e9 --- /dev/null +++ b/duck-control/tests/fixtures/generate.py @@ -0,0 +1,79 @@ +"""Regenerate tiny contract fixtures: python with onnx + numpy installed. + +The recurrent fixture uses the real ONNX LSTM operator, with deterministic weights. +No trained model or hardware is needed. Output order deliberately differs from mjlab. +""" +from pathlib import Path +import copy +import numpy as np +import onnx +from onnx import TensorProto as T, helper as h, numpy_helper as nh + +ROOT = Path(__file__).parent + +def info(name, shape, dtype=T.FLOAT): + return h.make_tensor_value_info(name, dtype, shape) + +def save(name, nodes, inputs, outputs, initializers): + model = h.make_model(h.make_graph(nodes, name, inputs, outputs, initializers), + opset_imports=[h.make_opsetid('', 17)], ir_version=8) + onnx.checker.check_model(model) + onnx.save(model, ROOT / (name + '.onnx')) + return model + +ff = save('feedforward', [h.make_node('Gather', ['obs', 'indices'], ['output'], axis=1)], + [info('obs', [1, 61])], [info('output', [1, 14])], + [nh.from_array(np.arange(14, dtype=np.int64), 'indices')]) +rng = np.random.default_rng(42) +weights = [nh.from_array(rng.normal(0, .15, shape).astype('float32'), name) + for name, shape in [('W', (1, 8, 61)), ('R', (1, 8, 2)), ('B', (1, 16))]] +model = save('lstm', [ + h.make_node('Unsqueeze', ['obs', 'axis'], ['x']), + h.make_node('LSTM', ['x', 'W', 'R', 'B', '', 'h_in', 'c_in'], + ['y', 'h_out', 'c_out'], hidden_size=2), + h.make_node('Squeeze', ['h_out', 'axis'], ['flat']), + h.make_node('Tile', ['flat', 'repeats'], ['actions']), +], [info('c_in', [1, 1, 2]), info('obs', [1, 61]), info('h_in', [1, 1, 2])], + [info('c_out', [1, 1, 2]), info('actions', [1, 14]), info('h_out', [1, 1, 2])], + weights + [nh.from_array(np.array([0], dtype=np.int64), 'axis'), + nh.from_array(np.array([1, 7], dtype=np.int64), 'repeats')]) + +def variant(name, mutate, source=model): + m = copy.deepcopy(source) + mutate(m) + # Some invalid contracts also violate graph shape inference: they must be + # rejected, whether by ORT itself or by our load-time contract validation. + onnx.save(m, ROOT / (name + '.onnx')) + +def dimension(m, io, name, index, value): + entry = next(x for x in getattr(m.graph, io) if x.name == name) + d = entry.type.tensor_type.shape.dim[index] + d.ClearField('dim_param') + if isinstance(value, str): + d.ClearField('dim_value') + d.dim_param = value + else: + d.dim_value = value + +variant('bad_width', lambda m: dimension(m, 'input', 'obs', 1, 60)) +variant('bad_batch', lambda m: dimension(m, 'input', 'obs', 0, 2)) +variant('bad_state_shape', lambda m: dimension(m, 'input', 'c_in', 2, 3)) +variant('dynamic_hidden', lambda m: dimension(m, 'input', 'h_in', 2, 'hidden')) +variant('missing_state', lambda m: m.graph.input.remove(m.graph.input[0])) +variant('extra_input', lambda m: m.graph.input.append(info('unexpected', [1]))) +variant('wrong_type', lambda m: setattr(m.graph.input[0].type.tensor_type, 'elem_type', T.DOUBLE)) +variant('dynamic_batch', lambda m: [dimension(m, io, x.name, 0 if x.name in ('obs', 'actions') else 1, 'batch') + for io in ('input', 'output') for x in getattr(m.graph, io)]) +variant('bad_rank', lambda m: m.graph.input[0].type.tensor_type.shape.dim.insert(0, h.make_tensor_type_proto(T.FLOAT, [1]).tensor_type.shape.dim[0]), ff) +variant('bad_action_count', lambda m: m.graph.initializer[0].CopyFrom(nh.from_array(np.arange(13, dtype=np.int64), 'indices')), ff) +# Non-finite state should fail during warm-up even when its action output is finite. +nan = copy.deepcopy(model) +for node in nan.graph.node: + for i, name in enumerate(node.output): + if name == 'c_out': node.output[i] = 'cell' +nan.graph.node.append(h.make_node('Add', ['cell', 'nan'], ['c_out'])) +nan.graph.initializer.append(nh.from_array(np.array(np.nan, dtype=np.float32), 'nan')) +onnx.save(nan, ROOT / 'nan_state.onnx') + +variant('lstm_changed', lambda m: next(x for x in m.graph.initializer if x.name == 'B').CopyFrom( + nh.from_array(np.full((1, 16), .3, dtype=np.float32), 'B'))) diff --git a/duck-control/tests/fixtures/lstm.onnx b/duck-control/tests/fixtures/lstm.onnx new file mode 100644 index 00000000..befdef32 Binary files /dev/null and b/duck-control/tests/fixtures/lstm.onnx differ diff --git a/duck-control/tests/fixtures/lstm_changed.onnx b/duck-control/tests/fixtures/lstm_changed.onnx new file mode 100644 index 00000000..120e03e0 Binary files /dev/null and b/duck-control/tests/fixtures/lstm_changed.onnx differ diff --git a/duck-control/tests/fixtures/missing_state.onnx b/duck-control/tests/fixtures/missing_state.onnx new file mode 100644 index 00000000..4ddcdcd9 Binary files /dev/null and b/duck-control/tests/fixtures/missing_state.onnx differ diff --git a/duck-control/tests/fixtures/nan_state.onnx b/duck-control/tests/fixtures/nan_state.onnx new file mode 100644 index 00000000..85f3fb90 Binary files /dev/null and b/duck-control/tests/fixtures/nan_state.onnx differ diff --git a/duck-control/tests/fixtures/wrong_type.onnx b/duck-control/tests/fixtures/wrong_type.onnx new file mode 100644 index 00000000..fa008079 Binary files /dev/null and b/duck-control/tests/fixtures/wrong_type.onnx differ diff --git a/duck-control/tests/recurrent_policy.rs b/duck-control/tests/recurrent_policy.rs new file mode 100644 index 00000000..1e6142d9 --- /dev/null +++ b/duck-control/tests/recurrent_policy.rs @@ -0,0 +1,164 @@ +//! Requires ONNX Runtime >= 1.23: cargo test -p duck-control --test recurrent_policy -- --ignored +//! Kept explicit so hosts without the dynamically loaded runtime still run the normal suite. +use duck_control::obs::Observation; +use duck_control::policy::{Net, Policy, PolicyPaths, validate}; +use std::path::PathBuf; + +fn fixture(name: &str) -> PathBuf { + PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("tests/fixtures") + .join(format!("{name}.onnx")) +} +fn load(walk: &str, stand: Option<&str>) -> Policy { + Policy::load( + &PolicyPaths { + walk: fixture(walk), + stand: stand.map(fixture), + ..Default::default() + }, + 0.05, + ) + .unwrap() +} +fn obs() -> Observation { + Observation::from([0.2; 61]) +} + +#[test] +#[ignore = "requires ONNX Runtime >= 1.23"] +fn feedforward_outputs_are_unchanged_by_selection_and_reset() { + let mut p = load("feedforward", None); + for net in [Net::Walk, Net::Stand, Net::Skill(999), Net::Walk] { + assert_eq!(p.infer(&obs(), net).unwrap(), [0.2; 14]); + p.reset(); + } +} + +#[test] +#[ignore = "requires ONNX Runtime >= 1.23"] +fn lstm_carries_state_and_reset_reproduces_first_action() { + let mut p = load("lstm", None); + let first = p.infer(&obs(), Net::Walk).unwrap(); + let second = p.infer(&obs(), Net::Walk).unwrap(); + assert_ne!(first, second, "memory must affect the next step"); + p.reset(); + assert_eq!(first, p.infer(&obs(), Net::Walk).unwrap()); + assert_eq!(second, p.infer(&obs(), Net::Walk).unwrap()); + // Missing slots resolve to the currently active walking network, without resetting it. + let third = p.infer(&obs(), Net::Walk).unwrap(); + p.reset(); + p.infer(&obs(), Net::Walk).unwrap(); + p.infer(&obs(), Net::Stand).unwrap(); + assert_eq!(third, p.infer(&obs(), Net::Skill(5)).unwrap()); +} + +#[test] +#[ignore = "requires ONNX Runtime >= 1.23"] +fn switching_networks_does_not_leak_or_resume_old_state() { + let mut p = load("lstm", Some("lstm")); + let first = p.infer(&obs(), Net::Walk).unwrap(); + assert_ne!(first, p.infer(&obs(), Net::Walk).unwrap()); + assert_eq!(first, p.infer(&obs(), Net::Stand).unwrap()); + assert_ne!(first, p.infer(&obs(), Net::Stand).unwrap()); + assert_eq!(first, p.infer(&obs(), Net::Walk).unwrap()); + let mut mixed = load("lstm", Some("feedforward")); + assert_eq!(first, mixed.infer(&obs(), Net::Walk).unwrap()); + assert_eq!([0.2; 14], mixed.infer(&obs(), Net::Stand).unwrap()); + assert_eq!(first, mixed.infer(&obs(), Net::Walk).unwrap()); +} + +#[test] +#[ignore = "requires ONNX Runtime >= 1.23"] +fn warmup_does_not_become_episode_history_and_dynamic_batch_works() { + let mut p = load("lstm", None); + let first = p.infer(&obs(), Net::Walk).unwrap(); + p.reset(); + assert_eq!(first, p.infer(&obs(), Net::Walk).unwrap()); + let mut dynamic = load("dynamic_batch", None); + assert_eq!(first, dynamic.infer(&obs(), Net::Walk).unwrap()); +} + +#[test] +#[ignore = "requires ONNX Runtime >= 1.23"] +fn unsupported_contracts_fail_before_the_control_loop() { + for name in [ + "bad_width", + "bad_batch", + "bad_state_shape", + "dynamic_hidden", + "missing_state", + "extra_input", + "wrong_type", + "bad_rank", + "bad_action_count", + ] { + assert!(validate(&fixture(name)).is_err(), "accepted {name}"); + } + assert!( + Policy::load( + &PolicyPaths { + walk: fixture("nan_state"), + ..Default::default() + }, + 0.05 + ) + .is_err() + ); +} + +#[test] +#[ignore = "requires ONNX Runtime >= 1.23"] +fn bad_inference_does_not_poison_the_next_episode() { + let mut p = load("lstm", None); + let first = p.infer(&obs(), Net::Walk).unwrap(); + assert!( + p.infer(&Observation::from([f32::NAN; 61]), Net::Walk) + .is_err() + ); + assert_eq!(first, p.infer(&obs(), Net::Walk).unwrap()); +} + +#[test] +#[ignore = "requires ONNX Runtime >= 1.23"] +fn replacing_another_slot_preserves_only_the_unchanged_active_network() { + let mut old = load("lstm", Some("feedforward")); + old.infer(&obs(), Net::Walk).unwrap(); + let mut replacement = load("lstm", Some("lstm")); + replacement.carry_over(&old); + assert_eq!( + old.infer(&obs(), Net::Walk).unwrap(), + replacement.infer(&obs(), Net::Walk).unwrap() + ); + let mut changed = load("feedforward", None); + changed.carry_over(&old); + assert_eq!([0.2; 14], changed.infer(&obs(), Net::Walk).unwrap()); + // The inactive standing slot must still start a new episode. + let mut fresh = load("lstm", None); + assert_eq!( + fresh.infer(&obs(), Net::Walk).unwrap(), + replacement.infer(&obs(), Net::Stand).unwrap() + ); +} + +#[test] +#[ignore = "requires ONNX Runtime >= 1.23"] +fn changed_weights_at_the_same_path_start_with_fresh_memory() { + let path = + std::env::temp_dir().join(format!("microduck-recurrent-{}.onnx", std::process::id())); + std::fs::copy(fixture("lstm"), &path).unwrap(); + let paths = PolicyPaths { + walk: path.clone(), + ..Default::default() + }; + let mut old = Policy::load(&paths, 0.05).unwrap(); + old.infer(&obs(), Net::Walk).unwrap(); + std::fs::copy(fixture("lstm_changed"), &path).unwrap(); + let mut replacement = Policy::load(&paths, 0.05).unwrap(); + replacement.carry_over(&old); + let mut fresh = load("lstm_changed", None); + assert_eq!( + fresh.infer(&obs(), Net::Walk).unwrap(), + replacement.infer(&obs(), Net::Walk).unwrap() + ); + std::fs::remove_file(path).unwrap(); +} diff --git a/duck-detect/models/duck_detect.onnx b/duck-detect/models/duck_detect.onnx deleted file mode 100644 index 42d1e6fc..00000000 Binary files a/duck-detect/models/duck_detect.onnx and /dev/null differ diff --git a/duck-detect/models/duck_detect.rknn b/duck-detect/models/duck_detect.rknn deleted file mode 100644 index 49fce2e2..00000000 Binary files a/duck-detect/models/duck_detect.rknn and /dev/null differ diff --git a/duck-detect/src/lib.rs b/duck-detect/src/lib.rs index 3407f7e5..9a9b2659 100644 --- a/duck-detect/src/lib.rs +++ b/duck-detect/src/lib.rs @@ -228,6 +228,72 @@ pub fn letterbox_from_uyvy( } } +/// The frame as an upright RGB picture, scaled to fit a box, with no padding. +/// +/// [`letterbox_from_uyvy`] above is for the model: a square, padded, at whatever size the network +/// wants. This is for a *person or a program looking at the picture* — a JPEG on its way to a +/// Space that runs a model of its own — so it keeps the aspect ratio and pads nothing. +/// +/// **The turn is applied here rather than reported.** `mediad`'s pipeline deliberately does not +/// rotate: a `videoflip` cost the encoder its zero-copy path and the board 22 fps, so a WebRTC +/// consumer is told the mount angle and turns the picture itself (`media.video`). That reasoning +/// does not carry over to this path, because the conversion is a per-pixel loop either way and the +/// turn is a change of which source pixel is fetched — free, inside a loop that is already +/// running. And what receives these frames is a model, which wants them the way up it was trained +/// on rather than a rotation flag to honour. +/// +/// Returns the size written, which is not `(long, short)` in any predictable order: a quarter turn +/// swaps the axes, so the caller is told rather than left to work it out. +pub fn rgb_from_uyvy( + uyvy: &[u8], + width: usize, + height: usize, + longest: usize, + turn: Turn, + out: &mut Vec, +) -> (usize, usize) { + let (upright_w, upright_h) = turn.upright(width, height); + // Downscale only. Asking for a box bigger than the sensor would interpolate detail that was + // never captured and cost the bandwidth of pretending. + let scale = (longest as f32 / upright_w.max(upright_h) as f32).min(1.0); + let out_w = ((upright_w as f32 * scale).round() as usize).max(1); + let out_h = ((upright_h as f32 * scale).round() as usize).max(1); + let stride = width * 2; + + out.clear(); + out.resize(out_w * out_h * 3, 0); + for y in 0..out_h { + let uy = (y * upright_h) / out_h; + for x in 0..out_w { + let ux = (x * upright_w) / out_w; + let (source_x, source_y) = turn.source(ux, uy, width, height); + let row = source_y * stride; + if row + stride > uyvy.len() { + // A frame that arrives mid-teardown is short. What is missing stays black rather + // than taking the daemon down over a picture — the same call `letterbox_from_uyvy` + // makes. + continue; + } + let pair = row + (source_x / 2) * 4; + // U Y0 V Y1: the luma is the odd byte of the half this pixel falls in. + let luma = uyvy[pair + 1 + 2 * (source_x & 1)] as i32 - 16; + let u = uyvy[pair] as i32 - 128; + let v = uyvy[pair + 2] as i32 - 128; + + // BT.601 limited range in fixed point, as above: the ISP's convention. + let r = (298 * luma + 409 * v + 128) >> 8; + let g = (298 * luma - 100 * u - 208 * v + 128) >> 8; + let b = (298 * luma + 516 * u + 128) >> 8; + + let target = (y * out_w + x) * 3; + out[target] = r.clamp(0, 255) as u8; + out[target + 1] = g.clamp(0, 255) as u8; + out[target + 2] = b.clamp(0, 255) as u8; + } + } + (out_w, out_h) +} + /// Candidates over `threshold`, suppressed, and mapped back to the original frame. /// /// **The head does not suppress anything.** 2100 candidates means one duck comes back as twenty @@ -289,6 +355,42 @@ fn iou(a: &[f32; 4], b: &[f32; 4]) -> f32 { #[cfg(test)] mod tests { + /// A quarter turn swaps the axes and moves a known corner, which is the whole of what a + /// rotation can get wrong: an upside-down picture and an unrotated one have the same shape. + #[test] + fn the_rgb_scaler_turns_the_picture_and_keeps_its_shape() { + // 8x4 UYVY, all grey except the top-left pixel, which is bright. + let (width, height) = (8usize, 4usize); + let mut uyvy = [128u8, 16, 128, 16].repeat(width * height / 2); + uyvy[1] = 235; // Y0 of the first pair: the top-left pixel, white. + + let mut out = Vec::new(); + let (w, h) = rgb_from_uyvy(&uyvy, width, height, 8, Turn::None, &mut out); + assert_eq!((w, h), (8, 4), "unturned, the shape is the frame's"); + assert!(out[0] > 200, "and the bright pixel is top-left: {}", out[0]); + + let (w, h) = rgb_from_uyvy(&uyvy, width, height, 8, Turn::Right, &mut out); + assert_eq!((w, h), (4, 8), "a quarter turn swaps the axes"); + // Turned clockwise, the frame's top-left corner is the picture's top-right — row zero, + // last column. + let top_right = (w - 1) * 3; + assert!( + out[top_right] > 200, + "the bright pixel moved to the top-right: {:?}", + &out[top_right..top_right + 3] + ); + assert!(out[0] < 200, "and is no longer top-left: {}", out[0]); + } + + /// Never upscales: a box larger than the sensor would interpolate detail nobody captured. + #[test] + fn the_rgb_scaler_only_ever_shrinks() { + let uyvy = [128u8, 16, 128, 16].repeat(8 * 4 / 2); + let mut out = Vec::new(); + assert_eq!(rgb_from_uyvy(&uyvy, 8, 4, 64, Turn::None, &mut out), (8, 4)); + assert_eq!(rgb_from_uyvy(&uyvy, 8, 4, 4, Turn::None, &mut out), (4, 2)); + } + use super::*; /// A tall frame fits inside the square with grey above and below, and nothing stretches. diff --git a/duck-ether/Cargo.toml b/duck-ether/Cargo.toml new file mode 100644 index 00000000..b489e1ff --- /dev/null +++ b/duck-ether/Cargo.toml @@ -0,0 +1,15 @@ +[package] +name = "duck-ether" +version.workspace = true +edition.workspace = true +license.workspace = true +description = "A radio for simulated ducks: what one advertises, the ones near it hear" + +[dependencies] +duck-ipc-proto = { path = "../duck-ipc-proto" } +thiserror.workspace = true +clap = { workspace = true, features = ["derive"] } +serde_json = { workspace = true } +tokio = { workspace = true, features = ["rt-multi-thread", "macros", "net", "io-util", "time", "sync"] } +tracing.workspace = true +tracing-subscriber = { workspace = true, features = ["env-filter"] } diff --git a/duck-ether/src/main.rs b/duck-ether/src/main.rs new file mode 100644 index 00000000..8c78dd69 --- /dev/null +++ b/duck-ether/src/main.rs @@ -0,0 +1,466 @@ +//! A radio for simulated ducks: what one advertises, the ones near it hear. +//! +//! ```text +//! duck-ether --duck duck-a=/run/duck-a/robotd.sock@7801 \ +//! --duck duck-b=/run/duck-b/robotd.sock@7802 +//! ``` +//! +//! **This replaces `btd`'s radio and nothing above it.** Presence is already an IPC contract on +//! `robotd`'s own socket — `chorale.subscribe` to be told what to put on the air, `chorale.beacon` +//! carrying it, `chorale.heard` carrying what came back — and `btd` is a *client* of `robotd` +//! rather than a server. So this impersonates nothing and steals no socket path: it holds one +//! connection per duck, exactly as `btd` does, and every duck's election, roster, beat and +//! conductor deference runs unmodified and cannot tell. +//! +//! ## What decides who hears whom +//! +//! Distance, because [`ChoraleHeard`] has no signal strength in it — a real scanner either sees an +//! advertisement or does not, and a beacon out of range simply never arrives. So the ether asks each +//! simulator where its duck is standing and delivers a beacon only to ducks within [`RANGE`] of it. +//! That is a cruder radio than a real one and a far more controllable one: a range that is a number +//! makes "these two can hear each other and those two cannot" a thing you can set up in a second, +//! which on real hardware means carrying robots into other rooms. +//! +//! ## Two things it gets right that are easy to get wrong +//! +//! **`age_us` is an age, not a timestamp.** The field exists because two daemons share a machine and +//! not an epoch, and `robotd` subtracts it from its own clock on arrival — so filling it with a real +//! elapsed time is what keeps the beat synchronisation being exercised rather than short-circuited. +//! +//! **The address rotates.** `from` is documented as an identity for de-duplication only, and a real +//! duck's BLE address changes underneath it — a fact that cost this project a day when something +//! keyed on it. `--rotate` makes that happen on a timer, which turns the bug into a test. +//! +//! ## Being a bad radio on purpose +//! +//! **A perfect ether hides the bugs a real one causes**, and that is not a hypothetical. Four ducks +//! in the twin converge on one piece every time, staggered starts included — because every duck is +//! visible to every other from the moment it boots, instantly and losslessly. On hardware, BLE +//! discovery is slow and lossy, so two ducks can be singing before the other two have seen them, +//! which is exactly the split-brain the chorale's election has to survive. +//! +//! So `--discovery` makes a duck take a while to be *noticed*, per pair rather than globally, +//! because it is the asymmetry that splits a flock: A and B finding each other quickly while C and D +//! are still deaf is the scenario, and one global delay cannot produce it. `--loss` drops a fraction +//! of deliveries. Both are driven by a seeded PRNG, so a split that happens once can be made to +//! happen again — a flaky radio is only useful for debugging if its flakiness repeats. + +use std::collections::HashMap; +use std::path::PathBuf; +use std::sync::Arc; +use std::time::{Duration, Instant}; + +use clap::Parser; +use duck_ipc_proto as proto; +use tokio::io::{AsyncBufReadExt, AsyncWriteExt, BufReader}; +use tokio::net::{TcpStream, UnixStream}; +use tokio::sync::Mutex; + +/// How far a beacon carries, in metres. +/// +/// Generous on purpose: a real advertisement crosses a room and then some, and the interesting +/// failures are about who is *out* of range, which is what `--range` is for. +const RANGE: f64 = 8.0; + +/// How often each duck's beacon is delivered to the ducks near it. +/// +/// A real scanner sees an advertisement repeatedly, not once — `robotd` ages every sighting and +/// drops a peer that stops arriving, so a beacon sent once would be a duck that vanishes. +const DELIVERY: Duration = Duration::from_millis(200); + +#[derive(Parser, Debug)] +#[command(about = "A radio for simulated ducks", version)] +struct Args { + /// A duck, as `name=/path/to/robotd.sock@body-port`. Repeat for each. + #[arg(long = "duck", value_name = "NAME=SOCKET@PORT", required = true)] + ducks: Vec, + + /// How far a beacon carries, in metres. + #[arg(long, default_value_t = RANGE)] + range: f64, + + /// Rotate each duck's address every N seconds, as a real one does. 0 leaves it alone. + #[arg(long, default_value_t = 0)] + rotate: u64, + + /// Take up to this many seconds to notice each duck, per pair. 0 is an instant, perfect radio. + #[arg(long, default_value_t = 0)] + discovery: u64, + + /// Drop this fraction of deliveries, 0.0 to 1.0. A real advertisement is missed often. + #[arg(long, default_value_t = 0.0)] + loss: f64, + + /// Seed for the discovery delays and the losses, so a split can be reproduced. + #[arg(long, default_value_t = 1)] + seed: u64, +} + +/// A radio's imperfections, as numbers. +#[derive(Debug, Clone, Copy)] +struct Weather { + discovery: Duration, + loss: f64, + seed: u64, +} + +/// Deterministic noise from a name and a counter — a splitmix step, which is short, well distributed +/// and needs no dependency. Deterministic is the point: a radio that is flaky differently every run +/// cannot be used to chase a bug. +fn noise(seed: u64, key: &str, salt: u64) -> u64 { + let mut x = seed ^ salt; + for byte in key.as_bytes() { + x = x.wrapping_mul(0x100_0000_01b3) ^ u64::from(*byte); + } + x = (x ^ (x >> 30)).wrapping_mul(0xbf58_476d_1ce4_e5b9); + x = (x ^ (x >> 27)).wrapping_mul(0x94d0_49bb_1331_11eb); + x ^ (x >> 31) +} + +#[derive(Debug, Clone)] +struct Duck { + name: String, + socket: PathBuf, + body: u16, +} + +fn parse_duck(text: &str) -> Result { + let (name, rest) = text + .split_once('=') + .ok_or_else(|| format!("{text:?} is not name=socket@port"))?; + let (socket, port) = rest + .rsplit_once('@') + .ok_or_else(|| format!("{text:?} has no @body-port"))?; + Ok(Duck { + name: name.to_owned(), + socket: PathBuf::from(socket), + body: port + .parse() + .map_err(|_| format!("{port:?} is not a port number"))?, + }) +} + +/// What one duck is putting on the air, and whether it is listening. +#[derive(Default)] +struct OnAir { + beacon: Option, + listening: bool, + /// Where this duck is standing, from its simulator. + at: [f64; 3], + /// When this duck first had something to advertise — discovery is timed from there, because a + /// scanner cannot notice a duck that is not yet on the air. + since: Option, + /// What its radio calls itself today. + address: String, +} + +type Air = Arc>>; + +#[tokio::main(flavor = "current_thread")] +async fn main() -> std::process::ExitCode { + let args = Args::parse(); + tracing_subscriber::fmt() + .with_env_filter( + tracing_subscriber::EnvFilter::try_from_default_env() + .unwrap_or_else(|_| tracing_subscriber::EnvFilter::new("info")), + ) + .init(); + + let ducks: Vec = match args.ducks.iter().map(|d| parse_duck(d)).collect() { + Ok(ducks) => ducks, + Err(why) => { + tracing::error!("{why}"); + return std::process::ExitCode::FAILURE; + } + }; + + let air: Air = Arc::new(Mutex::new(HashMap::new())); + { + let mut on_air = air.lock().await; + for (index, duck) in ducks.iter().enumerate() { + on_air.insert( + duck.name.clone(), + OnAir { + address: address_for(index, 0), + ..Default::default() + }, + ); + } + } + + let weather = Weather { + discovery: Duration::from_secs(args.discovery), + loss: args.loss.clamp(0.0, 1.0), + seed: args.seed, + }; + tracing::info!( + ducks = ducks.len(), + range = args.range, + discovery_s = args.discovery, + loss = weather.loss, + seed = args.seed, + "the ether is open" + ); + + let mut tasks = Vec::new(); + for duck in &ducks { + let (duck, air, range, weather) = (duck.clone(), air.clone(), args.range, weather); + tasks.push(tokio::spawn(async move { + loop { + if let Err(e) = serve(&duck, &air, range, weather).await { + tracing::warn!(duck = %duck.name, error = %e, "lost the duck; retrying"); + } + tokio::time::sleep(Duration::from_secs(1)).await; + } + })); + } + + tasks.push(tokio::spawn(positions(ducks.clone(), air.clone()))); + if args.rotate > 0 { + tasks.push(tokio::spawn(rotate( + ducks.clone(), + air.clone(), + args.rotate, + ))); + } + + for task in tasks { + let _ = task.await; + } + std::process::ExitCode::SUCCESS +} + +/// A believable BLE address, and a different one after every rotation. +fn address_for(index: usize, generation: u64) -> String { + let n = (index as u64 + 1) * 0x1_0000 + generation; + format!( + "E{:1X}:{:02X}:{:02X}:{:02X}:{:02X}:{:02X}", + index & 0xF, + (n >> 32) & 0xFF, + (n >> 24) & 0xFF, + (n >> 16) & 0xFF, + (n >> 8) & 0xFF, + n & 0xFF + ) +} + +/// One duck's connection: `btd`'s half of the conversation, without a radio underneath it. +async fn serve(duck: &Duck, air: &Air, range: f64, weather: Weather) -> std::io::Result<()> { + let stream = UnixStream::connect(&duck.socket).await?; + let (read_half, mut write_half) = stream.into_split(); + let mut lines = BufReader::new(read_half).lines(); + + let request = proto::Request::call(proto::Id::Number(1), &proto::Call::ChoraleSubscribe); + let mut line = serde_json::to_string(&request).map_err(std::io::Error::other)?; + line.push('\n'); + write_half.write_all(line.as_bytes()).await?; + tracing::info!(duck = %duck.name, "on the air"); + + let mut deliveries = tokio::time::interval(DELIVERY); + let mut delivered = usize::MAX; + let mut tick = 0u64; + loop { + tokio::select! { + line = lines.next_line() => { + let Some(line) = line? else { return Ok(()) }; + let Ok(request) = serde_json::from_str::(&line) else { continue }; + let Ok(proto::Call::ChoraleBeaconSet(want)) = request.as_call() else { continue }; + let mut on_air = air.lock().await; + if let Some(entry) = on_air.get_mut(&duck.name) { + let changed = entry.listening != want.listening + || entry.beacon.is_some() != want.beacon.is_some(); + if entry.beacon.is_none() && want.beacon.is_some() { + entry.since = Some(Instant::now()); + } + if want.beacon.is_none() { + entry.since = None; + } + entry.beacon = want.beacon.clone(); + entry.listening = want.listening; + if changed { + tracing::info!( + duck = %duck.name, + advertising = entry.beacon.is_some(), + listening = entry.listening, + "on the air" + ); + } + } + } + _ = deliveries.tick() => { + tick += 1; + let heard = nearby(duck, air, range, weather, tick).await; + if heard.len() != delivered { + delivered = heard.len(); + tracing::info!(duck = %duck.name, hears = delivered, "who is in range"); + } + for heard in heard { + let notify = proto::Request::notify(&proto::Call::ChoraleHeard(heard)); + let mut line = serde_json::to_string(¬ify).map_err(std::io::Error::other)?; + line.push('\n'); + if write_half.write_all(line.as_bytes()).await.is_err() { + return Ok(()); + } + } + } + } + } +} + +/// Every beacon this duck is close enough to hear. +async fn nearby( + duck: &Duck, + air: &Air, + range: f64, + weather: Weather, + tick: u64, +) -> Vec { + let on_air = air.lock().await; + let Some(me) = on_air.get(&duck.name) else { + return Vec::new(); + }; + if !me.listening { + return Vec::new(); + } + let here = me.at; + + let mut heard = Vec::new(); + for (name, other) in on_air.iter() { + if name == &duck.name { + continue; + } + let Some(beacon) = &other.beacon else { + continue; + }; + let distance = ((other.at[0] - here[0]).powi(2) + (other.at[1] - here[1]).powi(2)).sqrt(); + if distance > range { + continue; + } + + // Not noticed yet. Per pair, and timed from when the other duck went on the air: it is the + // asymmetry that splits a flock, so one delay shared by everybody would not produce it. + if !weather.discovery.is_zero() { + let pair = format!("{}<-{}", duck.name, name); + let wait = Duration::from_millis( + noise(weather.seed, &pair, 0) % (weather.discovery.as_millis() as u64).max(1), + ); + match other.since { + Some(since) if since.elapsed() >= wait => {} + _ => continue, + } + } + + // And an advertisement is missed often. Seeded by the pair and the tick, so the same run + // drops the same frames. + if weather.loss > 0.0 { + let pair = format!("{}<-{}", duck.name, name); + let roll = (noise(weather.seed, &pair, tick) % 10_000) as f64 / 10_000.0; + if roll < weather.loss { + continue; + } + } + heard.push(proto::ChoraleHeard { + beacon: beacon.clone(), + from: other.address.clone(), + // An age rather than a timestamp — see the module comment. Zero would claim the + // advertisement arrived at the instant it is being handed over, which is the one + // reading a real scanner never produces. + age_us: 1_000, + }); + } + heard +} + +/// Where every duck is standing, asked of its simulator. +async fn positions(ducks: Vec, air: Air) { + let mut ticker = tokio::time::interval(Duration::from_millis(250)); + loop { + ticker.tick().await; + for duck in &ducks { + if let Some(at) = ask_where(duck.body).await + && let Some(entry) = air.lock().await.get_mut(&duck.name) + { + entry.at = at; + } + } + } +} + +async fn ask_where(port: u16) -> Option<[f64; 3]> { + let stream = TcpStream::connect(("127.0.0.1", port)).await.ok()?; + let (read_half, mut write_half) = stream.into_split(); + let mut lines = BufReader::new(read_half).lines(); + write_half + .write_all(b"{\"op\":\"hello\",\"protocol\":1,\"joints\":15}\n") + .await + .ok()?; + lines.next_line().await.ok()??; + write_half.write_all(b"{\"op\":\"read\"}\n").await.ok()?; + let answer = lines.next_line().await.ok()??; + let value: serde_json::Value = serde_json::from_str(&answer).ok()?; + let trunk = value.get("trunk")?.as_array()?; + Some([ + trunk.first()?.as_f64()?, + trunk.get(1)?.as_f64()?, + trunk.get(2)?.as_f64()?, + ]) +} + +/// Give every duck a new address, as a real radio does, so that nothing may key on the old one. +async fn rotate(ducks: Vec, air: Air, seconds: u64) { + let mut ticker = tokio::time::interval(Duration::from_secs(seconds)); + let started = Instant::now(); + ticker.tick().await; + loop { + ticker.tick().await; + let generation = started.elapsed().as_secs() / seconds.max(1); + let mut on_air = air.lock().await; + for (index, duck) in ducks.iter().enumerate() { + if let Some(entry) = on_air.get_mut(&duck.name) { + entry.address = address_for(index, generation); + } + } + tracing::info!(generation, "every duck has a new address"); + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn a_duck_is_a_name_a_socket_and_a_body() { + let duck = parse_duck("duck-a=/run/duck-a/robotd.sock@7801").expect("parses"); + assert_eq!(duck.name, "duck-a"); + assert_eq!(duck.socket, PathBuf::from("/run/duck-a/robotd.sock")); + assert_eq!(duck.body, 7801); + } + + #[test] + fn a_socket_path_may_contain_an_at_sign() { + // Split from the right: the port is the last `@`, and a path is allowed to be strange. + let duck = parse_duck("d=/tmp/o@d/robotd.sock@7999").expect("parses"); + assert_eq!(duck.socket, PathBuf::from("/tmp/o@d/robotd.sock")); + assert_eq!(duck.body, 7999); + } + + #[test] + fn what_is_not_a_duck_says_so() { + assert!(parse_duck("duck-a").is_err()); + assert!(parse_duck("duck-a=/path/without/a/port").is_err()); + assert!(parse_duck("duck-a=/path@not-a-number").is_err()); + } + + #[test] + fn every_duck_has_its_own_address_and_a_new_one_each_generation() { + let first: Vec = (0..4).map(|i| address_for(i, 0)).collect(); + let later: Vec = (0..4).map(|i| address_for(i, 7)).collect(); + for window in first.windows(2) { + assert_ne!(window[0], window[1], "two ducks shared an address"); + } + for (a, b) in first.iter().zip(&later) { + assert_ne!(a, b, "an address survived a rotation: {a}"); + } + assert_eq!(first[0].len(), "E0:00:01:00:00:00".len()); + } +} diff --git a/duck-ipc-proto/Cargo.toml b/duck-ipc-proto/Cargo.toml index 141cc4d5..8ebe0c73 100644 --- a/duck-ipc-proto/Cargo.toml +++ b/duck-ipc-proto/Cargo.toml @@ -13,6 +13,8 @@ description = "IPC contracts between the robot's services and their clients" # it belongs in the crate that owns the behaviour, not here. [dependencies] serde.workspace = true +# `clock`: CLOCK_MONOTONIC / CLOCK_REALTIME, which std does not expose as raw nanoseconds. +libc = "0.2" serde_json.workspace = true semver.workspace = true diff --git a/duck-ipc-proto/src/lib.rs b/duck-ipc-proto/src/lib.rs index 69877586..dbf68a94 100644 --- a/duck-ipc-proto/src/lib.rs +++ b/duck-ipc-proto/src/lib.rs @@ -273,7 +273,71 @@ pub const JSONRPC_VERSION: &str = "2.0"; /// older `robotctl` against this `configd` prints no `units` block at all rather than a wrong one. /// Both come out of the same release and an apply restarts both, so the skew lasts as long as the /// update does; a board left mid-update sees a missing block, not a lie. -pub const API_VERSION: u32 = 23; +/// +/// # v24 — a clock every stream shares, and what a map needs from the robot +/// +/// Three additive fields and one read, for a mapper that pairs a camera frame with the joint +/// angles and depth cells of the same instant (`docs/design/robotd-design.md` §mapping): +/// +/// - [`RobotState::t_ns`] and [`TofFrame::t_ns`] are the same `CLOCK_MONOTONIC`, nanoseconds, +/// so a sample from `robotd` and a frame from `tofd` can be put on one axis without guessing +/// the offset between two daemons' start times. `t` and `at_us` stay as they were. +/// - [`RobotState::imu`] is the trunk IMU as the loop sees it: gyro and orientation, not only +/// the projected gravity [`SafetyState`] already carried. [`RobotState::frames`] is the +/// camera and ToF sensor pose in the trunk frame, from the same head FK `robot.look` uses — +/// published so no client has to carry a copy of the kinematics. +/// - `robot.model` ([`ModelResult`]) answers the static geometry those poses are stated in. +/// +/// `mediad`'s `media.video` answer gains `mono_ns` and `real_ns`, the two clocks read at the +/// same instant, so a peer can put RTP timestamps (whose RTCP sender reports are wall-clock) +/// onto the same monotonic axis. Additive everywhere: an older client ignores fields it does +/// not know, and an older daemon leaves them at their defaults (zero, or absent). +/// +/// # v25 — the whole skeleton's pose, for a viewer +/// +/// [`RobotState::skeleton`] carries every body's pose in the trunk frame this tick, and +/// [`ModelResult::skeleton`] the matching static tree (each link's name and parent). Together they +/// let a viewer draw the robot moving for real — the full kinematics, of which +/// [`RobotState::frames`] (camera, ToF, head IMU) is a few leaves — without carrying a copy of the +/// kinematics, the same reason `frames` and `tof_beams` come from the robot. Both from the same FK +/// `robot.look` uses. Additive: the `Vec`s are empty from a daemon predating it. +/// +/// # v26 — `system.logs` +/// +/// The tail of one daemon's journal, over the wire. Until now the answer to "what did it say +/// before it stopped" was `journalctl` over ssh, which needs a network the robot may not have and +/// an address a phone has no way to reach — so the one question support asks first was the one +/// question the wire could not answer. +/// +/// Additive as a method, and narrow on purpose: a unit from a fixed list, a line count, and which +/// boot. Not a `journalctl` command line — see [`LogsParams`] for why that boundary is where it is. +/// +/// An older `configd` answers [`code::METHOD_NOT_FOUND`] naming the method, which is the designed +/// skew and not a handshake refusal: a new `duckctl` against a robot on an older release reports +/// that the robot is too old rather than failing obscurely. +/// +/// # v28 — `detector.*` +/// +/// The duck detector leaves the release the way the policies did at v19: `mediad` reads it from +/// `/opt/robot/detector/current`, the release's postinstall hook seeds that from a pinned Hub +/// revision, and `detector.check` / `detector.install` are how a board asks what exists and +/// moves to it — `policy.check` / `policy.install` with a different root, answered by the same +/// daemon for the same reason (it has the network stack). An install restarts `mediad`, which is +/// where the model is loaded, and says whether that took. +/// +/// Additive as methods; the parameters and answers are the policy set's own types. +/// +/// # v27 — the pad's IMU, on the pad tap +/// +/// Three more [`PadReport`] variants: a pad's inertial unit as a second evdev node beside the one +/// that drives, its samples, and its going away. The "Pro Controller" Switch clones ship a +/// six-axis IMU and the kernel's `hid-nintendo` exposes it as a separate accelerometer device +/// under the same HID parent; an Xbox pad has none and a subscriber never sees the variants. +/// +/// A new variant on a tagged enum is what a robotctl built before it cannot decode, which is the +/// one reason this is a bump rather than a note: the tap is still `padd`'s own socket, and every +/// other client is untouched. +pub const API_VERSION: u32 = 28; /// The observation width every policy this robot family runs is built against. /// @@ -331,6 +395,10 @@ pub mod socket { /// Under `/run/tofd/` for the same reason as the pad's: it is that unit's /// `RuntimeDirectory=`, so systemd removes the socket when the daemon stops. pub const TOF: &str = "/run/tofd/tof.sock"; + + /// `mediad`'s on-demand raw-frame endpoint. It is local-only: a raw camera frame is for a + /// recorder or perception process on the robot, not a multi-megabyte WebRTC control reply. + pub const MEDIA: &str = "/run/mediad/media.sock"; } /// Where each daemon publishes what it is running: `/run//identity.json`. @@ -388,6 +456,12 @@ pub const JOINT_NAMES: [&str; 15] = [ pub mod method { pub const HELLO: &str = "hello"; + /// One raw camera frame. `mediad` answers the JSON-RPC header, followed immediately by the + /// bytes named in that header, on its local Unix socket. + /// Deliberately not a `Call`: its binary tail must never enter Service/Lane routing + /// or the WebRTC control datachannel. Local clients dial `socket::MEDIA` explicitly. + pub const MEDIA_FRAME: &str = "media.frame"; + pub const CHECK: &str = "update.check"; pub const APPLY: &str = "update.apply"; pub const ROLLBACK: &str = "update.rollback"; @@ -469,6 +543,12 @@ pub mod method { /// yield a fallen robot is commanded at, which keeps torque on. This is the register. pub const ROBOT_RELAX: &str = "robot.relax"; + /// Reboot servos: the REBOOT instruction, which clears a latched hardware error (overload, + /// overheating, electrical shock) that otherwise holds torque off until the battery is pulled. + /// Torque is cut on every joint first and the robot is back at limp afterwards, so `robot.init` + /// or `robot.enable` brings it up from a known state. Discrete; send as a request. + pub const ROBOT_REBOOT_MOTORS: &str = "robot.rebootMotors"; + // ── skills ─────────────────────────────────────────────────────────────── // // One-shot scripted moves, ported from `microduck_runtime`. Each swaps a dedicated @@ -547,6 +627,10 @@ pub mod method { /// is in them — and, when a load failed, what is in them instead of what was asked for. pub const ROBOT_POLICIES: &str = "robot.policies"; + /// The static geometry [`RobotState::frames`] and [`TofFrame`] are stated in — see + /// [`ModelResult`]. A read; asked once per session. + pub const ROBOT_MODEL: &str = "robot.model"; + /// Put a different `.onnx` in one slot, or drop an override and go back to the default. /// /// Answered like [`ROBOT_SET_MODE`] and for the same reason: the swap happens at the home @@ -579,6 +663,16 @@ pub mod method { /// Search the Hub for policies. pub const POLICY_SEARCH: &str = "policy.search"; + // ── detector.* ─────────────────────────────────────────────────────────── + // + // The duck detector's set, served by `updaterd` for `policy.*`'s reason. The answers are + // `policy.*`'s types: a set is a set, whatever is in it. + + /// Is there a newer duck detector than the one installed? + pub const DETECTOR_CHECK: &str = "detector.check"; + /// Install a duck detector from the Hub, and restart `mediad` onto it. + pub const DETECTOR_INSTALL: &str = "detector.install"; + // ── account.* ──────────────────────────────────────────────────────────── // // Which Hugging Face account this robot belongs to. Served by `updaterd` for `policy.*`'s @@ -630,6 +724,8 @@ pub mod method { pub const SYSTEM_INFO: &str = "system.info"; /// What systemd says about each daemon, and which release each is running from. pub const SYSTEM_SERVICES: &str = "system.services"; + /// The tail of one unit's journal, for a client with no shell on the robot. + pub const SYSTEM_LOGS: &str = "system.logs"; /// Rename the robot. This is the name a phone sees. pub const SYSTEM_SET_NAME: &str = "system.setName"; /// Reboot, cleanly, through systemd. @@ -710,6 +806,13 @@ pub mod method { /// One 8×8 depth frame, pushed after [`TOF_STREAM`]. pub const TOF_FRAME: &str = "tof.frame"; + + /// Subscribe to the head IMU (BMI088 on the HAT, same I²C bus as the ToF). The answer + /// describes the sensor, then [`HEAD_IMU_FRAME`] notifications arrive until the connection closes. + pub const HEAD_IMU_STREAM: &str = "head_imu.stream"; + + /// One head-IMU sample, pushed after [`HEAD_IMU_STREAM`]. + pub const HEAD_IMU_FRAME: &str = "head_imu.frame"; } /// JSON-RPC error codes. @@ -809,6 +912,8 @@ pub enum Call { RobotInit, /// Cut power to the joints. The robot collapses if nothing holds it. RobotRelax, + /// Reboot servos (all of them, or the ids named), then limp. See [`method::ROBOT_REBOOT_MOTORS`]. + RobotRebootMotors(RebootMotorsParams), /// Run a one-shot skill, or toggle sit↔stand. RobotDo(DoParams), /// Standing body pose. Continuous. Send as a notification. @@ -837,6 +942,8 @@ pub enum Call { RobotSetMode(SetModeParams), /// What each policy slot runs; see [`method::ROBOT_POLICIES`]. RobotPolicies, + /// Static robot geometry for a mapper; see [`method::ROBOT_MODEL`]. + RobotModel, /// Load one slot, or reset it; see [`method::ROBOT_LOAD_POLICY`]. RobotLoadPolicy(LoadPolicyParams), /// Re-read every slot from disk; see [`method::ROBOT_RELOAD_POLICIES`]. @@ -852,6 +959,12 @@ pub enum Call { /// Search the Hub; see [`method::POLICY_SEARCH`]. PolicySearch(PolicySearchParams), + // ── detector.* ─────────────────────────────────────────────────────────── + /// What detector is installed and what the Hub offers; see [`method::DETECTOR_CHECK`]. + DetectorCheck, + /// Install a detector and restart `mediad` onto it; see [`method::DETECTOR_INSTALL`]. + DetectorInstall(PolicyInstallParams), + // ── account.* ──────────────────────────────────────────────────────────── /// Start a device-code login; see [`method::ACCOUNT_LOGIN`]. AccountLogin(AccountLoginParams), @@ -869,6 +982,8 @@ pub enum Call { // ── system.* ───────────────────────────────────────────────────────────── SystemInfo, SystemServices, + /// The tail of one unit's journal; see [`method::SYSTEM_LOGS`]. + SystemLogs(LogsParams), SystemSetName(SetNameParams), SystemReboot, /// Read the pairing PIN. @@ -900,6 +1015,8 @@ pub enum Call { PadInput, /// Subscribe to the ToF depth stream. Answered by `tofd`. TofStream, + /// Subscribe to the head IMU (BMI088 on the HAT); see [`method::HEAD_IMU_STREAM`]. + HeadImuStream, } /// The service that owns the answer to a call. @@ -978,6 +1095,7 @@ impl Call { Call::RobotEnable(_) => method::ROBOT_ENABLE, Call::RobotInit => method::ROBOT_INIT, Call::RobotRelax => method::ROBOT_RELAX, + Call::RobotRebootMotors(_) => method::ROBOT_REBOOT_MOTORS, Call::RobotDo(_) => method::ROBOT_DO, Call::RobotPose(_) => method::ROBOT_POSE, Call::RobotMouth(_) => method::ROBOT_MOUTH, @@ -991,12 +1109,15 @@ impl Call { Call::RobotMode => method::ROBOT_MODE, Call::RobotSetMode(_) => method::ROBOT_SET_MODE, Call::RobotPolicies => method::ROBOT_POLICIES, + Call::RobotModel => method::ROBOT_MODEL, Call::RobotLoadPolicy(_) => method::ROBOT_LOAD_POLICY, Call::RobotReloadPolicies => method::ROBOT_RELOAD_POLICIES, Call::PolicyCheck => method::POLICY_CHECK, Call::PolicyInstall(_) => method::POLICY_INSTALL, Call::PolicyFetch(_) => method::POLICY_FETCH, Call::PolicySearch(_) => method::POLICY_SEARCH, + Call::DetectorCheck => method::DETECTOR_CHECK, + Call::DetectorInstall(_) => method::DETECTOR_INSTALL, Call::AccountLogin(_) => method::ACCOUNT_LOGIN, Call::AccountStatus => method::ACCOUNT_STATUS, Call::AccountLogout => method::ACCOUNT_LOGOUT, @@ -1007,6 +1128,7 @@ impl Call { Call::NetForget(_) => method::NET_FORGET, Call::SystemInfo => method::SYSTEM_INFO, Call::SystemServices => method::SYSTEM_SERVICES, + Call::SystemLogs(_) => method::SYSTEM_LOGS, Call::SystemSetName(_) => method::SYSTEM_SET_NAME, Call::SystemReboot => method::SYSTEM_REBOOT, Call::SystemPairingPin => method::SYSTEM_PAIRING_PIN, @@ -1022,6 +1144,7 @@ impl Call { Call::RobotRemoveSkill(_) => method::ROBOT_REMOVE_SKILL, Call::PadInput => method::PAD_INPUT, Call::TofStream => method::TOF_STREAM, + Call::HeadImuStream => method::HEAD_IMU_STREAM, } } @@ -1059,6 +1182,9 @@ impl Call { // replacing the official set. `policy.search` and `policy.fetch`'s read-only // cousins stay ungated — asking what exists changes nothing. | Call::PolicyFetch(_) + // Replacing the detector writes to the eMMC and restarts `mediad`, which drops + // every video session. `detector.check` is a read and stays ungated. + | Call::DetectorInstall(_) // Signing the robot in binds it to a Hugging Face account, and signing it out // takes it away again. That is the most consequential pair here by one measure // nothing else in this list shares: it decides who can reach the robot *from @@ -1116,6 +1242,9 @@ impl Call { // `robotd` to reload, which is the same order of magnitude as a small update — long, // but bounded and not a stream. Call::PolicyCheck | Call::PolicyInstall(_) => (Updater, Prompt), + // The same two, for the detector: one round trip, or a fourteen-megabyte download + // and a `mediad` restart. + Call::DetectorCheck | Call::DetectorInstall(_) => (Updater, Prompt), // `fetch` downloads one file and `search` is a single query; both are bounded and // neither streams. Call::PolicyFetch(_) | Call::PolicySearch(_) => (Updater, Prompt), @@ -1138,6 +1267,7 @@ impl Call { | Call::RobotModelApi | Call::RobotRemoteSessionActive | Call::RobotPolicies + | Call::RobotModel | Call::RobotMode => (Robot, Prompt), // Intents and one-shot skills. All fast: they store a value the control loop reads on // its next tick, and none of them waits for the robot to finish anything. @@ -1148,6 +1278,7 @@ impl Call { | Call::RobotEnable(_) | Call::RobotInit | Call::RobotRelax + | Call::RobotRebootMotors(_) | Call::RobotDo(_) | Call::RobotPose(_) | Call::RobotMouth(_) @@ -1168,6 +1299,7 @@ impl Call { | Call::NetForget(_) | Call::SystemInfo | Call::SystemServices + | Call::SystemLogs(_) | Call::SystemSetName(_) | Call::SystemReboot | Call::SystemPairingPin @@ -1204,6 +1336,7 @@ impl Call { // exists today reaches neither: what a call *is* does not depend on who may ask it. Call::PadInput => (Pad, Stream), Call::TofStream => (Tof, Stream), + Call::HeadImuStream => (Tof, Stream), // ── answered by no service ────────────────────────────────────── // @@ -1255,11 +1388,13 @@ impl Call { Call::RobotLook(p) => encode(p), Call::RobotEnable(p) => encode(p), Call::RobotDo(p) => encode(p), + Call::RobotRebootMotors(p) => encode(p), Call::RobotPose(p) => encode(p), Call::RobotMouth(p) => encode(p), Call::RobotSetMode(p) => encode(p), Call::RobotLoadPolicy(p) => encode(p), Call::PolicyInstall(p) => encode(p), + Call::DetectorInstall(p) => encode(p), Call::PolicyFetch(p) => encode(p), Call::PolicySearch(p) => encode(p), Call::AccountLogin(p) => encode(p), @@ -1271,6 +1406,7 @@ impl Call { Call::RobotSubscribe(p) => encode(p), Call::NetConnect(p) => encode(p), Call::NetForget(p) => encode(p), + Call::SystemLogs(p) => encode(p), Call::SystemSetName(p) => encode(p), Call::SystemSetPairingPin(p) => encode(p), Call::SystemAuthenticate(p) => encode(p), @@ -1290,8 +1426,10 @@ impl Call { | Call::RobotRelax | Call::RobotShutdown | Call::RobotPolicies + | Call::RobotModel | Call::RobotReloadPolicies | Call::PolicyCheck + | Call::DetectorCheck | Call::AccountStatus | Call::AccountLogout | Call::RobotMode => Value::Object(serde_json::Map::new()), @@ -1306,6 +1444,7 @@ impl Call { | Call::RobotSkills | Call::PadInput | Call::TofStream + | Call::HeadImuStream | Call::ChoraleSubscribe => Value::Object(serde_json::Map::new()), } } @@ -1345,6 +1484,7 @@ impl Call { method::ROBOT_ENABLE => Call::RobotEnable(decode(params)?), method::ROBOT_INIT => Call::RobotInit, method::ROBOT_RELAX => Call::RobotRelax, + method::ROBOT_REBOOT_MOTORS => Call::RobotRebootMotors(decode(params)?), method::ROBOT_DO => Call::RobotDo(decode(params)?), method::ROBOT_POSE => Call::RobotPose(decode(params)?), method::ROBOT_MOUTH => Call::RobotMouth(decode(params)?), @@ -1358,12 +1498,15 @@ impl Call { method::ROBOT_MODE => Call::RobotMode, method::ROBOT_SET_MODE => Call::RobotSetMode(decode(params)?), method::ROBOT_POLICIES => Call::RobotPolicies, + method::ROBOT_MODEL => Call::RobotModel, method::ROBOT_LOAD_POLICY => Call::RobotLoadPolicy(decode(params)?), method::ROBOT_RELOAD_POLICIES => Call::RobotReloadPolicies, method::POLICY_CHECK => Call::PolicyCheck, method::POLICY_INSTALL => Call::PolicyInstall(decode(params)?), method::POLICY_FETCH => Call::PolicyFetch(decode(params)?), method::POLICY_SEARCH => Call::PolicySearch(decode(params)?), + method::DETECTOR_CHECK => Call::DetectorCheck, + method::DETECTOR_INSTALL => Call::DetectorInstall(decode(params)?), method::ACCOUNT_LOGIN => Call::AccountLogin(decode(params)?), method::ACCOUNT_STATUS => Call::AccountStatus, method::ACCOUNT_LOGOUT => Call::AccountLogout, @@ -1374,6 +1517,7 @@ impl Call { method::NET_FORGET => Call::NetForget(decode(params)?), method::SYSTEM_INFO => Call::SystemInfo, method::SYSTEM_SERVICES => Call::SystemServices, + method::SYSTEM_LOGS => Call::SystemLogs(decode(params)?), method::SYSTEM_SET_NAME => Call::SystemSetName(decode(params)?), method::SYSTEM_REBOOT => Call::SystemReboot, method::SYSTEM_PAIRING_PIN => Call::SystemPairingPin, @@ -1397,6 +1541,7 @@ impl Call { method::ROBOT_REMOVE_SKILL => Call::RobotRemoveSkill(decode(params)?), method::PAD_INPUT => Call::PadInput, method::TOF_STREAM => Call::TofStream, + method::HEAD_IMU_STREAM => Call::HeadImuStream, other => { return Err(Error::new( code::METHOD_NOT_FOUND, @@ -1492,6 +1637,7 @@ pub mod test_support { }), Call::RobotInit, Call::RobotRelax, + Call::RobotRebootMotors(RebootMotorsParams { ids: vec![3, 11] }), Call::RobotDo(DoParams { skill: "ground_pick".into(), }), @@ -1509,6 +1655,7 @@ pub mod test_support { Call::RobotShutdown, Call::RobotMode, Call::RobotPolicies, + Call::RobotModel, Call::RobotReloadPolicies, Call::PolicyCheck, Call::PolicyInstall(PolicyInstallParams { @@ -1522,6 +1669,10 @@ pub mod test_support { Call::PolicySearch(PolicySearchParams { query: "microduck".into(), }), + Call::DetectorCheck, + Call::DetectorInstall(PolicyInstallParams { + version: Some("v2".into()), + }), Call::AccountLogin(AccountLoginParams { force: false }), Call::AccountStatus, Call::AccountLogout, @@ -1541,6 +1692,11 @@ pub mod test_support { }), Call::SystemInfo, Call::SystemServices, + Call::SystemLogs(LogsParams { + unit: "robotd".into(), + lines: 40, + boot: -1, + }), Call::SystemSetName(SetNameParams { name: "duck-01".into(), }), @@ -1572,6 +1728,7 @@ pub mod test_support { }), Call::PadInput, Call::TofStream, + Call::HeadImuStream, ] } } @@ -1674,6 +1831,24 @@ impl Request { serde_json::from_value(self.params.clone()?).ok() } + /// A head-IMU sample notification: no `id`. + pub fn notify_head_imu_frame(frame: &HeadImuFrame) -> Self { + Self { + jsonrpc: JSONRPC_VERSION.to_owned(), + id: None, + method: method::HEAD_IMU_FRAME.to_owned(), + params: Some(serde_json::to_value(frame).unwrap_or(Value::Null)), + } + } + + /// Read a head-IMU notification back. + pub fn as_head_imu_frame(&self) -> Option { + if self.method != method::HEAD_IMU_FRAME { + return None; + } + serde_json::from_value(self.params.clone()?).ok() + } + /// A progress notification: no `id`, so no response is expected. pub fn notify_progress(progress: &Progress) -> Self { Self { @@ -2030,6 +2205,13 @@ pub struct ThereminState { /// the list it does know, which is the same shape as a bad policy slot. pub type Skill = String; +/// Which servos [`method::ROBOT_REBOOT_MOTORS`] reboots. Empty means every servo. +#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(default, deny_unknown_fields)] +pub struct RebootMotorsParams { + pub ids: Vec, +} + #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(deny_unknown_fields)] pub struct DoParams { @@ -2177,7 +2359,7 @@ pub struct PolicySlot { pub error: Option, } -/// Which set to install, for [`Call::PolicyInstall`]. +/// Which set to install, for [`Call::PolicyInstall`] and [`Call::DetectorInstall`]. #[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] #[serde(default, deny_unknown_fields)] pub struct PolicyInstallParams { @@ -2715,6 +2897,40 @@ pub struct HelloResult { pub revision: Option, } +/// Metadata preceding the binary tail of a local `media.frame` response. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct MediaFrameHeader { + pub width: u32, + pub height: u32, + pub format: String, + pub bytes: usize, + pub captured_at_unix_us: u128, + /// Degrees clockwise the camera is mounted from upright — the same number `media.video` + /// tells a WebRTC peer, and zero when `--flip-in-pipeline` already turned these pixels. + /// + /// **Carried rather than written down.** The geometry above describes the bytes exactly as + /// they are, and a consumer cannot recover the mount from them: a 180° mount is + /// indistinguishable from an upright one, and a quarter turn is only a guess from the aspect + /// ratio. A recorder building a dataset needs the angle programmatically, and a human + /// converting a frame should not have to find a document to learn their picture is sideways. + pub rotate: u32, +} +impl MediaFrameHeader { + /// Bound allocation and reject malformed geometry before decoding pixels. + pub fn valid_uyvy(&self) -> bool { + self.width > 0 + && self.height > 0 + && self.width.is_multiple_of(2) + && self.format == "UYVY" + && matches!(self.rotate, 0 | 90 | 180 | 270) + && self.bytes <= 16 * 1024 * 1024 + && (self.width as usize) + .checked_mul(self.height as usize) + .and_then(|n| n.checked_mul(2)) + == Some(self.bytes) + } +} + /// Where an in-flight update has got to. Mirrors the state machine in /// `docs/design/updater-design.md` §7. #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] @@ -3252,6 +3468,103 @@ pub struct RobotState { /// state of a duck — see [`ChoraleState`]. #[serde(default, skip_serializing_if = "Option::is_none")] pub chorale: Option, + /// `CLOCK_MONOTONIC` at this tick, nanoseconds — the same clock as [`TofFrame::t_ns`], so a + /// mapper can put a state sample and a depth frame on one axis. Zero from a daemon predating + /// it; `t` is still the loop's own elapsed time. (v24) + #[serde(default)] + pub t_ns: u64, + /// The trunk IMU as the loop read it this tick. Absent from a daemon predating it, or while + /// no sensors have been read. (v24) + #[serde(default, skip_serializing_if = "Option::is_none")] + pub imu: Option, + /// Where the camera and the ToF sensor are in the trunk frame at this tick's measured head + /// joints, from the same head FK `robot.look` uses. Published so a client that pairs a picture + /// with a pose never carries its own copy of the kinematics. (v24) + #[serde(default, skip_serializing_if = "Option::is_none")] + pub frames: Option, + /// Every body's pose in the trunk frame this tick, in [`ModelResult::skeleton`] order, from the + /// same head FK `robot.look` uses — the whole skeleton, so a viewer can draw the robot moving + /// for real. `frames` is a few leaves of this. Empty from a daemon predating it. (v25) + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub skeleton: Vec, +} + +/// The trunk IMU, in [`RobotState::imu`]. Trunk frame: x forward, y left, z up. +#[derive(Debug, Default, Clone, Copy, PartialEq, Serialize, Deserialize)] +#[serde(default)] +pub struct ImuState { + /// Angular velocity, rad/s. + pub gyro: [f64; 3], + /// Orientation trunk → world, scalar-first `[w, x, y, z]`. The world is the IMU's own + /// gravity-aligned frame with an arbitrary yaw at boot — the same frame `odom` lives in. + pub quat: [f64; 4], +} + +/// A pose in the trunk frame: position in metres, orientation scalar-first `[w, x, y, z]`. +#[derive(Debug, Default, Clone, Copy, PartialEq, Serialize, Deserialize)] +#[serde(default)] +pub struct PoseState { + pub pos: [f64; 3], + pub quat: [f64; 4], +} + +/// Sensor poses in the trunk frame, in [`RobotState::frames`]. +/// +/// Conventions, stated once: `camera` is the OpenCV camera frame (+x right, +y down, +z along +/// the optical axis) — the frame intrinsics and a pixel's ray are expressed in. `tof` is the +/// VL53L5CX/L8CX integration frame (+x along the optical axis, +y left, +z up), the frame +/// [`ModelResult::tof_beams`] is stated in. Both come from `kinematics::head::HeadFk`. +#[derive(Debug, Default, Clone, Copy, PartialEq, Serialize, Deserialize)] +#[serde(default)] +pub struct FramesState { + pub camera: PoseState, + pub tof: PoseState, + /// The head IMU (BMI088) in the trunk frame. The `head_imu.stream` samples are in the IMU's + /// own tilted axes; this pose (sensor→trunk, from the same head FK) is how a consumer rotates + /// them into the trunk/camera frame. Absent from a daemon predating it. (v24) + #[serde(default, skip_serializing_if = "Option::is_none")] + pub head_imu: Option, +} + +/// Answer to [`Call::RobotModel`]: the geometry that does not change while the robot runs. +/// +/// Everything a mapper needs beyond the per-tick poses in [`RobotState::frames`], so that the +/// kinematics stay in one place — the `kinematics` crate — and a laptop or a server asks rather +/// than transcribes. Angles and distances in radians and metres. +#[derive(Debug, Default, Clone, PartialEq, Serialize, Deserialize)] +#[serde(default)] +pub struct ModelResult { + /// Which MJCF asset the numbers come from, e.g. `alpha`. + pub asset: String, + /// Height of the trunk origin above the floor when standing at rest. + pub trunk_height_m: f64, + /// Every joint, in [`RobotState::joints`] order. + pub joint_names: Vec, + /// The four head joints in the order [`RobotState::head`] uses. + pub head_joints: Vec, + /// Unit direction of each ToF zone in the sensor frame, row-major like + /// [`TofFrame::distance_mm`]: row 0 is the top of the grid, column 0 the sensor's left. + pub tof_beams: Vec<[f64; 3]>, + /// The sensor's field of view per axis, degrees. + pub tof_fov_deg: f64, + /// Sensor poses at every head joint zero — a fixed reference; the live ones are in + /// [`RobotState::frames`]. + pub frames_at_zero: FramesState, + /// The body tree the per-tick [`RobotState::skeleton`] poses are stated in: each link's name + /// and parent, in the same order. Empty from a daemon predating it. (v25) + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub skeleton: Vec, +} + +/// One link in the body tree — a name and its parent — so a viewer can match a +/// [`RobotState::skeleton`] pose to a link and draw the edge to its parent. (v25) +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct SkeletonLink { + /// The MJCF body name, e.g. `trunk_base`, `upper_leg_left`. + pub name: String, + /// Index of this link's parent in [`ModelResult::skeleton`], or `None` for the root. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub parent: Option, } /// What the duck chorale is doing, in [`RobotState`]. @@ -3589,6 +3902,63 @@ pub struct ServiceUnit { pub identity: Option, } +/// Which unit's journal to read, and how much of it. Parameters of [`Call::SystemLogs`]. +/// +/// **A unit name, not a filter expression.** The service picks from a fixed list and refuses +/// anything else, because this call is reachable from a phone in radio range and `journalctl` +/// arguments are not a language to hand such a peer. What that costs is `-g`, `--since` and +/// several other things a person with a shell would reach for; what it buys is that the worst a +/// client can ask for is the tail of a daemon this project ships. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct LogsParams { + /// `robotd` or `robotd.service` — the service resolves either. See `configd::logs` for the + /// units it will read. + pub unit: String, + /// How many lines from the end. Clamped to [`MAX_LOG_LINES`], and the byte budget below + /// usually binds first. + pub lines: usize, + /// Which boot: `0` is the current one, `-1` the one before it, and so on backwards. + /// + /// Negative rather than an index because that is the question — "what did it say before it + /// restarted" — and because it is `journalctl -b`'s own convention, so the answer to "which + /// boot did I just read" is the same number in both places. + pub boot: i32, +} + +/// Ceiling on [`LogsParams::lines`], applied by the service rather than trusted from the caller. +pub const MAX_LOG_LINES: usize = 500; + +/// Ceiling on the serialised `lines` array, in bytes. +/// +/// **This exists because of the transport, and it is the binding limit in practice.** BLE +/// reassembles a reply into one line, and the client's buffer for that is bounded — a peer that +/// never sends a newline must not be able to grow it without limit. 48 KiB leaves the rest of a +/// JSON-RPC envelope room inside a 64 KiB client buffer, and at a typical ATT MTU it is already +/// several seconds on the air, which is the other reason not to raise it. +/// +/// The service drops the *oldest* lines to fit and says it did, because the newest are the ones +/// somebody asked for. +pub const MAX_LOG_BYTES: usize = 48 * 1024; + +/// Answer to [`Call::SystemLogs`]. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct LogsResult { + /// The unit as systemd names it, so a caller that typed `robotd` sees what it actually read. + pub unit: String, + /// Oldest first, the way a journal reads. No trailing newlines. + /// + /// **A line in `-- … --` is the robot's, not the journal's.** `journalctl` uses that shape + /// for what it inserts rather than recorded (`-- Reboot --`, `-- No entries --`) and this + /// borrows it for the one thing a tail cannot otherwise show: `-- new robotd process, pid + /// 3227 --`, where the daemon restarted. A tail spanning an update carries two different + /// builds' output, and nothing in the lines themselves says where one ends. + pub lines: Vec, + /// Lines were dropped from the front to fit [`MAX_LOG_BYTES`]. Not an error — it is what + /// asking for more than the radio can carry looks like, and a caller can say so. + pub truncated: bool, +} + /// Answer to [`Call::PadStatus`]. #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] pub struct PadStatusResult { @@ -3862,6 +4232,81 @@ pub enum PadReport { /// usually an errno the operator wants verbatim. why: String, }, + /// The pad has an inertial unit and its node is open. Sent on subscribing if one is already + /// being read, and again each time one appears — the same one code path as `Attached`. + /// + /// Independent of `Attached`: the IMU is a second evdev device with a life of its own, and a + /// pad without one simply never sends this. Everything in a [`PadImuSample`] is read against + /// the device here. + ImuAttached { device: Box }, + /// Inertial samples, as many as the kernel handed over in one read — see [`PadImuBatch`]. + Imu(PadImuBatch), + /// The IMU node closed. + ImuDetached { why: String }, +} + +/// A pad's inertial unit, as the kernel describes it. +/// +/// One device rather than six axes in [`PadInputDevice::axes`], because the kernel keeps them +/// apart: an accelerometer node carries `INPUT_PROP_ACCELEROMETER` and its `ABS_X..Z` are metres +/// per second squared, not a stick. Reading them as a stick is what gilrs would do, which is why +/// `padd` drives from the other node and this one is only ever tapped. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct PadImuDevice { + /// As the kernel names it: "Nintendo Switch Pro Controller IMU". + pub name: String, + /// The event node being read. + pub node: String, + /// Raw units per **g** on the accelerometer axes, from the driver's `resolution`. 4096 on + /// `hid-nintendo`. Zero when the driver did not say, in which case the raw numbers are all a + /// reader has. + pub accel_per_g: i32, + /// Raw units per **degree per second** on the gyro axes. 14247 on `hid-nintendo`. + pub gyro_per_dps: i32, + /// The accelerometer's full scale, raw units, so a reader can tell a clipped sample. + pub accel_max: i32, + /// The gyro's full scale, raw units. + pub gyro_max: i32, +} + +/// The inertial samples one read of the IMU node produced. +/// +/// A batch rather than one sample per report, because of what a sample costs to send: at six +/// hundred a second, one JSON line and one socket write each was measured at 6.6% of a core on the +/// board (2026-09-09, `padd` with a subscriber, above its 1.6% idle). The kernel already groups +/// them — the clone packs three samples into every HID packet — so the tap sends what one `read` +/// returned, and the viewer takes them in order: measured at exactly three per batch, two hundred +/// batches a second, and 4.4% of a core. Nothing is summarised: every sample is here. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct PadImuBatch { + /// In the order the kernel delivered them, oldest first. Never empty on the wire. + pub samples: Vec, + /// Batches this subscriber missed because its own socket was behind, since the last one it did + /// receive. Counted apart from [`PadFrame::socket_dropped`]: a dropped IMU batch says nothing + /// about the stick reports. + #[serde(default, skip_serializing_if = "is_zero")] + pub socket_dropped: u64, +} + +/// One inertial sample: everything the IMU node delivered between two `SYN_REPORT`s. +/// +/// Raw kernel units, deliberately — the tap hands out what the device said and the resolution to +/// read it with, and the conversion happens once, in the viewer. Six integers rather than a +/// `PadFrame`'s event list because this arrives at several hundred a second and every byte is +/// paid for on the board's CPU. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct PadImuSample { + /// Samples since this IMU attached, counted by `padd`. A hole is a batch this subscriber + /// missed — [`PadImuBatch::socket_dropped`]. + pub seq: u64, + /// The kernel's timestamp, microseconds since the epoch — the same clock and the same + /// caveats as [`PadFrame::at_us`]. + pub at_us: u64, + /// `ABS_X`, `ABS_Y`, `ABS_Z`: acceleration, including gravity. At rest on a table the axis + /// pointing up reads about `+accel_per_g`. + pub accel: [i32; 3], + /// `ABS_RX`, `ABS_RY`, `ABS_RZ`: angular rate about the same three axes. + pub gyro: [i32; 3], } /// One report from the pad: everything the kernel delivered between two `SYN_REPORT`s. @@ -4008,12 +4453,61 @@ pub struct TofFrame { /// Microseconds since `tofd` started — the sender's monotonic clock, like /// [`PadFrame::at_us`]. pub at_us: u64, + /// `CLOCK_MONOTONIC` when the frame was read, nanoseconds — the clock + /// [`RobotState::t_ns`] shares. Zero from a `tofd` predating it. (v24) + #[serde(default)] + pub t_ns: u64, pub rows: u8, pub cols: u8, pub distance_mm: Vec, pub status: Vec, } +/// Answer to [`Call::HeadImuStream`]. Describes the head IMU rather than merely accepting, like +/// [`TofStreamResult`]: a duck without the sensor still gets `accepted` with `sensor: None`. +#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(default)] +pub struct HeadImuStreamResult { + pub accepted: bool, + /// The IMU that answered, e.g. `BMI088`. `None` when there is none — see `unavailable`. + #[serde(skip_serializing_if = "Option::is_none")] + pub sensor: Option, + /// Why there is no IMU: not fitted, bus unreadable, chip-id mismatch. + #[serde(skip_serializing_if = "Option::is_none")] + pub unavailable: Option, + /// Sample rate the reader runs at, Hz. + pub hz: u8, +} + +/// One head-IMU sample — a [`method::HEAD_IMU_FRAME`] notification. +/// +/// The BMI088 on the HAT, read by `tofd` (it owns that I²C bus). All values are in the IMU's own +/// axes, which are tilted relative to the head/camera — the mount is not axis-aligned. To place a +/// sample in the trunk/camera frame, rotate it by [`FramesState::head_imu`] (the sensor→trunk +/// pose the kinematics compute for this tick). This is the head IMU, distinct from the body IMU +/// that [`RobotState::imu`] carries on the motor bus. Units: rad/s, m/s², unitless quaternion. +#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)] +#[serde(default)] +pub struct HeadImuFrame { + /// Samples since this `tofd` started — a consumer can see a gap it did not cause. + pub seq: u64, + /// Microseconds since `tofd` started (sender's monotonic clock, like [`TofFrame::at_us`]). + pub at_us: u64, + /// `CLOCK_MONOTONIC` when the sample was read, ns — the clock [`RobotState::t_ns`] shares. + pub t_ns: u64, + /// Angular velocity, rad/s, in the BMI088's own (tilted) sensor axes — NOT the head or + /// camera frame. Combine with [`FramesState::head_imu`] (the sensor→trunk pose from the + /// kinematics) to place it. See that field. + pub gyro: [f32; 3], + /// Specific force, m/s², BMI088 sensor axes. + pub accel: [f32; 3], + /// Madgwick orientation, scalar-first `[w, x, y, z]`, sensor→world (gravity down, yaw + /// arbitrary). The world here is the IMU's own; relate it to the trunk via the mount pose. + pub quat: [f32; 4], + /// Chip temperature, °C. + pub temp_c: f32, +} + /// See [`method::ROBOT_CHORALE`]. #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] #[serde(deny_unknown_fields)] @@ -4502,6 +4996,46 @@ macro_rules! log_startup_identity { }}; } +/// The clocks every daemon stamps with, so no two of them disagree about which clock +/// "monotonic" means. Nanoseconds. `monotonic` is `CLOCK_MONOTONIC` — what +/// [`RobotState::t_ns`] and [`TofFrame::t_ns`] carry; `realtime` is `CLOCK_REALTIME`, the +/// clock RTCP sender reports are stated in, read together in `media.video` so a peer can +/// relate the two. +pub mod clock { + fn read(clock: libc::clockid_t) -> u64 { + let mut ts = libc::timespec { + tv_sec: 0, + tv_nsec: 0, + }; + // SAFETY: `ts` is a valid, writable timespec and the clock ids are the constants libc + // exports for this platform. + let rc = unsafe { libc::clock_gettime(clock, &mut ts) }; + debug_assert_eq!(rc, 0, "clock_gettime failed"); + (ts.tv_sec as u64) * 1_000_000_000 + ts.tv_nsec as u64 + } + + /// `CLOCK_MONOTONIC`, nanoseconds. + pub fn monotonic_ns() -> u64 { + read(libc::CLOCK_MONOTONIC) + } + + /// `CLOCK_REALTIME`, nanoseconds since the Unix epoch. + pub fn realtime_ns() -> u64 { + read(libc::CLOCK_REALTIME) + } + + #[cfg(test)] + mod tests { + #[test] + fn monotonic_advances_and_realtime_is_this_century() { + let a = super::monotonic_ns(); + let b = super::monotonic_ns(); + assert!(b >= a); + assert!(super::realtime_ns() > 1_600_000_000_000_000_000); + } + } +} + #[cfg(test)] mod tests { use super::test_support::every_call; @@ -4624,6 +5158,7 @@ mod tests { | Call::RobotSubscribe(_) | Call::PadInput | Call::TofStream + | Call::HeadImuStream ), "{} is on the Stream lane but is not a subscription", call.method() @@ -4636,7 +5171,7 @@ mod tests { fn every_call_covers_every_variant() { assert_eq!( every_call().len(), - 61, + 67, "a Call variant was added or removed — update every_call() and this count" ); } @@ -4850,6 +5385,9 @@ mod tests { // list: asking what exists is inspection, and support has to be able to ask it // on a robot it may not change. method::POLICY_FETCH, + // Replacing the detector writes fourteen megabytes to the eMMC and restarts + // `mediad`. `detector.check` stays off this list, like `policy.check`. + method::DETECTOR_INSTALL, // Binding the robot to an account, and unbinding it. On this list for a reason // none of the others share: it decides who can reach the robot from outside the // building, and it survives every reboot. `account.status` must stay off it — @@ -5216,6 +5754,10 @@ mod tests { fn the_theremin_block_is_absent_until_there_is_a_theremin() { let mut state = RobotState { t: 1.5, + t_ns: 0, + imu: None, + frames: None, + skeleton: Vec::new(), movement: MoveState { requested: [0.0; 3], applied: [0.0; 3], @@ -5273,6 +5815,10 @@ mod tests { fn robot_state_uses_the_documented_field_names() { let state = RobotState { t: 1.5, + t_ns: 0, + imu: None, + frames: None, + skeleton: Vec::new(), movement: MoveState { requested: [0.4, 0.0, 0.0], applied: [0.15, 0.0, 0.0], diff --git a/duckctl/src/main.rs b/duckctl/src/main.rs index b029d6d5..c5cfd98e 100644 --- a/duckctl/src/main.rs +++ b/duckctl/src/main.rs @@ -20,6 +20,11 @@ //! module the robot uses, so if the framing were asymmetric this would not work — which makes //! it a real test of the protocol rather than a reimplementation that could agree with itself. //! +//! The one thing that *is* asymmetric is how long a line may be, and it has to be: the robot's cap +//! bounds what an unpaired peer in radio range can make it buffer, and this end has no such peer. +//! Sharing the tight one made every reply over 8 KiB unreadable here while `btd` served it +//! correctly — see `framing::MAX_REPLY_LINE`. +//! //! ```text //! cargo run -p duckctl -- scan # robots in range, and their addresses //! cargo run -p duckctl -- status @@ -27,6 +32,7 @@ //! cargo run -p duckctl -- wifi connect "Pollen" --psk secret //! cargo run -p duckctl -- name "Ducky" //! cargo run -p duckctl -- call robot.health +//! cargo run -p duckctl -- logs robotd -n 100 //! ``` //! //! `DUCK_ROBOT` and `DUCK_PIN` in the environment are the defaults for `--name` and `--pin`, for @@ -422,6 +428,14 @@ fn deliver(command: &Command, address: &str) -> Result<(), Box { + let user = ssh_user(user.as_deref(), std::env::var("DUCK_BOARD_USER").ok()); + become_program("ssh", &ssh_argv(&user, address, command)) + } + Command::Scp { user, paths } => { + let user = ssh_user(user.as_deref(), std::env::var("DUCK_BOARD_USER").ok()); + become_program("scp", &scp_argv(&user, address, paths)) + } Command::Open { print, port } => { let url = console_url(address, *port); if *print { @@ -437,11 +451,104 @@ fn deliver(command: &Command, address: &str) -> Result<(), Box Err("this command does not resolve an address".into()), } } +/// Which account `ssh` and `scp` log into: the flag, else a non-empty `DUCK_BOARD_USER`, else +/// `radxa`. +/// +/// Empty is unset — `DUCK_BOARD_USER= duckctl ssh` reads as "not set", the same rule `DUCK_ROBOT` +/// follows, because a variable emptied to switch it off must not become an ssh login of `@host`. +/// `radxa` is the image's account and `dev-push.sh`'s default for the same variable. +fn ssh_user(flag: Option<&str>, env: Option) -> String { + flag.map(str::to_owned) + .or_else(|| env.filter(|user| !user.trim().is_empty())) + .unwrap_or_else(|| "radxa".to_owned()) +} + +/// `ssh`'s arguments: `user@address`, then whatever is to run there. +/// +/// The command's words are passed through as separate arguments and ssh joins them with spaces +/// on the far side, which is what `ssh host sudo robotctl pad pair` does at a prompt. Quoting for +/// the remote shell is the caller's, exactly as it would be there. +fn ssh_argv(user: &str, address: &str, command: &[String]) -> Vec { + let mut argv = vec![format!("{user}@{address}")]; + argv.extend(command.iter().cloned()); + argv +} + +/// What `scp` is refused for, checked **before** the radio is turned on. +/// +/// Two of them, and neither could be left to `scp`: its usage error cannot mention the rule that +/// is actually being broken. A copy with no `:` anywhere in it is the one worth catching — it is a +/// local-to-local copy, `scp` performs it happily, and nothing in the output says the robot was +/// never involved. Here rather than in `scp_argv` because the alternative is charging eight +/// seconds of scanning for a robot the command was never going to touch. +fn scp_refusal(paths: &[String]) -> Result<(), String> { + if !paths.iter().any(|path| path.starts_with(':')) { + return Err(format!( + "no path on the robot in `{}`\nA leading `:` is the robot: `duckctl scp report.md \ + :/tmp/` sends a file up, `duckctl scp :/var/log/robotd.log .` brings one down. \ + Without one this is a local-to-local copy that has nothing to do with a robot.", + paths.join(" "), + )); + } + if paths.len() < 2 { + return Err(format!( + "`{}` is a source with no destination. `scp` takes both: `duckctl scp \ + :/var/log/robotd.log .`", + paths[0], + )); + } + Ok(()) +} + +/// `scp`'s arguments: every `:path` pointed at the robot, everything else as typed. +/// +/// The rewrite is textual and deliberately narrow — a leading `:` becomes `user@address:` and +/// nothing else is touched — so `-r`, `-P`, a local path, and a path with a colon in the middle of +/// it all reach `scp` exactly as they were typed. `scp` itself decides what they mean. +fn scp_argv(user: &str, address: &str, paths: &[String]) -> Vec { + paths + .iter() + .map(|path| match path.strip_prefix(':') { + Some(remote) => format!("{user}@{address}:{remote}"), + None => path.clone(), + }) + .collect() +} + +/// Hand the terminal to `ssh` or `scp` and never come back. +/// +/// Become the program rather than run it: the terminal is then its from here on — its prompts, +/// its progress meter, its exit status, its handling of a dropped link — and nothing of this +/// process is left behind to be `Ctrl-C`d separately. The radio was released before this was +/// called, so there is nothing to clean up. +/// +/// The command line is echoed first, on stderr, because a tool that resolved the address for you +/// still owes you the address it resolved — and it is the line to paste when the next copy needs +/// a flag this does not pass. +fn become_program(program: &str, argv: &[String]) -> Result<(), Box> { + eprintln!("{program} {}", argv.join(" ")); + let mut child = std::process::Command::new(program); + child.args(argv); + #[cfg(unix)] + { + use std::os::unix::process::CommandExt; + let e = child.exec(); + Err(format!("could not run {program}: {e}").into()) + } + #[cfg(not(unix))] + { + let status = child + .status() + .map_err(|e| format!("could not run {program}: {e}"))?; + std::process::exit(status.code().unwrap_or(1)); + } +} + /// Where the console is, given where the robot is. /// /// Plain `http`, because a robot on a LAN has no certificate to offer and `ws://` from an `https` @@ -795,6 +902,55 @@ enum Command { /// stopped advertising the service to it is asked over BLE instead, which is slower and always /// answers. Ip, + /// ssh into the robot. + /// + /// `duckctl ssh` is `ssh @$(duckctl ip)` as one step: the address comes from the + /// advertisement the way `ip` finds it, and then this process *becomes* `ssh`, so the terminal, + /// the exit status and the key prompts are ssh's own. Anything after `--` is run on the robot + /// instead of opening a shell: `duckctl ssh -- sudo robotctl pad pair`. + /// + /// The user is `--user`, else `DUCK_BOARD_USER` from the environment — the same variable + /// `scripts/dev-push.sh` reads, so a laptop set up for pushing is set up for this — else + /// `radxa`, the image's default account. + Ssh { + /// The account on the robot. Without it, `DUCK_BOARD_USER`; without that, `radxa`. + #[arg(long, value_name = "USER")] + user: Option, + /// A command to run on the robot instead of opening a shell. Put it after `--`. + #[arg( + trailing_var_arg = true, + allow_hyphen_values = true, + value_name = "COMMAND" + )] + command: Vec, + }, + /// Copy files to or from the robot with `scp`. + /// + /// A path that starts with `:` is on the robot: `duckctl scp report.md :/tmp/` sends one up, + /// `duckctl scp :/var/log/robotd.log .` brings one down. That is `scp`'s own `host:path` with + /// the host left out, because the host is the thing this tool exists to find. Everything else + /// — local paths, `-r`, any other `scp` flag — is passed through as typed, and then this + /// process *becomes* `scp`, so the progress meter, the key prompts and the exit status are + /// `scp`'s own. + /// + /// `duckctl`'s own flags come before the paths — `duckctl --name ducky scp -r logs/ :/tmp/` — + /// and `--` ends them for anything after that this tool would otherwise read as its own. A + /// local file that really is named `:foo` is `./:foo`. + /// + /// The user resolves the way `ssh`'s does: `--user`, else `DUCK_BOARD_USER`, else `radxa`. + Scp { + /// The account on the robot. Without it, `DUCK_BOARD_USER`; without that, `radxa`. + #[arg(long, value_name = "USER")] + user: Option, + /// What to copy, `scp`-style, with a leading `:` for a path on the robot. + #[arg( + trailing_var_arg = true, + allow_hyphen_values = true, + required = true, + value_name = "PATH" + )] + paths: Vec, + }, /// Open the robot's console in a browser. /// /// The page `mediad` serves: the camera, and the controls a WebRTC peer is allowed to drive. @@ -811,6 +967,38 @@ enum Command { #[arg(long, default_value_t = 8080)] port: u16, }, + /// The tail of one daemon's journal. + /// + /// `duckctl logs robotd` after a robot has misbehaved, which is the question `journalctl` + /// answers on the robot and nothing answered from here. Reachable with no network at all, so + /// it works on the robot whose wifi never came up — the one whose logs are hardest to get. + /// + /// `duckctl call system.services` names the units, and an unknown name is refused with the + /// real list. `bluetooth` and `NetworkManager` are readable too: they are what is broken when + /// the robot cannot be reached at all. + /// + /// Not a `journalctl` command line, and it never will be: this is served over a radio anyone + /// in range can talk to, so the robot picks the unit from a fixed list. For searching, take + /// the `ip` and use ssh. + Logs { + /// `robotd`, `btd`, `configd`, `updaterd`, `padd`, `mediad`, `tofd`, `bluetooth` or + /// `NetworkManager`. The `.service` suffix is optional. + #[arg(value_name = "SERVICE")] + service: String, + /// How many lines from the end. + /// + /// The default is what a BLE link carries in a couple of seconds. More is allowed and + /// costs proportionally — the robot trims the oldest lines to fit what the radio can + /// carry, and says so when it did. + #[arg(long, short = 'n', default_value_t = 40)] + lines: usize, + /// Which boot: `0` is this one, `-1` the one before it. + /// + /// `--boot -1` is "what did it say before it restarted", which is the reason the journal + /// is configured to survive a reboot at all (`deploy/journald.conf.d`). + #[arg(long, short = 'b', default_value_t = 0, allow_hyphen_values = true)] + boot: i32, + }, /// Version handshake plus update status. Status, /// What the robot is running: the API version, the release, and the revision it was built from. @@ -1155,16 +1343,26 @@ async fn run() -> Result<(), Box> { let target = Target::new(cli.name.clone(), std::env::var("DUCK_ROBOT").ok()); let pin = resolve_pin(cli.pin.clone(), std::env::var("DUCK_PIN").ok()); + // Before the scan rather than after it: a copy that names no path on the robot is wrong on its + // own terms, and finding the robot first would charge eight seconds for the privilege. + if let Command::Scp { paths, .. } = &cli.command { + scp_refusal(paths)?; + } + // `scan` shares the discovery below and then stops, because a listing and a search look for the // same thing and differ only in what they do with it. It connects to nothing at all: that is // what makes it the safe command to reach for when a robot cannot be reached, and it is also why // it can only report what an advertisement carries. let list_only = matches!(cli.command, Command::Scan); - // `ip` and `open` want one field out of an advertisement, so they read it the way `scan` does — - // and unlike `scan` they connect after all when no advertisement carried one. Cheap read first, - // call second: without the fallback these two commands would fail on exactly the laptops that - // use them most, because a robot bonded to this Mac often stops advertising the service to it. - let resolving = matches!(cli.command, Command::Ip | Command::Open { .. }); + // `ip`, `open`, `ssh` and `scp` want one field out of an advertisement, so they read it the way + // `scan` does — and unlike `scan` they connect after all when no advertisement carried one. + // Cheap read first, call second: without the fallback these commands would fail on exactly the + // laptops that use them most, because a robot bonded to this Mac often stops advertising the + // service to it. + let resolving = matches!( + cli.command, + Command::Ip | Command::Open { .. } | Command::Ssh { .. } | Command::Scp { .. } + ); let manager = Manager::new().await?; let adapter = manager @@ -1471,7 +1669,10 @@ async fn run() -> Result<(), Box> { .await?; } - let mut reassembler = Reassembler::new(); + // `for_replies`, not `new`: this is the client end, and the robot's answers are bounded by + // `MAX_REPLY_LINE` rather than by the tighter cap that limits what a radio peer may send. A + // `logs` tail is the reply that outgrew the other one. + let mut reassembler = Reassembler::for_replies(); // The deadline is **idle**, not total: it is pushed back by every notification that arrives, // because a robot sending progress is a robot that is working. See `REPLY_TIMEOUT`. let mut deadline = tokio::time::Instant::now() + timeout; @@ -1522,6 +1723,17 @@ async fn run() -> Result<(), Box> { }; } + // `logs` asked for text, so it prints text. A journal tail rendered as a JSON array + // of escaped strings is the one reply nobody can read, and reading it is the whole + // point of the command. A refusal still prints as JSON below, because the refusal + // names the units it would have accepted and that is worth seeing whole. + if let Command::Logs { service, boot, .. } = &cli.command + && value.get("error").is_none() + { + let _ = peripheral.disconnect().await; + return print_journal(&value["result"], service, *boot); + } + println!("{}", serde_json::to_string_pretty(&value)?); let _ = peripheral.disconnect().await; // A JSON-RPC error is the robot answering, not this tool failing — so it is @@ -1546,6 +1758,49 @@ async fn run() -> Result<(), Box> { } } +/// Print a journal tail: the lines on stdout, everything about them on stderr. +/// +/// The split is what makes `duckctl logs robotd | grep panic` work — a truncation note or an +/// empty-tail explanation in that pipe would be a line the robot never logged. +fn print_journal( + result: &serde_json::Value, + service: &str, + boot: i32, +) -> Result<(), Box> { + let unit = result["unit"].as_str().unwrap_or(service); + let Some(lines) = result["lines"].as_array() else { + // A well-formed answer in a shape this build does not know. Printed rather than + // paraphrased, because the reply itself is the only evidence of what happened. + return Err(format!( + "the robot answered system.logs with no `lines`, which this version of duckctl cannot read:\n{}", + serde_json::to_string_pretty(result)? + ) + .into()); + }; + + for line in lines { + match line.as_str() { + Some(text) => println!("{text}"), + None => println!("{line}"), + } + } + + if lines.is_empty() { + // Not an error: a daemon that has not run this boot is a real answer, and it is a + // different one from a daemon that is running and silent. `system.services` tells them + // apart, so that is where this points. + eprintln!( + "no lines for {unit} in boot {boot} — it may not have run then. `duckctl call system.services` says whether it is running now." + ); + } + if result["truncated"] == serde_json::Value::Bool(true) { + eprintln!( + "note: older lines were dropped to fit what BLE can carry. Ask for fewer with `-n`, or read the whole journal over ssh — `duckctl ip` has the address." + ); + } + Ok(()) +} + /// How a wait for the next notification ended. /// /// Three outcomes rather than two, because "nothing arrived" hides the one that is diagnosable: @@ -1665,7 +1920,7 @@ async fn read_line( notifications: &mut (impl futures::Stream + Unpin), timeout: Duration, ) -> Result> { - let mut reassembler = Reassembler::new(); + let mut reassembler = Reassembler::for_replies(); let deadline = tokio::time::Instant::now() + timeout; loop { @@ -1774,7 +2029,9 @@ fn request_line(command: &Command) -> Result<(String, Duration), Box ("net.status", serde_json::json!({}), REPLY_TIMEOUT), + Command::Ip | Command::Open { .. } | Command::Ssh { .. } | Command::Scp { .. } => { + ("net.status", serde_json::json!({}), REPLY_TIMEOUT) + } Command::Version => ( "hello", serde_json::json!({ "api_version": duck_ipc_proto::API_VERSION }), @@ -1782,6 +2039,18 @@ fn request_line(command: &Command) -> Result<(String, Duration), Box return update_request_line(update), Command::Info => ("system.info", serde_json::json!({}), REPLY_TIMEOUT), + // A journal read is a `journalctl` spawn on the robot plus a reply several times larger + // than any other, chunked at 20 bytes a notification — seconds rather than milliseconds, + // so it gets the slow budget for the same reason `wifi scan` does. + Command::Logs { + service, + lines, + boot, + } => ( + proto::method::SYSTEM_LOGS, + serde_json::json!({ "unit": service, "lines": lines, "boot": boot }), + SLOW_REPLY_TIMEOUT, + ), Command::Health => ("robot.health", serde_json::json!({}), REPLY_TIMEOUT), Command::Name { name } => ( "system.setName", @@ -2649,6 +2918,104 @@ mod tests { )); } + /// `ssh` takes a user and a trailing command, and the user falls back the documented way: the + /// flag, a non-empty `DUCK_BOARD_USER`, then the image's account. An emptied variable is unset, + /// not a login of `@host`. + #[test] + fn ssh_resolves_its_user_and_passes_the_command_through() { + let cli = Cli::try_parse_from(["duckctl", "ssh", "--user", "pierre"]).expect("parses"); + let Command::Ssh { user, command } = &cli.command else { + panic!("not an ssh command"); + }; + assert_eq!(user.as_deref(), Some("pierre")); + assert!(command.is_empty()); + + let cli = Cli::try_parse_from(["duckctl", "ssh", "--", "sudo", "robotctl", "pad", "pair"]) + .expect("a trailing command parses"); + let Command::Ssh { user, command } = &cli.command else { + panic!("not an ssh command"); + }; + assert_eq!(user, &None); + assert_eq!( + ssh_argv(&ssh_user(user.as_deref(), None), "192.168.10.136", command), + ["radxa@192.168.10.136", "sudo", "robotctl", "pad", "pair"] + ); + + assert_eq!(ssh_user(Some("pierre"), Some("antoine".into())), "pierre"); + assert_eq!(ssh_user(None, Some("antoine".into())), "antoine"); + assert_eq!(ssh_user(None, Some("".into())), "radxa"); + assert_eq!(ssh_user(None, None), "radxa"); + } + + /// A leading `:` is the robot, in either operand, and everything else reaches `scp` as typed. + #[test] + fn scp_points_colon_paths_at_the_robot_and_leaves_the_rest_alone() { + let up = scp_argv("radxa", "192.168.10.136", &paths(&["report.md", ":/tmp/"])); + assert_eq!(up, ["report.md", "radxa@192.168.10.136:/tmp/"]); + + let down = scp_argv( + "pierre", + "192.168.10.136", + &paths(&[":/var/log/robotd.log", "."]), + ); + assert_eq!(down, ["pierre@192.168.10.136:/var/log/robotd.log", "."]); + + // Flags, several sources, and a bare `:` for the home directory — all of it passes + // through, because the rewrite only ever looks at the first character. + let many = scp_argv("radxa", "192.168.10.136", &paths(&["-r", "a", "b:c", ":"])); + assert_eq!(many, ["-r", "a", "b:c", "radxa@192.168.10.136:"]); + } + + /// The two refusals `scp`'s own usage error could not have explained: a copy that names no + /// path on the robot, and a source with nothing to copy it to. + #[test] + fn scp_refuses_a_copy_the_robot_has_nothing_to_do_with() { + let local = scp_refusal(&paths(&["a", "b"])).expect_err("neither side is the robot"); + assert!(local.contains("leading `:`"), "names the rule: {local}"); + assert!(local.contains("duckctl scp"), "shows the shape: {local}"); + + let lonely = scp_refusal(&paths(&[":/tmp/x"])).expect_err("a source with no destination"); + assert!( + lonely.contains("destination"), + "says what is missing: {lonely}" + ); + + scp_refusal(&paths(&["report.md", ":/tmp/"])).expect("a copy that touches the robot"); + scp_refusal(&paths(&["-r", ":/tmp/logs", "."])).expect("a flag is not an operand"); + } + + /// `scp` takes the same `--user` as `ssh`, its paths are trailing, and it needs at least one. + #[test] + fn scp_parses_its_user_and_requires_a_path() { + let cli = Cli::try_parse_from(["duckctl", "scp", "--user", "pierre", "x", ":/tmp/"]) + .expect("parses"); + let Command::Scp { user, paths } = &cli.command else { + panic!("not an scp command"); + }; + assert_eq!(user.as_deref(), Some("pierre")); + assert_eq!(paths, &["x".to_owned(), ":/tmp/".to_owned()]); + + // `--` ends `duckctl`'s flags, so an `scp` flag of the same shape is not mistaken for one. + let cli = Cli::try_parse_from(["duckctl", "scp", "--", "-r", "logs/", ":/tmp/"]) + .expect("a flag for scp parses after `--`"); + let Command::Scp { paths, .. } = &cli.command else { + panic!("not an scp command"); + }; + assert_eq!( + paths, + &["-r".to_owned(), "logs/".to_owned(), ":/tmp/".to_owned()] + ); + + assert!( + Cli::try_parse_from(["duckctl", "scp"]).is_err(), + "scp with nothing to copy" + ); + } + + fn paths(paths: &[&str]) -> Vec { + paths.iter().map(|path| (*path).to_owned()).collect() + } + #[test] fn a_robot_broadcasts_where_it_is() { let (properties, duck) = advertised( @@ -2724,6 +3091,48 @@ mod tests { assert!(!answers_to("duck-c51b [", "duck-c51b")); } + /// A negative boot offset is an argument, not a mistyped flag. + /// + /// `--boot -1` is the whole reason this command is worth having — "what did it say before it + /// restarted" — and clap rejects a value starting with `-` unless the argument says + /// otherwise. Pinned because the failure is at parse time, on the one invocation nobody + /// reaches for until something is already wrong. + #[test] + fn the_boot_before_this_one_can_be_asked_for() { + let wire = |args: &[&str]| { + let cli = Cli::try_parse_from([&["duckctl"], args].concat()).expect("parses"); + request_line(&cli.command).expect("a request").0 + }; + + let previous = wire(&["logs", "btd", "--boot", "-1"]); + assert!( + previous.contains(duck_ipc_proto::method::SYSTEM_LOGS), + "{previous}" + ); + assert!(previous.contains(r#""boot":-1"#), "{previous}"); + + let default = wire(&["logs", "btd"]); + assert!(default.contains(r#""boot":0"#), "{default}"); + assert!(default.contains(r#""lines":40"#), "{default}"); + // The unit goes over verbatim, suffix and all: the robot resolves it against its own + // list, so this tool has no list to keep in step. + assert!( + wire(&["logs", "NetworkManager.service"]) + .contains(r#""unit":"NetworkManager.service""#) + ); + } + + /// A journal read spawns `journalctl` and carries a reply several times larger than any + /// other, chunked at 20 bytes a notification. The reply budget has to match. + #[test] + fn reading_a_journal_gets_the_slow_budget() { + let cli = Cli::try_parse_from(["duckctl", "logs", "robotd", "-n", "500"]).expect("parses"); + assert_eq!( + request_line(&cli.command).expect("a request").1, + SLOW_REPLY_TIMEOUT + ); + } + /// **The Hub commands take the budget their work needs**, because the failure otherwise /// looks like a robot that stopped talking rather than a mirror having a bad day. #[test] diff --git a/hooks/postinstall b/hooks/postinstall index ad5e6513..0a3001e5 100755 --- a/hooks/postinstall +++ b/hooks/postinstall @@ -62,6 +62,16 @@ if [ -f "$script" ]; then || echo "postinstall: the login-shell files did not install; they are cosmetic" >&2 fi +# Apt off the boot path: Armbian's `@reboot` simulated upgrade and the stock apt-daily timers. Here +# rather than in setup-board.sh alone because every board in the field was provisioned with them +# running, and this hook is what reaches those boards. Never fatal — a slower boot is not worth a +# rollback. +script=scripts/setup-quiet-boot.sh +if [ -f "$script" ]; then + sh "$script" \ + || echo "postinstall: could not take apt off the boot path; boot stays slower than it needs to be" >&2 +fi + # The robot's voice bank, rendered from the SoC serial by the release's own generator. # Idempotent — a marker records the seed and bank version, and a current bank is a no-op — # so this runs on every install and only actually renders on the first one (or when the @@ -84,6 +94,16 @@ if [ -f "$script" ]; then || echo "postinstall: policies were not seeded; robotctl health will say so" >&2 fi +# The duck detector, the same way and for the same reason: `mediad` reads it from +# /opt/robot/detector/current, a retrain is a tag on the Hub rather than a daemon release, and the +# seeder never touches a set it did not install. Warned about, never fatal — the detector is +# off by default, and a robot asked to look for ducks with no model says so in mediad's journal. +script=scripts/seed-detector.sh +if [ -f "$script" ]; then + sh "$script" \ + || echo "postinstall: the duck detector was not seeded; mediad will say so if asked for it" >&2 +fi + [ -d systemd ] || exit 0 # Users and groups before units: a unit naming a `User=` that does not exist fails to start, and diff --git a/kinematics/src/hand.rs b/kinematics/src/hand.rs index 8ca49019..1b08ec4c 100644 --- a/kinematics/src/hand.rs +++ b/kinematics/src/hand.rs @@ -143,7 +143,9 @@ impl Tracker { } } - if in_band.len() < self.config.min_zones { + // The band must hold at least one zone whatever `min_zones` says: at 0 the length + // check passes on an empty band, and the percentile index below would panic on it. + if in_band.len() < self.config.min_zones.max(1) { // Hold the last hand briefly. This is the whole anti-chop mechanism: the note // rides over a dropout, and stops when one lasts. return match self.last { @@ -318,6 +320,21 @@ mod tests { assert!(tracker.track(&distance, &status, Instant::now()).is_some()); } + /// `min_zones = 0` is a legal config, and an empty band then passes the length check — + /// straight into a percentile index that has nothing to index. No zones is no hand, + /// whatever the floor is. + #[test] + fn an_empty_band_is_not_a_hand_even_with_min_zones_zero() { + let mut tracker = Tracker::new(Config { + min_zones: 0, + ..Config::default() + }); + assert_eq!(tracker.track(&[], &[], Instant::now()), None); + // But a single usable zone still is one: the floor is on emptiness, not on count. + let (distance, status) = frame(0.25, 5, 1); + assert!(tracker.track(&distance, &status, Instant::now()).is_some()); + } + /// A negative distance under a believed status is a failed convergence, and a frame /// shorter than the grid must not panic or read past its end — the wire carries vectors, /// and a peer from another release can send fewer. diff --git a/kinematics/src/head.rs b/kinematics/src/head.rs index 95e1ec5e..4e43b0b8 100644 --- a/kinematics/src/head.rs +++ b/kinematics/src/head.rs @@ -31,6 +31,9 @@ pub struct HeadFk { /// true position, a couple of centimetres from the camera it would /// otherwise borrow. tof: Option, + /// The MJCF's `head_imu` site (the BMI088 on the HAT), when the asset carries one. Tilted + /// relative to the camera — the mount is not axis-aligned. + head_imu: Option, joints: [usize; 4], } @@ -44,6 +47,7 @@ impl HeadFk { .site("head_camera") .expect("model has a head_camera site"), tof: model.site("tof"), + head_imu: model.site("head_imu"), joints: HEAD_JOINTS .map(|name| model.joint_index(name).expect("model has the head joints")), } @@ -84,6 +88,16 @@ impl HeadFk { } } + /// Head-IMU (BMI088) pose in the trunk frame, when the asset has a `head_imu` site. + /// + /// The site frame is the sensor's own axes as mounted — tilted, not aligned with the camera. + /// Rotating a sensor-frame vector by the result's quat expresses it in the trunk, which is how + /// a consumer places `head_imu.stream` samples in the robot frame. `None` for an asset with no + /// such site (the caller then has no mount and must skip the transform). + pub fn head_imu_in_trunk(&self, joints: [f64; 4]) -> Option { + self.head_imu.map(|site| self.site_in_trunk(site, joints)) + } + /// A head-chain site posed by the four head joints, everything else at /// zero. fn site_in_trunk(&self, site: SiteId, joints: [f64; 4]) -> Pose { diff --git a/kinematics/src/lib.rs b/kinematics/src/lib.rs index 48898f31..5a0407af 100644 --- a/kinematics/src/lib.rs +++ b/kinematics/src/lib.rs @@ -59,6 +59,13 @@ pub struct Model { /// Per site, the flattened root→site chain (the site's own rest pose is the /// final, joint-less link). chains: Vec>, + /// The body tree, in MJCF order (a body's parent always precedes it), for + /// whole-skeleton FK. Parallel arrays: name, parent index (`None` = root), + /// rest pose in the parent frame, and the hinge joint to the parent. + body_names: Vec, + body_parents: Vec>, + body_rest: Vec, + body_joint: Vec>, /// The trunk's standing height above the floor, metres — the scene's drop /// height for `trunk_base`, which is where the training world puts the /// floor relative to the trunk frame. @@ -75,12 +82,18 @@ impl Model { let mut joint_names = Vec::new(); let mut joint_ranges = Vec::new(); let mut body_joint: Vec> = Vec::with_capacity(tree.bodies.len()); + let mut body_names = Vec::with_capacity(tree.bodies.len()); + let mut body_parents = Vec::with_capacity(tree.bodies.len()); + let mut body_rest = Vec::with_capacity(tree.bodies.len()); for body in &tree.bodies { body_joint.push(body.joint.as_ref().map(|j| { joint_names.push(j.name.clone()); joint_ranges.push(j.range); (joint_names.len() - 1, j.axis) })); + body_names.push(body.name.clone()); + body_parents.push(body.parent); + body_rest.push(body.rest); } let mut site_names = Vec::with_capacity(tree.sites.len()); @@ -115,6 +128,10 @@ impl Model { joint_ranges, site_names, chains, + body_names, + body_parents, + body_rest, + body_joint, trunk_height: tree.trunk_pos[2], }) } @@ -188,6 +205,44 @@ impl Model { } t } + + /// Every body's pose in the trunk frame, in [`Model::body_names`] order, at + /// these angles — the whole skeleton, of which the sites are a few leaves. + /// + /// One pass in tree order (a parent always precedes its children, so its + /// pose is already known): a body is its parent's pose, moved by its rest + /// offset, then turned by its own hinge. `angles` covers every joint or it + /// panics, the same contract as [`Model::site_pose`]. + pub fn body_poses(&self, angles: &[f64]) -> Vec { + assert_eq!( + angles.len(), + self.joint_names.len(), + "angle slice must cover every joint" + ); + let mut poses = vec![Pose::IDENTITY; self.body_names.len()]; + for i in 0..self.body_names.len() { + let mut t = match self.body_parents[i] { + Some(p) => poses[p] * self.body_rest[i], + None => self.body_rest[i], // the root's world drop is stripped: it rests at identity + }; + if let Some((idx, axis)) = self.body_joint[i] { + t.quat = t.quat * Quat::from_axis_angle(axis, angles[idx]); + } + poses[i] = t; + } + poses + } + + /// Body names in [`Model::body_poses`] order. + pub fn body_names(&self) -> impl Iterator { + self.body_names.iter().map(String::as_str) + } + + /// Each body's parent index in [`Model::body_poses`] order (`None` = root) — + /// the skeleton's edges, for drawing it or matching visual meshes to links. + pub fn body_parents(&self) -> &[Option] { + &self.body_parents + } } #[cfg(test)] @@ -238,6 +293,50 @@ mod tests { assert_eq!(model.site_pose(origin, &[0.0, 0.0]).pos, [0.0; 3]); } + #[test] + fn body_poses_place_the_whole_skeleton() { + let model = Model::parse(ARM).expect("parses"); + let names: Vec<&str> = model.body_names().collect(); + assert_eq!(names, ["trunk_base", "upper", "lower"]); + assert_eq!(model.body_parents(), &[None, Some(0), Some(1)]); + + // Straight: upper sits one out from the root, lower one further. + let p = model.body_poses(&[0.0, 0.0]); + assert!((p[1].pos[0] - 1.0).abs() < 1e-12 && p[1].pos[1].abs() < 1e-12); + assert!((p[2].pos[0] - 2.0).abs() < 1e-12 && p[2].pos[1].abs() < 1e-12); + + // Shoulder at 90°: upper's origin does not move (a hinge turns the frame + // in place), but lower swings out along +y. + let p = model.body_poses(&[std::f64::consts::FRAC_PI_2, 0.0]); + assert!( + (p[1].pos[0] - 1.0).abs() < 1e-12, + "upper x: {}", + p[1].pos[0] + ); + assert!( + (p[2].pos[0] - 1.0).abs() < 1e-12, + "lower x: {}", + p[2].pos[0] + ); + assert!( + (p[2].pos[1] - 1.0).abs() < 1e-12, + "lower y: {}", + p[2].pos[1] + ); + + // The body under a site agrees with the site FK, minus the site offset. + let tip = model.site("tip").expect("tip"); + let s = model.site_pose(tip, &[0.3, -0.4]); + let lower = model.body_poses(&[0.3, -0.4])[2]; + // The tip site sits [1,0,0] out in `lower`; walking it back lands on lower's origin. + let back = lower + * Pose { + pos: [1.0, 0.0, 0.0], + quat: Quat::IDENTITY, + }; + assert!((s.pos[0] - back.pos[0]).abs() < 1e-12 && (s.pos[1] - back.pos[1]).abs() < 1e-12); + } + #[test] fn the_embedded_alpha_model_has_what_the_daemon_asks_for() { let model = Model::alpha(); diff --git a/kinematics/src/mjcf.rs b/kinematics/src/mjcf.rs index 374b538d..7860e4f2 100644 --- a/kinematics/src/mjcf.rs +++ b/kinematics/src/mjcf.rs @@ -36,6 +36,8 @@ pub enum ParseError { } pub(crate) struct Body { + /// The MJCF `name`, retained for whole-skeleton FK (matching a pose to a link). + pub name: String, /// Index into `Tree::bodies`. `None` only for the root (trunk_base). pub parent: Option, /// Rest pose of this body in its parent's frame (identity for the root). @@ -94,6 +96,7 @@ pub(crate) fn parse(xml: &str) -> Result { // The root is anchored at identity — its MJCF `pos` is where MuJoCo drops // the robot into the world, which trunk-frame FK must not inherit. tree.bodies.push(Body { + name: trunk.attribute("name").unwrap_or("trunk_base").to_owned(), parent: None, rest: Pose::IDENTITY, joint: None, @@ -130,6 +133,7 @@ fn walk_body(node: roxmltree::Node, parent: usize, tree: &mut Tree) -> Result<() let idx = tree.bodies.len(); tree.bodies.push(Body { + name: node.attribute("name").unwrap_or("").to_owned(), parent: Some(parent), rest: rest_pose(node)?, joint, diff --git a/kinematics/src/tof.rs b/kinematics/src/tof.rs index 35cc94e4..9fc152fc 100644 --- a/kinematics/src/tof.rs +++ b/kinematics/src/tof.rs @@ -31,7 +31,7 @@ const N_ZONES: usize = ROWS * COLS; /// The sensor's square field of view, degrees per axis — 45°×45° per ST's /// datasheet for both generations, the value the prototype's beam table used. -const FOV_DEG: f64 = 45.0; +pub const FOV_DEG: f64 = 45.0; /// What one zone's return turned out to be, once it has a place in the world. #[derive(Debug, Clone, Copy, PartialEq)] diff --git a/mediad/Cargo.toml b/mediad/Cargo.toml index e77b50ff..1fa76ed2 100644 --- a/mediad/Cargo.toml +++ b/mediad/Cargo.toml @@ -16,14 +16,24 @@ description = "Camera, mic, WebRTC — and the remote gateway" # that other, and the cost is a sysroot the whole workspace now builds against. [dependencies] +# One function: `read_access_token`. The crate that performs the login is the one that defines the +# credential's file format, so the reader lives next to the writer and a test there pins the key +# this depends on — rather than a two-field struct here and a test in `updater` describing it. +hf-robot-account = "0.1" duck-ipc-proto = { path = "../duck-ipc-proto" } -# `[media]` and `[detect]` in /etc/robot/robotd.toml — what this daemon streams, and what it looks +# `[media]` and `[duck_detector]` in /etc/robot/robotd.toml — what this daemon streams, and what it looks # for. The same crate `robotd` parses that file with and the same one `robotctl configure` edits it # through, so the schema, the defaults and the editor cannot drift from what is read here. robotd-params = { path = "../robotd-params" } anyhow = "1" +# `getgrnam` and `chown`, to hand the frame socket to the `robot` group after binding — the same +# thing `tof` and `padd` do with theirs. Mode 0660 alone would leave it `mediad:mediad`, and the +# operator is only ever added to `robot`. +libc = "0.2" # The detector itself: the letterbox+turn, the NPU binding, the ONNX fallback and the decoder. duck-detect = { path = "../duck-detect" } +# JPEG for the frame stream and PNG for lossless console snapshots. +image = { version = "0.25", default-features = false, features = ["jpeg", "png"] } # The console's one route. Already in `Cargo.lock` — `updater` uses it for its test mirror — so it # is known-good against this toolchain and the cross build. The alternative was a hand-rolled # HTTP/1.1 responder: about sixty lines of hand-written request parsing bound to 0.0.0.0, written to @@ -34,6 +44,36 @@ duck-detect = { path = "../duck-detect" } # tracing middleware are all weight for routes that do not exist. axum = { version = "0.8.9", default-features = false, features = ["http1", "tokio"] } clap = { workspace = true, features = ["derive"] } +# The relay's half of the bridge: HTTPS to the rendezvous service — SSE inbound, `POST /send` +# outbound. Same line `updater` uses, minus `form` (nothing here posts a form), so the TLS stack +# and its version are the ones already resolved for this workspace and its cross build. +# `stream` is what makes the SSE body readable as it arrives rather than at the end. +reqwest = { version = "0.13.4", default-features = false, features = ["rustls", "json", "stream", "http2", "charset"] } +# Server-Sent Events framing — `event:`/`data:` lines, multi-line data, comments and the +# reconnection field. Sixty lines to hand-roll and one of them would be the bug: a `data:` split +# across two TCP reads. +eventsource-stream = "0.2.3" +# Parsing, not fetching: the one endpoint this daemon sends the account token to is +# checked before the token can reach it, and "is this loopback" and "does this carry +# userinfo" are the two questions a hand-rolled split on `:` gets wrong. Already in the +# lock via `reqwest`. +url.workspace = true +# `StreamExt`, for the byte stream `reqwest` hands over. Already in the lock via `updater`. +futures-util = "0.3.33" +# The local half of the bridge: a WebSocket client to `webrtcsink`'s own signalling server on +# 127.0.0.1:8443, where this daemon is a *consumer* — the same role the console page plays on a +# LAN. `ws://` only, on loopback, so no TLS: `connect` and `handshake` and nothing else. +# `rustls-tls-webpki-roots` because the frame stream dials a `wss://` Space, where the +# signalling bridge only ever dialled `ws://127.0.0.1`. The roots are webpki's rather +# than the platform's: a board's trust store is whatever the image shipped, and a +# consumer that stops working after an OS update is a worse failure than a pinned set. +tokio-tungstenite = { version = "0.30", default-features = false, features = ["connect", "handshake", "rustls-tls-webpki-roots"] } +# Jitter on the reconnect backoff, so a fleet coming back after a service restart does not arrive +# in lockstep. Already in the lock. +rand = "0.10.2" +# The relay's message types. `serde_json` was enough while every JSON object here was forwarded +# verbatim; the rendezvous protocol is the first one this daemon has to *write*. +serde = { workspace = true, features = ["derive"] } serde_json.workspace = true tokio = { workspace = true, features = ["rt", "rt-multi-thread", "net", "io-util", "sync", "time", "macros"] } tracing.workspace = true @@ -69,4 +109,10 @@ glib = "0.21" duck-ipc-proto = { path = "../duck-ipc-proto", features = ["test-support"] } # Fake daemons on real unix sockets, which is how the pipe is tested without a WebRTC peer. tempfile = "3" +# The fake rendezvous service the relay is tested against: `axum` needs a `Stream` to serve SSE +# from, and one `async_stream::stream!` is what a hand-written `Stream` impl would be. +async-stream = "0.3.6" +# `ws` for the fake signalling server the bridge is tested against — the local side of a session +# is a WebSocket, and a test that faked it with a channel would not exercise the framing. +axum = { version = "0.8.9", default-features = false, features = ["http1", "tokio", "json", "ws"] } tokio = { workspace = true, features = ["rt-multi-thread", "macros"] } diff --git a/mediad/src/camera.rs b/mediad/src/camera.rs new file mode 100644 index 00000000..860aea79 --- /dev/null +++ b/mediad/src/camera.rs @@ -0,0 +1,460 @@ +//! What the camera's geometry is, so a consumer can do more than look at the picture. +//! +//! A frame is pixels. Turning pixels into directions — which is what SLAM, visual odometry, or +//! "how far away is that duck" all need — takes the **intrinsics**: the focal length in pixels and +//! where the optical axis crosses the image. Without them a monocular reconstruction is +//! scale-free and its angles are wrong; with them it maps a room. +//! +//! # Derived from the field of view, so it can be checked rather than trusted +//! +//! The IMX219's full array is 3280×2464 at a 1.12 µm pitch — 3.67 mm wide — behind the head +//! camera's ~3.05 mm M12 lens, a **horizontal field of view of ~62°** across the full width. On +//! this board's Rockchip driver the production **1920×1080@30** mode is a *scaled full-frame +//! readout* (the whole 62° width, downscaled and cropped to 16:9), **not** the native-pixel 39° +//! crop the datasheet's "1080p" implies — validated on hardware 2026-09-08: a calibration of +//! confirmed-1920×1080 frames solves to 62°, and the family default in `robotd-params` carries it +//! (`fx` ≈ 1062 at 1280×720). So the **field of view** — not a native focal length — is what the +//! nominal geometry rests on, because the field of view is what stays fixed as the ISP scales the +//! frame to whatever `[media] quality` asks for: +//! +//! - Focal length in pixels of a delivered frame `W` wide: `fx = (W/2) / tan(62°/2)`, so 1280 wide +//! gives `fx` ≈ 1065; a uniform scale moves `fy`, `cx`, `cy` with it. +//! - The vertical is the 16:9 crop of that, and the principal point is *assumed* central — which is +//! the part a real calibration corrects (the measured `cy` sits ~110 px low). +//! +//! # And when the pinned mode is not confirmed, this publishes nothing +//! +//! `pin_sensor_mode` shells out to `media-ctl` and can fail — a board without `v4l-utils`, an +//! entity name that moved. Capture still works from the 3280×2464 boot mode (also ~62°, at 21 fps), +//! but its exact 4:3→16:9 framing is not the pinned mode's, so this withholds nominal intrinsics +//! rather than publish a geometry it cannot vouch for. **Numbers that are quietly wrong are worse +//! than none**: a consumer told nothing knows it must calibrate. +//! +//! # Nominal is not calibrated, and the wire says which +//! +//! Everything above is the *design* of the camera, not a measurement of the one on this robot: +//! lens focal lengths vary by a few percent unit to unit, the principal point is never exactly the +//! centre, and nothing here models distortion at all. That is enough for a room-scale map and not +//! enough for photogrammetry, so a record built from them carries `calibrated: false`. In practice +//! every alpha robot publishes a measurement: `robotd-params` ships the family's solve as the +//! `[media.intrinsics]` default (the camera and lens are one part), and a robot with its own solve +//! written there publishes that. The nominal path is the fallback for a camera nobody has solved. + +/// The head camera's horizontal field of view, degrees — the full IMX219 array (3.67 mm wide) +/// behind the ~3.05 mm M12 lens. The one physical number the nominal geometry rests on, and the one +/// the family calibration confirms (it solves to 62.2°). See the module header. +const FULL_FIELD_HFOV_DEG: f64 = 62.0; + +/// The MuJoCo twin head camera's vertical field of view. The MJCF sets no `fovy`, so MuJoCo's +/// default 45° applies; see [`Intrinsics::sim`]. +const SIM_VFOV_DEG: f64 = 45.0; + +/// Horizontal focal length in pixels for a delivered frame `width` wide, from [`FULL_FIELD_HFOV_DEG`]. +/// The delivered field of view is the sensor's full width in every mode this driver offers, so this +/// depends only on the output width, not on which sensor mode fed it. +fn nominal_focal_px(width: u32) -> f64 { + f64::from(width) / 2.0 / (FULL_FIELD_HFOV_DEG / 2.0).to_radians().tan() +} + +/// The sensor readout mode `pipeline::pin_sensor_mode` puts the IMX219 in — carried so +/// [`Intrinsics::nominal`] can tell "the mode we pinned" from "we could not confirm it", and refuse +/// a delivered frame whose aspect ratio is not the mode's. +/// +/// # Both modes this driver offers are the full ~62° field +/// +/// The IMX219's full array is 3280×2464 at a 1.12 µm pitch behind the ~3.05 mm M12 lens — `2·atan( +/// 3280·1.12µm / 2 / 3.05mm)` ≈ **62°** horizontally. On this board's Rockchip driver `1920×1080` +/// is a *scaled full-frame* readout, not a native crop, so it is the same 62° field (validated on +/// hardware; see the module header): +/// +/// | mode | how | horizontal FOV | note | +/// |---|---|---|---| +/// | 1920×1080 | full field, scaled/cropped to 16:9 | **62°** | what this daemon pins, at 30 fps | +/// | 3280×2464 | full 4:3 array | **62°** | the boot mode, at 21 fps | +/// +/// So the field of view does not change with the mode — only the resolution and frame rate do, and +/// the geometry a consumer needs is the same either way. `1920×1080` is pinned for the frame rate; +/// there is no wide-vs-fast trade to make here, because the wide field is already the fast mode. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct SensorMode { + pub width: u32, + pub height: u32, +} + +impl SensorMode { + /// The mode `pipeline::pin_sensor_mode` asks for: the full field at 30 fps. + pub const PINNED: Self = Self { + width: 1920, + height: 1080, + }; +} + +/// Where the optical axis is and how long the focal length is, in pixels of a delivered frame. +/// +/// **For the frame as it is sent**, which is not rotated: the camera is mounted a quarter turn off +/// and nothing on the robot turns the pixels back (`pipeline`'s header says why). A consumer that +/// rotates the image has to rotate these too — `cx` and `cy` swap, and so do `fx` and `fy` — and +/// the `rotate` field alongside these in `media.video` is what tells it by how much. +/// Where a published calibration came from — so a consumer can tell *this* robot's own solve from +/// the family's, which look identical in the numbers. +#[derive(Debug, Clone, Copy, PartialEq, Eq, serde::Serialize)] +#[serde(rename_all = "lowercase")] +pub enum Source { + /// A `[media.intrinsics]` calibration measured on this robot. + Robot, + /// The hardware family's shared calibration (`robotd-params` ships it): a real solve of the + /// same camera-and-lens part, not of this particular unit. + Family, + /// The module's design figures — arithmetic from the datasheet, not a measurement. + Nominal, + /// The MuJoCo twin's rendered camera — exact geometry from the simulator's field of view, not a + /// physical measurement (there is no physical sensor). Lets twin recordings self-describe. + Sim, +} + +#[derive(Debug, Clone, PartialEq, serde::Serialize)] +pub struct Intrinsics { + pub fx: f64, + pub fy: f64, + pub cx: f64, + pub cy: f64, + /// **Whether these came from a real solve.** `false` is `Source::Nominal` — the module's design + /// figures, no distortion model, enough for a room-scale map rather than metrology. `true` + /// covers both a per-robot and the family calibration; [`Intrinsics::source`] says which, so a + /// consumer that needs *this* robot's own solve can tell it must still ask. + pub calibrated: bool, + /// Whose solve this is: this robot's, the family's, or none (the datasheet). See [`Source`]. + pub source: Source, + /// Radial and tangential terms in OpenCV's order — `k1 k2 p1 p2 k3` — or empty for "no model + /// of the distortion", which is what a nominal record has. + #[serde(skip_serializing_if = "Vec::is_empty")] + pub distortion: Vec, +} + +impl Intrinsics { + /// The design figures for a delivered frame, or `None` when the geometry is not known. + /// + /// `None` has one cause and it is worth surfacing rather than papering over: the sensor is not + /// in the mode this code pins, so how much of the sensor a frame covers is unknown. + pub fn nominal(mode: Option, width: u32, height: u32) -> Option { + let mode = mode?; + if width == 0 || height == 0 || mode.width == 0 || mode.height == 0 { + return None; + } + + // The delivered frame has to be a uniform scale of the mode — the same 16:9 aspect. The ISP + // *can* scale the axes independently, and then part of the frame was cropped or squashed on + // the way out, which the output size alone cannot tell apart from a resize. + let scale_x = f64::from(width) / f64::from(mode.width); + let scale_y = f64::from(height) / f64::from(mode.height); + if (scale_x - scale_y).abs() > 0.01 { + return None; + } + + // The focal length is fixed by the field of view and the delivered width. The field of view + // is the sensor's full ~62° in every mode, so it does not depend on which mode fed the frame. + let focal = nominal_focal_px(width); + Some(Self { + fx: focal, + fy: focal, + // The principal point is *assumed* central, which is what makes this nominal: on a real + // module it is a few pixels off (the measured cy sits ~110 px low), only a calibration + // knows in which direction. + cx: f64::from(width) / 2.0, + cy: f64::from(height) / 2.0, + calibrated: false, + source: Source::Nominal, + distortion: Vec::new(), + }) + } + + /// A calibration, scaled from the resolution it was measured at to the one being delivered. + /// + /// Scaling a calibration is exact for a uniform resize — every intrinsic is in pixels and + /// pixels all change size together — and wrong for a crop, which is why the record carries the + /// resolution it was taken at rather than assuming one. An aspect change between the two means + /// the second image is not the first one resized, so this refuses it. + pub fn scaled_from( + measured: &robotd_params::CameraIntrinsics, + width: u32, + height: u32, + ) -> Option { + Self::scaled(measured, width, height, Source::Robot) + } + + /// A calibration scaled to the delivered frame, tagged with whose solve it is. `Robot` for a + /// per-robot `[media.intrinsics]` table, `Family` for the shipped family default. + fn scaled( + measured: &robotd_params::CameraIntrinsics, + width: u32, + height: u32, + source: Source, + ) -> Option { + if measured.width == 0 || measured.height == 0 || width == 0 || height == 0 { + return None; + } + let scale_x = f64::from(width) / f64::from(measured.width); + let scale_y = f64::from(height) / f64::from(measured.height); + if (scale_x - scale_y).abs() > 0.01 { + tracing::warn!( + measured = format!("{}x{}", measured.width, measured.height), + delivered = format!("{width}x{height}"), + "the calibration was measured at a different aspect ratio than the stream, so it \ + cannot be scaled to it; publishing nominal intrinsics instead" + ); + return None; + } + Some(Self { + fx: measured.fx * scale_x, + fy: measured.fy * scale_y, + cx: measured.cx * scale_x, + cy: measured.cy * scale_y, + calibrated: true, + source, + // Distortion coefficients are dimensionless in normalised image coordinates, so a + // uniform resize leaves them alone. + distortion: measured.distortion.clone(), + }) + } + + /// The hardware family's calibration, scaled to the delivered frame. The camera and lens are + /// one part across a revision, so this is a real solve of the same optics — just not of this + /// particular unit, which is why it is published as `Source::Family` rather than `Robot`. + /// + /// Gated on a known sensor mode for the same reason [`Intrinsics::nominal`] is: the solve was + /// taken in the pinned mode, and an unconfirmed one (the boot mode — the same ~62° field, but a + /// different 4:3→16:9 framing and principal point) would place it slightly wrong — so an + /// unconfirmed mode publishes nothing rather than the family's numbers off by that framing. + pub fn family(mode: Option, width: u32, height: u32) -> Option { + mode?; + Self::scaled( + &robotd_params::CameraIntrinsics::alpha(), + width, + height, + Source::Family, + ) + } + + /// The MuJoCo twin's head camera. The MJCF sets no `fovy` on the camera, so MuJoCo's default + /// **45° VERTICAL** field applies over the rendered height: `fx = fy = (height/2)/tan(45°/2)`, + /// principal point central, no distortion (a rendered pinhole has none). Exact for the simulator, + /// so twin recordings self-describe and need no `--calib`. Tagged [`Source::Sim`], `calibrated` + /// false (it is not a measurement of a physical sensor). If a scene ever sets a custom camera + /// `fovy`, update `SIM_VFOV_DEG`. + pub fn sim(width: u32, height: u32) -> Option { + if width == 0 || height == 0 { + return None; + } + let focal = f64::from(height) / 2.0 / (SIM_VFOV_DEG / 2.0).to_radians().tan(); + Some(Self { + fx: focal, + fy: focal, + cx: f64::from(width) / 2.0, + cy: f64::from(height) / 2.0, + calibrated: false, + source: Source::Sim, + distortion: Vec::new(), + }) + } + + /// What to publish, in order of preference: this robot's own `[media.intrinsics]` calibration; + /// else the hardware family's, which every unit shares; else the module's design figures. Each + /// carries its [`Source`], so "calibrated" never has to stand in for "measured on *this* robot". + pub fn published( + configured: Option<&robotd_params::CameraIntrinsics>, + mode: Option, + width: u32, + height: u32, + ) -> Option { + configured + .and_then(|measured| Self::scaled_from(measured, width, height)) + .or_else(|| Self::family(mode, width, height)) + .or_else(|| Self::nominal(mode, width, height)) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn measured(width: u32, height: u32) -> robotd_params::CameraIntrinsics { + robotd_params::CameraIntrinsics { + width, + height, + fx: 1800.0, + fy: 1802.0, + cx: 646.0, + cy: 358.0, + distortion: vec![-0.31, 0.12, 0.0, 0.0, -0.02], + } + } + + /// The one physical number, and the focal length it produces. + #[test] + fn nominal_focal_length_comes_from_the_field_of_view() { + // 62° across a 1280-wide frame. Written out so a wrong constant is a failing test rather + // than a plausible-looking number in an SDP nobody checks. + assert!( + (nominal_focal_px(1280) - 1065.14).abs() < 0.1, + "{}", + nominal_focal_px(1280) + ); + // And it implies the 62° it was built from. + let hfov = 2.0 * (640.0 / nominal_focal_px(1280)).atan().to_degrees(); + assert!((hfov - 62.0).abs() < 0.01, "{hfov}"); + } + + /// 1280x720 out of the pinned 1920x1080 mode: the 62° field at that width. + #[test] + fn the_pinned_mode_scales_to_what_is_streamed() { + let at_720p = Intrinsics::nominal(Some(SensorMode::PINNED), 1280, 720).expect("known"); + assert!((at_720p.fx - 1065.14).abs() < 0.1, "{}", at_720p.fx); + assert_eq!(at_720p.fy, at_720p.fx, "square pixels, uniform scale"); + assert_eq!((at_720p.cx, at_720p.cy), (640.0, 360.0)); + assert!( + !at_720p.calibrated, + "these are the module's, not this robot's" + ); + assert!( + at_720p.distortion.is_empty(), + "nominal models no distortion" + ); + + // The field of view is the same at the mode's own resolution — only the pixel count grows. + let at_1080p = Intrinsics::nominal(Some(SensorMode::PINNED), 1920, 1080).expect("known"); + assert!((at_1080p.fx - nominal_focal_px(1920)).abs() < 0.01); + let hfov = 2.0 * (960.0 / at_1080p.fx).atan().to_degrees(); + assert!((hfov - 62.0).abs() < 0.01, "{hfov}"); + } + + /// The twin's 45° vertical FOV over a 640x360 render — a wider (~72.7°) horizontal field than the + /// real camera's 62°, which is why the nominal K skews twin maps. + #[test] + fn sim_geometry_from_the_default_fovy() { + let k = Intrinsics::sim(640, 360).expect("nonzero"); + assert!((k.fy - 434.57).abs() < 0.1, "{}", k.fy); + assert_eq!(k.fx, k.fy, "square pixels"); + assert_eq!((k.cx, k.cy), (320.0, 180.0), "principal point central"); + assert_eq!(k.source, Source::Sim); + assert!(!k.calibrated && k.distortion.is_empty()); + let hfov = 2.0 * (320.0 / k.fx).atan().to_degrees(); + assert!((hfov - 72.7).abs() < 0.5, "{hfov}"); + } + + /// The delivered field of view is the sensor's full ~62° at every resolution — the production + /// 1920x1080 mode is a scaled full-frame readout, not a crop, so the FOV does not change. + #[test] + fn the_delivered_field_of_view_is_the_full_62_degrees() { + for w in [640_u32, 1280, 1920] { + let h = w * 9 / 16; + let k = Intrinsics::nominal(Some(SensorMode::PINNED), w, h).expect("known"); + let hfov = 2.0 * (f64::from(w) / 2.0 / k.fx).atan().to_degrees(); + assert!((hfov - 62.0).abs() < 0.5, "{w}w -> {hfov}"); + } + } + + /// **A sensor whose pinned mode we could not confirm publishes nothing.** + /// + /// `pin_sensor_mode` can fail, leaving the sensor in its 3280x2464 boot mode — also ~62°, but a + /// different 4:3→16:9 framing than the pinned mode the calibration is for. Numbers that are + /// quietly wrong are worse than none — a consumer told nothing calibrates. + #[test] + fn an_unknown_sensor_mode_yields_no_intrinsics() { + assert!(Intrinsics::nominal(None, 1280, 720).is_none()); + assert!(Intrinsics::published(None, None, 1280, 720).is_none()); + } + + /// A frame whose aspect ratio is not the mode's has been cropped or squashed on the way out, + /// and which of those cannot be told from the size. + #[test] + fn a_changed_aspect_ratio_is_refused_rather_than_guessed() { + assert!( + Intrinsics::nominal(Some(SensorMode::PINNED), 640, 480).is_none(), + "4:3 out of a 16:9 mode is not a resize" + ); + assert!(Intrinsics::nominal(Some(SensorMode::PINNED), 1280, 0).is_none()); + } + + /// A calibration wins, and scales. + #[test] + fn a_calibration_is_preferred_and_carried_to_the_streamed_size() { + // Measured at 1280x720, delivered at 640x360: everything halves. + let published = Intrinsics::published( + Some(&measured(1280, 720)), + Some(SensorMode::PINNED), + 640, + 360, + ) + .expect("a calibration"); + assert!(published.calibrated); + assert!((published.fx - 900.0).abs() < 0.01); + assert!((published.cx - 323.0).abs() < 0.01); + assert_eq!( + published.distortion, + vec![-0.31, 0.12, 0.0, 0.0, -0.02], + "distortion is dimensionless, so a resize leaves it alone" + ); + + // And at the resolution it was measured at, it is published as measured. + let same = Intrinsics::published(Some(&measured(1280, 720)), None, 1280, 720) + .expect("as measured"); + assert_eq!( + (same.fx, same.fy, same.cx, same.cy), + (1800.0, 1802.0, 646.0, 358.0) + ); + } + + /// A per-robot calibration that cannot be scaled falls back to the family's, not to nothing — + /// a real solve of the same optics, published as `family` so a consumer can see this robot's + /// own record was unusable and someone should fix it. + #[test] + fn an_unusable_robot_calibration_falls_back_to_the_family() { + let published = Intrinsics::published( + Some(&measured(640, 480)), + Some(SensorMode::PINNED), + 1280, + 720, + ) + .expect("the family calibration"); + assert_eq!(published.source, Source::Family); + assert!( + published.calibrated, + "the family solve is a real measurement" + ); + // The alpha solve is 1280x720, delivered 1280x720, so it is carried across unscaled. + assert!((published.fx - 1061.81).abs() < 0.01, "{}", published.fx); + } + + /// With no per-robot table, a robot in a known mode publishes the family's calibration, tagged + /// so a consumer can tell it is not this unit's own solve. + #[test] + fn the_family_fills_in_when_the_robot_has_no_table() { + let published = Intrinsics::published(None, Some(SensorMode::PINNED), 640, 360) + .expect("the family calibration"); + assert_eq!(published.source, Source::Family); + assert!(published.calibrated); + // Half of the 1280x720 solve. + assert!((published.fx - 530.9).abs() < 0.5, "{}", published.fx); + let json = serde_json::to_value(&published).unwrap(); + assert_eq!(json["source"], "family"); + assert_eq!(json["calibrated"], true); + } + + /// The shape a consumer reads. `calibrated` is not optional in the JSON: a consumer that has + /// to guess whether numbers were measured will guess that they were. + #[test] + fn the_wire_shape_names_what_it_is() { + let json = + serde_json::to_value(Intrinsics::nominal(Some(SensorMode::PINNED), 1280, 720).unwrap()) + .unwrap(); + assert_eq!(json["calibrated"], false); + assert_eq!(json["source"], "nominal"); + assert!((json["fx"].as_f64().unwrap() - 1065.14).abs() < 0.1); + assert_eq!(json["cx"], 640.0); + assert!( + json.get("distortion").is_none(), + "an empty distortion model is absent rather than an empty list: a consumer reading \ + `[]` has to know that means `unknown` rather than `none`" + ); + } +} diff --git a/mediad/src/config.rs b/mediad/src/config.rs index ff38b936..a029a115 100644 --- a/mediad/src/config.rs +++ b/mediad/src/config.rs @@ -1,7 +1,7 @@ //! What this daemon streams and what it looks for, out of the config file `robotd` already reads. //! //! `[media]` in `/etc/robot/robotd.toml` — camera or test pattern, frame size, rate, bitrate — and -//! `[detect]` beside it, which is this daemon's too because the frames are on this daemon's tee. +//! `[duck_detector]` beside it, which is this daemon's too because the frames are on this daemon's tee. //! The schema, the defaults and the validation are `robotd_params`'s, which is the point: the crate //! read here is the one `robotctl configure` writes through, so the editor cannot offer a value //! this daemon would not understand. @@ -45,7 +45,7 @@ pub fn load(path: &Path, explicit: bool) -> Params { #[cfg(test)] mod tests { use super::*; - use robotd_params::MediaParams; + use robotd_params::{MediaParams, MediaSource}; fn write(dir: &Path, text: &str) -> PathBuf { let path = dir.join("robotd.toml"); @@ -62,7 +62,11 @@ mod tests { assert_eq!(media.quality.size(), (640, 360)); assert_eq!(media.quality.fps(), 30); assert_eq!(media.bitrate_resolved(), media.quality.default_bitrate()); - assert!(media.camera, "untouched keys keep their defaults"); + assert_eq!( + media.source, + MediaSource::Camera, + "untouched keys keep their defaults" + ); } /// A robot with no file at the default path streams its camera, and says nothing about it. @@ -71,7 +75,7 @@ mod tests { let dir = tempfile::tempdir().unwrap(); let media = load(&dir.path().join("absent.toml"), false).media; assert_eq!(media.quality, MediaParams::default().quality); - assert!(media.camera); + assert_eq!(media.source, MediaSource::Camera); } /// The claim the doc comment makes, pinned: a params file `robotd` will not start on still @@ -82,7 +86,7 @@ mod tests { let path = write(dir.path(), "[media\nquality = "); let media = load(&path, true).media; assert_eq!(media.quality, MediaParams::default().quality); - assert!(media.camera); + assert_eq!(media.source, MediaSource::Camera); } /// A `[media]` section from a build that had a key this one does not is ignored key by key, diff --git a/mediad/src/detect.rs b/mediad/src/detect.rs index 2320d86b..db990b74 100644 --- a/mediad/src/detect.rs +++ b/mediad/src/detect.rs @@ -139,7 +139,9 @@ pub fn spawn_first( } } anyhow::bail!( - "no model would load ({}). For the NPU: sudo /usr/local/sbin/robot-setup-npu", + "no model would load ({}). A missing file means the set was never installed — \ + `sudo robotctl duck-detector update` fetches it from the Hub; for the NPU: \ + sudo /usr/local/sbin/robot-setup-npu", refused.join("; ") ) } diff --git a/mediad/src/exposure.rs b/mediad/src/exposure.rs index 7264d588..9860ccc5 100644 --- a/mediad/src/exposure.rs +++ b/mediad/src/exposure.rs @@ -424,6 +424,7 @@ mod tests { width: 1280, height: 720, format: CAPTURE_FORMAT, + captured_at: std::time::SystemTime::UNIX_EPOCH, data, } } diff --git a/mediad/src/frame.rs b/mediad/src/frame.rs new file mode 100644 index 00000000..3e80376e --- /dev/null +++ b/mediad/src/frame.rs @@ -0,0 +1,501 @@ +//! A local `media.frame` endpoint for a recorder or perception process on the robot. +//! +//! A frame stays out of the WebRTC control channel: at the default geometry the UYVY payload is +//! about 1.8 MiB, so JSON/base64 would make a control request several MiB and let a slow peer tie +//! camera data to the network. This socket sends one JSON-RPC response header, then precisely +//! `bytes` raw bytes, which keeps the metadata inspectable without copying pixels through a text +//! encoding. +//! +//! **It asks for a frame rather than taking the last one.** [`Frames`] is a rendezvous, not a +//! cache: the capture branch copies a buffer only when a reader has asked for one +//! ([`crate::pipeline::Frames`] explains why — 1.84 MiB thirty times a second for readers that +//! want two). So a caller here waits for the capture that answers it, bounded by +//! [`pipeline::FRAME_TIMEOUT`], and a camera that has stopped is reported as a timeout rather than +//! answered with the frame it stopped on. +//! +//! The socket is group-readable like the other observation sockets: whoever may watch +//! `robot.state` may ask for a picture. + +use std::os::unix::fs::PermissionsExt; +use std::path::Path; +use std::sync::Arc; +use std::time::{Duration, UNIX_EPOCH}; + +use anyhow::{Context, Result}; +use duck_ipc_proto as proto; + +use tokio::io::{AsyncBufReadExt, AsyncReadExt, AsyncWriteExt, BufReader}; +use tokio::net::{UnixListener, UnixStream}; + +use crate::pipeline::Frames; + +const SOCKET_MODE: u32 = 0o660; + +/// The group that may ask for a frame. Deliberately the same one as `robotd`'s socket, `padd`'s +/// tap and `tof`'s stream: whoever may watch the robot may watch what it sees. +const GROUP: &str = "robot"; + +/// The longest request this endpoint will read. `media.frame` takes no parameters worth naming, so +/// anything approaching this is a client that has lost the plot. +const MAX_REQUEST_BYTES: usize = 4096; + +/// Claim the socket without replacing a live listener or a non-socket file. +pub async fn bind(socket: &Path) -> Result<(std::fs::File, UnixListener)> { + use std::os::unix::fs::FileTypeExt; + if let Some(parent) = socket.parent().filter(|p| !p.as_os_str().is_empty()) { + std::fs::create_dir_all(parent)?; + } + // Keep the lock inode for the listener's lifetime, including the stale-socket probe. + let mut lock_path = socket.as_os_str().to_os_string(); + lock_path.push(".lock"); + let lock = std::fs::OpenOptions::new() + .create(true) + .truncate(false) + .read(true) + .write(true) + .open(lock_path)?; + lock.try_lock() + .context("another mediad owns the frame socket")?; + let listener = match UnixListener::bind(socket) { + Ok(listener) => listener, + Err(error) if error.kind() == std::io::ErrorKind::AddrInUse => { + if !std::fs::symlink_metadata(socket)?.file_type().is_socket() { + return Err(error.into()); + } + match tokio::time::timeout(Duration::from_secs(1), UnixStream::connect(socket)).await { + Ok(Err(probe)) if probe.kind() == std::io::ErrorKind::ConnectionRefused => { + std::fs::remove_file(socket)?; + UnixListener::bind(socket)? + } + _ => return Err(error.into()), + } + } + Err(error) => return Err(error.into()), + }; + std::fs::set_permissions(socket, std::fs::Permissions::from_mode(SOCKET_MODE)) + .with_context(|| format!("setting permissions on {}", socket.display()))?; + if let Err(error) = give_to_group(socket, GROUP) { + // Not fatal, and said out loud with what it means: the socket exists, and only `mediad` + // and root can reach it. On a board that is a broken install; on a laptop it is a machine + // with no `robot` group, which is ordinary. + tracing::warn!( + error = %error, group = GROUP, socket = %socket.display(), + "media.frame stays private to mediad — nothing else can ask for a picture" + ); + } + tracing::info!( + path = %socket.display(), + mode = format!("{SOCKET_MODE:o}"), + "serving media.frame locally" + ); + + Ok((lock, listener)) +} + +/// Bound both simultaneous clients and the lifetime of silent or slow clients. +/// +/// `rotate` is degrees clockwise the camera is mounted from upright, reported in every frame +/// header. It is the caller's business whether the pipeline already applied it. +pub async fn serve(listener: UnixListener, frames: Frames, rotate: u32) -> Result<()> { + let slots = Arc::new(tokio::sync::Semaphore::new(16)); + loop { + match listener.accept().await { + Ok((stream, _)) => { + let Ok(permit) = slots.clone().try_acquire_owned() else { + continue; + }; + let frames = frames.clone(); + tokio::spawn(async move { + let _permit = permit; + let _ = tokio::time::timeout( + Duration::from_secs(5), + handle(stream, frames, rotate), + ) + .await; + }); + } + Err(error) => tracing::warn!(error = %error, "media.frame accept failed"), + } + } +} + +async fn handle(stream: UnixStream, frames: Frames, rotate: u32) -> Result<()> { + let (read, mut write) = stream.into_split(); + // Bounded *before* the line is buffered. Checking the length afterwards would mean a client + // could make this process hold an arbitrarily long line first, which is the thing the cap is + // for. One byte over the cap is read so that "too large" stays distinguishable from a request + // that exactly fills it. + let mut reader = BufReader::new(read); + loop { + let mut line = Vec::new(); + let read = (&mut reader) + .take(MAX_REQUEST_BYTES as u64 + 1) + .read_until(b'\n', &mut line) + .await?; + if read == 0 { + return Ok(()); + } + if line.len() > MAX_REQUEST_BYTES { + write_response( + &mut write, + proto::Response::err( + None, + proto::Error::new(proto::code::INVALID_PARAMS, "request is too large"), + ), + ) + .await?; + return Ok(()); + } + let request: proto::Request = match serde_json::from_slice(&line) { + Ok(request) => request, + Err(error) => { + write_response( + &mut write, + proto::Response::err( + None, + proto::Error::new(proto::code::PARSE_ERROR, error.to_string()), + ), + ) + .await?; + return Ok(()); + } + }; + if request.method == proto::method::HELLO { + write_response( + &mut write, + proto::Response::ok( + request.id, + &proto::HelloResult { + api_version: proto::API_VERSION, + daemon_version: proto::semver::Version::parse(env!("CARGO_PKG_VERSION")) + .ok(), + revision: proto::build_info!().revision.map(str::to_owned), + }, + ), + ) + .await?; + continue; + } + if request.method != proto::method::MEDIA_FRAME { + write_response( + &mut write, + proto::Response::err( + request.id, + proto::Error::new( + proto::code::METHOD_NOT_FOUND, + format!("{} is not served by mediad", request.method), + ), + ), + ) + .await?; + continue; + } + // `next_frame` registers the demand and parks on a condvar until the capture that answers it + // lands, so it cannot run on the runtime's thread. + let frame = tokio::task::spawn_blocking(move || frames.next_frame()).await?; + let Some(frame) = frame else { + write_response( + &mut write, + proto::Response::err( + request.id, + proto::Error::new( + proto::code::INTERNAL_ERROR, + "no frame arrived within the capture timeout", + ), + ), + ) + .await?; + return Ok(()); + }; + let captured_at_unix_us = frame + .captured_at + .duration_since(UNIX_EPOCH) + .unwrap_or_default() + .as_micros(); + let header = proto::MediaFrameHeader { + width: frame.width, + height: frame.height, + format: frame.format.to_owned(), + bytes: frame.data.len(), + captured_at_unix_us, + rotate, + }; + write_response(&mut write, proto::Response::ok(request.id, &header)).await?; + write.write_all(&frame.data).await?; + write.flush().await?; + return Ok(()); + } +} + +/// Hand the socket to `GROUP`. Mirrors `tof`'s stream and `padd`'s tap, including that a missing +/// group is a warning rather than a failure. +fn give_to_group(socket: &Path, group: &str) -> std::io::Result<()> { + use std::ffi::CString; + use std::os::unix::ffi::OsStrExt; + + let name = CString::new(group).map_err(std::io::Error::other)?; + // SAFETY: `getgrnam` reads the group database and returns a pointer into storage it owns. The + // name is a valid C string for the length of the call, and nothing else in this process calls + // into the group database. + let entry = unsafe { libc::getgrnam(name.as_ptr()) }; + if entry.is_null() { + return Err(std::io::Error::other(format!( + "no {group} group on this system" + ))); + } + // SAFETY: checked non-null immediately above, and `struct group` is fully initialised by + // `getgrnam` when it returns a pointer at all. + let gid = unsafe { (*entry).gr_gid }; + + let path = CString::new(socket.as_os_str().as_bytes()).map_err(std::io::Error::other)?; + // SAFETY: a valid C string path; `-1` for the owner is the documented "leave it alone". + if unsafe { libc::chown(path.as_ptr(), u32::MAX, gid) } != 0 { + return Err(std::io::Error::last_os_error()); + } + Ok(()) +} + +async fn write_response( + write: &mut tokio::net::unix::OwnedWriteHalf, + response: proto::Response, +) -> Result<()> { + let mut line = serde_json::to_vec(&response)?; + line.push(b'\n'); + write.write_all(&line).await?; + write.flush().await?; + Ok(()) +} + +#[cfg(test)] +mod tests { + use std::time::{Duration, UNIX_EPOCH}; + + use super::*; + use crate::pipeline::Frame; + use tokio::io::AsyncReadExt; + + #[tokio::test] + async fn hello_and_unknown_method_keep_the_connection_for_a_frame() { + let frames = Frames::default(); + let producer = answer_once( + frames.clone(), + Frame { + width: 2, + height: 1, + format: "UYVY", + captured_at: UNIX_EPOCH, + data: vec![128, 16, 128, 235], + }, + ); + let (mut client, server) = UnixStream::pair().unwrap(); + let task = tokio::spawn(handle(server, frames, 90)); + client.write_all(b"{\"jsonrpc\":\"2.0\",\"id\":1,\"method\":\"hello\"}\n{\"jsonrpc\":\"2.0\",\"id\":2,\"method\":\"unknown\"}\n{\"jsonrpc\":\"2.0\",\"id\":3,\"method\":\"media.frame\"}\n").await.unwrap(); + let mut reader = BufReader::new(client); + for id in 1..=3 { + let mut line = String::new(); + reader.read_line(&mut line).await.unwrap(); + let response: proto::Response = serde_json::from_str(&line).unwrap(); + assert_eq!(response.id, Some(proto::Id::Number(id))); + if id == 1 { + assert_eq!(response.result.unwrap()["api_version"], proto::API_VERSION); + } + if id == 2 { + assert_eq!(response.error.unwrap().code, proto::code::METHOD_NOT_FOUND); + } + } + let mut bytes = Vec::new(); + reader.read_to_end(&mut bytes).await.unwrap(); + assert_eq!(bytes, [128, 16, 128, 235]); + task.await.unwrap().unwrap(); + producer.join().unwrap(); + } + + #[tokio::test] + async fn invalid_utf8_gets_a_parse_error() { + let (mut client, server) = UnixStream::pair().unwrap(); + let task = tokio::spawn(handle(server, Frames::default(), 90)); + client.write_all(&[255, b'\n']).await.unwrap(); + let mut line = String::new(); + BufReader::new(client).read_line(&mut line).await.unwrap(); + let response: proto::Response = serde_json::from_str(&line).unwrap(); + assert_eq!(response.error.unwrap().code, proto::code::PARSE_ERROR); + task.await.unwrap().unwrap(); + } + + #[tokio::test] + async fn socket_claim_preserves_files_and_live_listeners_but_recovers_stale_sockets() { + let dir = tempfile::tempdir().unwrap(); + let regular = dir.path().join("regular"); + std::fs::write(®ular, b"keep").unwrap(); + assert!(bind(®ular).await.is_err()); + assert_eq!(std::fs::read(®ular).unwrap(), b"keep"); + let live = dir.path().join("live"); + let listener = UnixListener::bind(&live).unwrap(); + assert!(bind(&live).await.is_err()); + assert!(UnixStream::connect(&live).await.is_ok()); + drop(listener); + let (lock, listener) = bind(&live).await.unwrap(); + assert!(bind(&live).await.is_err()); + drop(listener); + drop(lock); + assert!(bind(&live).await.is_ok()); + } + + #[tokio::test] + async fn idle_clients_are_bounded_and_expire() { + let dir = tempfile::tempdir().unwrap(); + let socket = dir.path().join("media.sock"); + let listener = UnixListener::bind(&socket).unwrap(); + let server = tokio::spawn(serve(listener, Frames::default(), 90)); + let mut clients = Vec::new(); + for _ in 0..16 { + let mut client = UnixStream::connect(&socket).await.unwrap(); + client + .write_all(b"{\"jsonrpc\":\"2.0\",\"id\":1,\"method\":\"hello\"}\n") + .await + .unwrap(); + let mut reader = BufReader::new(client); + let mut line = String::new(); + reader.read_line(&mut line).await.unwrap(); + assert!(!line.is_empty()); + clients.push(reader); + } + let mut excess = UnixStream::connect(&socket).await.unwrap(); + let mut byte = [0]; + assert_eq!( + tokio::time::timeout(Duration::from_secs(1), excess.read(&mut byte)) + .await + .unwrap() + .unwrap(), + 0 + ); + assert_eq!( + tokio::time::timeout(Duration::from_secs(6), clients[0].read(&mut byte)) + .await + .unwrap() + .unwrap(), + 0 + ); + server.abort(); + } + + /// Stand in for the capture branch: wait for the demand this endpoint registers, then answer + /// it once. Mirrors what `wire_frames` does on a buffer somebody asked for. + fn answer_once(frames: Frames, frame: Frame) -> std::thread::JoinHandle<()> { + std::thread::spawn(move || { + let deadline = std::time::Instant::now() + Duration::from_secs(5); + while !frames.take_request() { + assert!( + std::time::Instant::now() < deadline, + "the endpoint never asked for a frame" + ); + std::thread::yield_now(); + } + frames.deliver(frame); + }) + } + + async fn reply(frames: Frames, request: &str) -> proto::Response { + let (mut client, server) = UnixStream::pair().unwrap(); + let task = tokio::spawn(handle(server, frames, 90)); + client.write_all(request.as_bytes()).await.unwrap(); + client.shutdown().await.unwrap(); + let mut text = String::new(); + BufReader::new(client).read_line(&mut text).await.unwrap(); + task.await.unwrap().unwrap(); + serde_json::from_str(text.trim()).unwrap() + } + + /// A camera that never delivers is a timeout, not a silent hang and not a stale frame. + #[tokio::test] + async fn a_capture_that_never_comes_is_an_explicit_error() { + let response = reply( + Frames::default(), + "{\"jsonrpc\":\"2.0\",\"id\":7,\"method\":\"media.frame\",\"params\":{}}\n", + ) + .await; + assert_eq!(response.id, Some(proto::Id::Number(7))); + assert_eq!(response.error.unwrap().code, proto::code::INTERNAL_ERROR); + } + + /// Refused before any demand is registered, so an unknown method cannot make the capture + /// branch copy 1.8 MiB for nothing. + #[tokio::test] + async fn an_unknown_method_is_refused_without_asking_for_a_frame() { + let frames = Frames::default(); + let response = reply( + frames.clone(), + "{\"jsonrpc\":\"2.0\",\"id\":\"request\",\"method\":\"media.other\"}\n", + ) + .await; + assert_eq!(response.id, Some(proto::Id::Text("request".into()))); + assert_eq!(response.error.unwrap().code, proto::code::METHOD_NOT_FOUND); + assert!( + !frames.take_request(), + "a refused method must not leave demand behind" + ); + } + + #[tokio::test] + async fn an_oversized_request_is_rejected_before_it_is_parsed() { + let request = format!("{}\n", "x".repeat(MAX_REQUEST_BYTES + 1)); + let response = reply(Frames::default(), &request).await; + assert_eq!(response.id, None); + assert_eq!(response.error.unwrap().code, proto::code::INVALID_PARAMS); + } + + /// The read is bounded before the line is buffered, so a client that never sends a newline + /// cannot make this process hold an unbounded string. + #[tokio::test] + async fn a_request_without_a_newline_is_still_bounded() { + let (mut client, server) = UnixStream::pair().unwrap(); + let task = tokio::spawn(handle(server, Frames::default(), 90)); + client + .write_all("y".repeat(MAX_REQUEST_BYTES * 4).as_bytes()) + .await + .unwrap(); + client.shutdown().await.unwrap(); + let mut text = String::new(); + BufReader::new(client).read_line(&mut text).await.unwrap(); + task.await.unwrap().unwrap(); + let response: proto::Response = serde_json::from_str(text.trim()).unwrap(); + assert_eq!(response.error.unwrap().code, proto::code::INVALID_PARAMS); + } + + #[tokio::test] + async fn a_frame_reply_names_and_follows_with_exactly_its_pixels() { + let frames = Frames::default(); + let producer = answer_once( + frames.clone(), + Frame { + width: 2, + height: 1, + format: "UYVY", + captured_at: UNIX_EPOCH + Duration::from_secs(1), + data: vec![128, 32, 128, 64], + }, + ); + let (mut client, server) = UnixStream::pair().unwrap(); + let task = tokio::spawn(handle(server, frames, 90)); + client + .write_all(b"{\"jsonrpc\":\"2.0\",\"id\":1,\"method\":\"media.frame\"}\n") + .await + .unwrap(); + let mut reader = BufReader::new(client); + let mut line = String::new(); + reader.read_line(&mut line).await.unwrap(); + let response: proto::Response = serde_json::from_str(line.trim()).unwrap(); + let result = response.result.unwrap(); + assert_eq!(result["width"], 2); + assert_eq!(result["height"], 1); + assert_eq!(result["format"], "UYVY"); + assert_eq!(result["bytes"], 4); + assert_eq!(result["captured_at_unix_us"], 1_000_000); + assert_eq!(result["rotate"], 90); + let mut pixels = [0; 4]; + reader.read_exact(&mut pixels).await.unwrap(); + assert_eq!(pixels, [128, 32, 128, 64]); + task.await.unwrap().unwrap(); + producer.join().unwrap(); + } +} diff --git a/mediad/src/lib.rs b/mediad/src/lib.rs index cf7ee95a..fbb8bd02 100644 --- a/mediad/src/lib.rs +++ b/mediad/src/lib.rs @@ -17,10 +17,22 @@ //! signalling server in this process, `mpph264enc` in front of it, and a `control` datachannel per //! peer wired to [`session::run`]. +/// What the camera's geometry is — the intrinsics a consumer needs to turn pixels into +/// directions, and which sensor mode they belong to. +pub mod camera; pub mod config; +/// The account credential `updaterd` writes, read by the two things here that need it. pub mod producer; +/// The outward connection to the rendezvous service — what makes a duck reachable from off its +/// own LAN. `docs/design/remote-access-design.md` §3. +pub mod relay; pub mod route; pub mod session; +mod snapshot; +pub mod stream; +/// Relay candidates, so a robot behind a router is reachable from a network that cannot punch a +/// hole to it. `docs/design/remote-access-design.md` §6. +pub mod turn; pub mod upstream; pub mod web; @@ -38,3 +50,9 @@ pub mod exposure; /// reads the same raw branch, in the same pixel format the pipeline names. #[cfg(target_os = "linux")] pub mod detect; + +/// The local, on-demand raw-frame endpoint. Linux only because it reads the pipeline's frame +/// rendezvous, which is; the WebRTC control channel deliberately does not carry camera-sized +/// replies. `npu-bringup.md` §"media.frame". +#[cfg(target_os = "linux")] +pub mod frame; diff --git a/mediad/src/main.rs b/mediad/src/main.rs index 6289656d..9855a29c 100644 --- a/mediad/src/main.rs +++ b/mediad/src/main.rs @@ -37,6 +37,39 @@ struct Args { #[arg(long, default_value_t = 8443)] port: u32, + /// The rendezvous service this robot registers with, so it can be reached from off its LAN. + /// + /// Defaults to the Space the mini's fleet uses. A flag rather than a config key because there + /// is nothing to choose on a real robot — what it is for is pointing a board at a fake, or at + /// a self-hosted copy on the day somebody wants one. `docs/design/remote-access-design.md` §4. + #[arg(long, default_value = mediad::relay::DEFAULT_RENDEZVOUS)] + rendezvous_url: String, + + /// The account credential `updaterd` writes, which the relay needs to prove whose robot this + /// is. Absent means nobody has signed this robot in, and remote access is simply off. + #[arg(long, default_value = mediad::relay::DEFAULT_TOKEN_PATH)] + token: PathBuf, + + /// Where short-lived TURN credentials come from. + /// + /// Hugging Face hosts this proxy and mints Cloudflare credentials for the account the token + /// belongs to, which is why offering a relay needs no new secret on the robot. A flag for + /// pointing a board at a fake; there is nothing to choose on a real one. + /// + /// Checked here rather than trusted: this is the one URL the account token is sent to, and + /// `parse_endpoint` says what it will not send it over. + #[arg(long, default_value = mediad::turn::DEFAULT_TURN_ENDPOINT, + value_parser = mediad::turn::parse_endpoint)] + turn_url: String, + + /// Do not register with the rendezvous service, whatever the token file says. + /// + /// For a board that is signed in and being worked on: a duck registering from a bench while + /// somebody drives the same account's robot elsewhere is a producer in a list nobody wants, + /// and evicting it means finding this flag afterwards. + #[arg(long)] + no_remote: bool, + /// Where the console is served. `http://:8080/`, and nothing else to run. /// /// **Two ports, and only this one is ever typed.** `webrtcsink` owns the listener on `--port` @@ -75,12 +108,15 @@ struct Args { /// How far the camera is mounted from upright, clockwise: 0, 90, 180 or 270. /// - /// **90, because the head camera is mounted a quarter turn off**, and this is the one place that - /// fact is written down. It no longer means "rotate the pixels": it is told to whoever displays - /// the video, and they rotate for free — the console with a CSS transform on the GPU. Rotating - /// here cost 145% of a core and 22 fps; `pipeline::Rotation` has the numbers. - #[arg(long, default_value_t = 90)] - rotate: u32, + /// **90 by default, because the head camera is mounted a quarter turn off**, and this is the one + /// place that fact is written down. It no longer means "rotate the pixels": it is told to + /// whoever displays the video, and they rotate for free — the console with a CSS transform on + /// the GPU. Rotating here cost 145% of a core and 22 fps; `pipeline::Rotation` has the numbers. + /// + /// True of a simulated camera too: the one in MuJoCo is rolled to match the mount, so a frame + /// from a duck in the twin needs the same quarter turn as a frame from a duck on the desk. + #[arg(long)] + rotate: Option, /// Leave the exposure where `--exposure` and `--analogue-gain` put it, instead of metering. /// @@ -93,6 +129,17 @@ struct Args { #[arg(long)] no_auto_exposure: bool, + /// Take frames from a duck in MuJoCo at `host:port` instead of a camera. + /// + /// The geometry has to match the simulator's camera — set `[media] quality` (the rung `mediad` + /// streams; `mediad` has no `--width`/`--height` of its own) to the resolution and rate the body + /// renders at — because the frames arrive raw and length-prefixed with no handshake, and a + /// mismatch is a picture nobody can read rather than an error the pipeline can recover from. + /// `mediad` says so and refuses the frame if the sizes disagree. + /// Takes precedence over `[media] source`, which is a fact about a robot and not about this. + #[arg(long)] + sim_camera: Option, + /// Rotate in the pipeline as well, so the *encoded stream* comes out upright. /// /// **Off by default because it is expensive in a way that does not look like rotation.** It @@ -101,6 +148,51 @@ struct Args { /// Worth it only for a consumer that cannot rotate for itself. #[arg(long)] flip_in_pipeline: bool, + + /// Where the daemons listen, when not at `proto::socket`'s paths. + /// + /// On a robot the defaults are right and none of these is ever typed. They exist for the twin + /// (`scripts/duck-sim`), where every duck's `robotd` and `tofd` listen under a per-duck state + /// directory — without them a peer's `control` channel reaches a `mediad` whose routes all end + /// at `/run/*.sock`, and every call but `media.video` answers "not answering". + #[arg(long)] + robot_socket: Option, + #[arg(long)] + tof_socket: Option, + #[arg(long)] + config_socket: Option, + #[arg(long)] + pad_socket: Option, + #[arg(long)] + updater_socket: Option, + /// Local raw camera snapshot endpoint (not the control datachannel). + #[arg(long, default_value = duck_ipc_proto::socket::MEDIA)] + frame_socket: std::path::PathBuf, +} + +// Gated with the `main` that calls it: off Linux there is no pipeline, so there is nothing to +// point at a socket and `-D warnings` would call this dead. +#[cfg(target_os = "linux")] +impl Args { + fn sockets(&self) -> mediad::upstream::Sockets { + let mut s = mediad::upstream::Sockets::default(); + if let Some(p) = &self.robot_socket { + s.robot = p.clone(); + } + if let Some(p) = &self.tof_socket { + s.tof = p.clone(); + } + if let Some(p) = &self.config_socket { + s.config = p.clone(); + } + if let Some(p) = &self.pad_socket { + s.pad = p.clone(); + } + if let Some(p) = &self.updater_socket { + s.updater = p.clone(); + } + s + } } #[cfg(target_os = "linux")] @@ -130,15 +222,20 @@ fn main() -> ExitCode { // should say so rather than opening a camera first. // Validated even when the pipeline will not use it, because it is still what every consumer is // told about the mount — a typo should not reach the console as a rotation nobody can apply. - let mount = match mediad::pipeline::Rotation::from_degrees(args.rotate) { + // 90 whatever the source. The head camera is mounted a quarter turn off and every consumer is + // told so — and the *simulated* camera is rolled the same way on purpose, so that a frame from a + // duck in MuJoCo needs the same turn as a frame from a duck on the desk. Overridable, because a + // scene could mount it differently, but there is one default and it is the robot's. + let rotate = args.rotate.unwrap_or(90); + let mount = match mediad::pipeline::Rotation::from_degrees(rotate) { Ok(rotation) => rotation, Err(e) => { tracing::error!(error = %e, "mediad cannot start"); return ExitCode::FAILURE; } }; - // What the stream is and what it looks for, from `[media]` and `[detect]` — see - // `--config` and `mediad::config`. One file, one read: `[detect]` is `mediad`'s section + // What the stream is and what it looks for, from `[media]` and `[duck_detector]` — see + // `--config` and `mediad::config`. One file, one read: `[duck_detector]` is `mediad`'s section // too, and a second config file for the second daemon that wants one is how a fleet ends // up with settings nobody can find. let explicit = args.config.is_some(); @@ -147,30 +244,46 @@ fn main() -> ExitCode { .clone() .unwrap_or_else(mediad::config::default_path); let params = mediad::config::load(&config, explicit); - let (media, detect) = (params.media, params.detect); + let (media, detect) = (params.media, params.duck_detector); + + // **What will actually run, not what is configured.** `[media] quality` is the rung a camera + // streams at; a test pattern ignores it and runs at `TEST_PATTERN_GEOMETRY`, so a log line + // reporting the rung on a board with no camera named a resolution nothing was producing. + // + // `--sim-camera` wins over `[media] source`, exactly as the source selection further down + // does: a simulated camera is a camera, and it renders the configured rung. + let (width, height, fps) = if args.sim_camera.is_some() { + ( + media.quality.width(), + media.quality.height(), + media.quality.fps(), + ) + } else { + media.geometry() + }; tracing::info!( - camera = media.camera, + source = media.source.label(), quality = media.quality.label(), - width = media.quality.width(), - height = media.quality.height(), - fps = media.quality.fps(), + width, + height, + fps, bitrate = media.bitrate_resolved(), congestion_control = media.congestion_control.nick(), "streaming" ); // The same angle the detector needs, in its own vocabulary: it folds the turn into the // resampling it already does, which is why nothing in the pipeline has to. - let turn = match duck_detect::Turn::from_degrees(args.rotate) { + let turn = match duck_detect::Turn::from_degrees(rotate) { Some(turn) => turn, None => { - tracing::error!(degrees = args.rotate, "mediad cannot start"); + tracing::error!(degrees = rotate, "mediad cannot start"); return ExitCode::FAILURE; } }; let rotation = if args.flip_in_pipeline { tracing::warn!( - degrees = args.rotate, + degrees = rotate, "--flip-in-pipeline: rotating in the pipeline costs the encoder its zero-copy path" ); mount @@ -188,8 +301,9 @@ fn main() -> ExitCode { // neither. So this is logged at error and the daemon carries on. let page = mediad::web::page(args.port); let (web_host, web_port) = (args.host.clone(), args.web_port); + let web_frame_socket = args.frame_socket.clone(); tokio::spawn(async move { - if let Err(e) = mediad::web::serve(&web_host, web_port, page).await { + if let Err(e) = mediad::web::serve(&web_host, web_port, page, web_frame_socket).await { tracing::error!( error = %format!("{e:#}"), "the console is not being served; video and control are unaffected" @@ -201,9 +315,9 @@ fn main() -> ExitCode { // producer that registered without a name would keep it until this daemon restarts. Costs a // unix-socket round trip on a boot where `configd` may not be up yet, which is why it is // bounded and why a failure is a warning rather than an exit. + let sockets = args.sockets(); let producer = - mediad::producer::Producer::learn(Default::default(), duck_ipc_proto::build_info!()) - .await; + mediad::producer::Producer::learn(sockets.clone(), duck_ipc_proto::build_info!()).await; tracing::info!( name = producer.name.as_deref().unwrap_or("unknown"), release = %producer.release, @@ -211,14 +325,75 @@ fn main() -> ExitCode { "producing as" ); - let source = if media.camera { - mediad::pipeline::Source::Camera(mediad::pipeline::Camera { - device: args.camera_device.clone(), - exposure: args.exposure, - analogue_gain: args.analogue_gain, - }) + // Relay candidates, so a consumer on a network that cannot punch a hole to this robot + // still reaches it. Spawned whatever the account state — it is inert without a token and + // starts on its own when a login lands — and *before* the pipeline, because the first + // consumer's offer is built as the pipeline comes up. + let relays = mediad::turn::Relays::empty(); + tokio::spawn(mediad::turn::maintain( + std::sync::Arc::clone(&relays), + args.token.clone(), + args.turn_url.clone(), + )); + + // What a control lane can say about this robot's own media: the picture's geometry, and + // the frame streamer. Empty until the pipeline is up — `sensor_mode()` is only truthful + // once something has tried to set it, and there are no frames to encode before then — and + // the relay below is spawned before that on purpose, so the answer has to be able to + // arrive late rather than be a value passed in now. + let (video_tx, video_rx) = + tokio::sync::watch::channel::>(None); + + // The outward half of remote access, and it is deliberately *after* the producer is + // learned: the name a client sees in the service's listing comes from the same place the + // local `meta` gets it, and a relay that registered first would publish an unnamed robot + // until the next restart. + // + // Spawned whatever happens next. It is inert without a token, it holds no lock, and a + // pipeline that fails to build should not take remote access down with it — a robot that + // appears in its owner's list and cannot stream is still a robot somebody can reach to + // find out why. + if args.no_remote { + tracing::info!("--no-remote: this robot will not register with the rendezvous service"); } else { - mediad::pipeline::Source::Test + match mediad::relay::Meta::of(&producer, None) { + None => tracing::warn!( + "no serial and no machine id, so this robot has no stable identity to \ + register with; remote access is off" + ), + Some(meta) => { + if let Some(relay) = + mediad::relay::Relay::new(&args.rendezvous_url, &args.token, meta) + { + // The bridge is a *consumer* of the signalling server this same process + // runs, so it has to be told the port `--port` chose rather than assuming + // the default — a robot moved off 8443 would otherwise register happily + // and fail every session. + tokio::spawn( + relay + .with_local_signalling(format!("ws://127.0.0.1:{}", args.port)) + .with_video(video_rx.clone()) + .run(), + ); + } + } + } + } + + // Matched rather than tested, so a source added to `MediaSource` fails the build here + // instead of quietly arriving as a test pattern. + let source = match args.sim_camera.clone() { + Some(addr) => mediad::pipeline::Source::Sim(addr), + None => match media.source { + robotd_params::MediaSource::Camera => { + mediad::pipeline::Source::Camera(mediad::pipeline::Camera { + device: args.camera_device.clone(), + exposure: args.exposure, + analogue_gain: args.analogue_gain, + }) + } + robotd_params::MediaSource::Test => mediad::pipeline::Source::Test, + }, }; // Frame size and rate are still pinned rather than negotiated — both branches of the tee @@ -231,9 +406,9 @@ fn main() -> ExitCode { port: args.port, bitrate: media.bitrate_resolved(), congestion_control: media.congestion_control, - width: media.quality.width(), - height: media.quality.height(), - fps: media.quality.fps(), + width, + height, + fps, rotation, }; @@ -241,17 +416,44 @@ fn main() -> ExitCode { // `get_frame` surface in `architecture.md` §5.3 is what the rest of it is for. The branch // runs from the start rather than being added later, because a tee inserted into a live // pipeline is a different and much harder problem than a tee that was always there. - let (_pipeline, mut channels, frames) = - match mediad::pipeline::start(source.clone(), &producer, &settings) { - Ok(started) => started, - Err(e) => { - // The message names which step failed and what usually causes it — a missing - // plugin, a missing library, or a device node nobody can open. Those look - // identical from a log line that only says "failed". - tracing::error!(error = %format!("{e:#}"), "mediad cannot start"); - return ExitCode::FAILURE; - } - }; + let (_pipeline, mut channels, frames, stream_branch) = match mediad::pipeline::start( + source.clone(), + &producer, + &settings, + std::sync::Arc::clone(&relays), + ) { + Ok(started) => started, + Err(e) => { + // The message names which step failed and what usually causes it — a missing + // plugin, a missing library, or a device node nobody can open. Those look + // identical from a log line that only says "failed". + tracing::error!(error = %format!("{e:#}"), "mediad cannot start"); + return ExitCode::FAILURE; + } + }; + + // A recorder or perception process asks the local Unix socket for one raw frame. It is + // deliberately not the datachannel: a snapshot is camera-sized, and control has to stay + // prompt even while a slow local reader is being served. `npu-bringup.md` names this. + let (frame_lock, frame_listener) = match mediad::frame::bind(&args.frame_socket).await { + Ok(bound) => bound, + Err(error) => { + tracing::error!(error = %error, "cannot bind media.frame; refusing a partial start"); + return ExitCode::FAILURE; + } + }; + let frame_source = frames.clone(); + // The mount angle every frame header carries — and zero when the pipeline was asked to + // flip, for the detector's sampler and the JPEG streamer's reason: those pixels arrive + // upright already, and turning them twice is a picture on its side with nothing to say why. + let frame_rotate = if args.flip_in_pipeline { 0 } else { rotate }; + tokio::spawn(async move { + let _lock = frame_lock; + if let Err(error) = mediad::frame::serve(frame_listener, frame_source, frame_rotate).await + { + tracing::error!(error = %format!("{error:#}"), "media.frame endpoint stopped"); + } + }); // After the pipeline, because it meters the pipeline's own frames — and only with a real // camera, since a test pattern has no sensor to write and the loop would spend the daemon's @@ -275,9 +477,11 @@ fn main() -> ExitCode { None } (mediad::pipeline::Source::Test, _) => None, + // A simulated camera has no sensor to write, and its brightness is the renderer's. + (mediad::pipeline::Source::Sim(_), _) => None, }; - // **The duck detector, from the same config file as everything else.** `[detect]` lives in + // **The duck detector, from the same config file as everything else.** `[duck_detector]` lives in // robotd.toml because that is the file `robotctl configure` edits and a robot has one place // for its switches — even though it is this daemon that reads that section. // @@ -286,7 +490,7 @@ fn main() -> ExitCode { // boot because a model file moved" is a bad trade. let models = detect.models(); let detector = if models.is_empty() { - tracing::info!("duck detector off ([detect] enabled = false, or no model)"); + tracing::info!("duck detector off ([duck_detector] enabled = false, or no model)"); None } else { // The frames on the tee are as the camera took them — unless the pipeline was asked @@ -317,18 +521,96 @@ fn main() -> ExitCode { // What every peer is told about the picture. The geometry is the *encoded* frame — the // pipeline does not rotate, so it is the capture geometry — and the rotation is the mount. + // The camera's geometry, for a consumer that has to turn pixels into directions. Read + // *after* the pipeline is up, because which sensor mode is in force is only known once + // something tried to set it — and a mode nobody knows the field of view of publishes + // nothing rather than a plausible wrong number. `mediad::camera` has the arithmetic. + let intrinsics = if args.sim_camera.is_some() { + // The MuJoCo twin renders a known field of view, so publish its exact geometry — twin + // recordings then self-describe (no `--calib` needed on the duckslam side). + mediad::camera::Intrinsics::sim(width, height) + } else { + mediad::camera::Intrinsics::published( + media.intrinsics.as_ref(), + mediad::pipeline::sensor_mode(), + width, + height, + ) + }; + match &intrinsics { + Some(geometry) => tracing::info!( + fx = geometry.fx, + fy = geometry.fy, + cx = geometry.cx, + cy = geometry.cy, + calibrated = geometry.calibrated, + "camera geometry" + ), + None => tracing::info!( + "no camera geometry to publish: the sensor is not in a mode whose field of view \ + is known, so a consumer is told nothing rather than something wrong" + ), + } + let video = mediad::session::Video { - width: media.quality.width(), - height: media.quality.height(), - rotate: args.rotate, + width, + height, + rotate, + intrinsics, }; + // Frames out to a WebSocket this robot dials, when something asks for them. Built here + // because it needs the tee — and given the same `turn` the detector's sampler gets, for + // the same reason: a pipeline that was asked to flip has already turned the frames, and + // turning them twice is a picture on its side with nothing to say why. + let streamer = std::sync::Arc::new(mediad::stream::Streamer::new( + mediad::stream::Encoders { + // The same `turn` the detector's sampler gets, and for the same reason: a pipeline + // asked to flip has already turned the frames, and turning them twice is a + // picture on its side with nothing to say why. + jpeg: mediad::stream::jpeg_encoder( + frames.clone(), + if args.flip_in_pipeline { + duck_detect::Turn::None + } else { + turn + }, + ), + // The H.264 branch turns nothing: it is downstream of the same tee, so the flip — + // or its absence — is already in the pixels it encodes. + h264: stream_branch.clone().map(mediad::stream::h264_encoder), + gate: stream_branch.clone().map(|branch| { + std::sync::Arc::new(move |open: bool| { + if open { + branch.open(); + } else { + branch.close(); + } + }) as std::sync::Arc + }), + }, + producer.clone(), + // The resolved mount angle, not the flag: `--rotate` is an `Option` now and the + // default lives in one place at the top of `main`. + rotate, + &args.token, + )); + + let media = mediad::session::Media { + video: video.clone(), + streamer: std::sync::Arc::clone(&streamer), + }; + + // The relay has been up since before the pipeline; this is the point its control lanes can + // start answering for the robot's own media. + let _ = video_tx.send(Some(media.clone())); + // One session per peer, each with its own connections to the services it talks to. Per // peer rather than shared, so one peer's minutes-long update cannot silence another's // telemetry — which is the same reason a session keeps one connection per lane. while let Some(channel) = channels.recv().await { let (replies_tx, mut replies_rx) = tokio::sync::mpsc::channel::(256); - let pool = mediad::upstream::Pool::new(Default::default(), replies_tx); + let pool = mediad::upstream::Pool::new(sockets.clone(), replies_tx); let to_peer = channel.outbound.clone(); tokio::spawn(async move { @@ -343,7 +625,7 @@ fn main() -> ExitCode { // (`media.video`), which is why that path exists and this one is best-effort. { let to_peer = channel.outbound.clone(); - let line = mediad::session::video_notification(video); + let line = mediad::session::video_notification(&video); tokio::spawn(async move { let _ = to_peer.send(line).await; }); @@ -380,7 +662,10 @@ fn main() -> ExitCode { channel.inbound, channel.outbound, pool, - video, + // Cloned per session: it carries the camera's intrinsics and a handle to the + // frame streamer. Always `Some` here — a datachannel exists because the pipeline + // handed over a consumer, so by definition there is media behind it. + Some(media.clone()), )); } diff --git a/mediad/src/pipeline.rs b/mediad/src/pipeline.rs index ac96d756..600b8eee 100644 --- a/mediad/src/pipeline.rs +++ b/mediad/src/pipeline.rs @@ -69,6 +69,12 @@ //! without the capture path existing. The camera arrives as a different source element behind the //! same encoder, and `media-bringup.md` records why capture cannot simply be `v4l2src`. //! +//! **It runs at `robotd_params::TEST_PATTERN_GEOMETRY` rather than `[media] quality`**, and that is +//! a CPU decision. A camera's frames come off the ISP in hardware; a test pattern's are drawn by +//! this process, so at the configured rung an idle board with no camera burned 29.4% of a core +//! against a real camera's 6.1% — synthesising 1.84 MB of UYVY thirty times a second for a tee +//! whose readers had all said no. The session it exists to provide needs none of those pixels. +//! //! ## What is not verified //! //! **Nothing in a signal handler here may panic.** These closures are invoked from C, so a panic @@ -83,6 +89,7 @@ //! naming the arity rather than as an abort. use std::sync::{Arc, Mutex}; +use std::time::SystemTime; use anyhow::{Context, Result, anyhow, bail}; use duck_ipc_proto as proto; @@ -193,6 +200,14 @@ pub enum Source { Test, /// The head camera, through the rkisp capture path. Camera(Camera), + /// A simulated head camera, at `host:port`: what a duck in MuJoCo sees. + /// + /// Frames arrive length-prefixed and raw rather than as JSON, unlike the rest of the simulator + /// links — 640x360 UYVY is 460,800 bytes, and at 15 fps that is 6.9 MB/s. There is no handshake, + /// because there is nothing to negotiate that both ends do not already have to agree on to be + /// useful: the geometry is fixed on both sides — `[media] quality` here, the body's camera + /// there — or nothing works. + Sim(String), } /// The head camera, and the two things it will not work without. @@ -219,6 +234,10 @@ pub struct Frame { /// The GStreamer format name — [`CAPTURE_FORMAT`], carried rather than assumed so a consumer /// reading this cannot silently misinterpret the bytes if the capture format changes again. pub format: &'static str, + /// When this buffer was taken, for a consumer joining a frame to a separately sampled + /// robot state. Observation time, not a scheduling clock: it is never used to pace capture, + /// so an NTP step cannot reach the pipeline through it. + pub captured_at: SystemTime, /// Tightly packed as the caps describe it, in `format`. pub data: Vec, } @@ -323,14 +342,19 @@ impl Frames { /// Whether a reader is waiting, taking the request if one is. The callback's whole cost on a /// frame nobody asked for. - fn take_request(&self) -> bool { + /// + /// `pub(crate)` so that [`crate::frame`]'s tests can stand in for the capture branch: the + /// endpoint's behaviour on a real delivery is only testable by driving this rendezvous. + pub(crate) fn take_request(&self) -> bool { self.0 .wanted .swap(false, std::sync::atomic::Ordering::Relaxed) } /// Hand the captured frame to whoever is waiting for it. - fn deliver(&self, frame: Frame) { + /// + /// `pub(crate)` for the same reason as [`Frames::take_request`]. + pub(crate) fn deliver(&self, frame: Frame) { let mut latest = self.0.latest.lock().expect("frame lock"); latest.frame = Some(frame); latest.generation = latest.generation.wrapping_add(1); @@ -357,7 +381,13 @@ pub fn start( source: Source, producer: &crate::producer::Producer, settings: &Settings, -) -> Result<(gst::Pipeline, mpsc::Receiver, Frames)> { + relays: Arc, +) -> Result<( + gst::Pipeline, + mpsc::Receiver, + Frames, + Option, +)> { let &Settings { port, bitrate, @@ -404,6 +434,7 @@ pub fn start( src } Source::Camera(camera) => camera_source(camera, fps)?, + Source::Sim(addr) => sim_source(addr, width, height, fps)?, }; // Pinned rather than negotiated, because both branches of the tee depend on the answer, and a @@ -557,7 +588,7 @@ pub fn start( let consumers: Consumers = Arc::new(std::sync::atomic::AtomicU32::new(0)); let (channels_tx, channels_rx) = mpsc::channel::(4); - wire_consumers(&sink, channels_tx, runtime, consumers.clone())?; + wire_consumers(&sink, channels_tx, runtime, consumers.clone(), relays)?; // ── the raw branch ────────────────────────────────────────────────────── // @@ -595,6 +626,27 @@ pub fn start( .build(); wire_frames(&appsink, frames.clone(), out_width, out_height); + // ── the H.264 branch, for a Space this robot streams to ───────────────── + // + // A third branch off the same raw tee, and **valved shut**: with `drop=true` nothing reaches + // the encoder, so a second encode costs nothing until `media.stream` asks for it. Built once + // rather than added on demand, because adding elements to a live pipeline means pad-blocking + // surgery and this file's history with a `videoflip` is a warning about touching this path. + // + // `webrtcsink` owns the encoder on the other branch and is handed raw video on purpose (see + // the header): with pre-encoded input its congestion control cannot reach the encoder. So + // there is nothing to tap and this is a second encoder — cheap on a VPU at a few frames a + // second, and the reason the rate and the size are pinned here rather than left to the caller. + let stream_branch = build_stream_branch(&pipeline, out_width, out_height, fps) + .inspect_err(|error| { + tracing::warn!( + error = %format!("{error:#}"), + "no H.264 branch, so this robot cannot stream frames to a Space; the rest of the \ + pipeline is unaffected" + ); + }) + .ok(); + if let Some(flip) = flip.as_ref() { pipeline .add(flip) @@ -642,6 +694,10 @@ pub fn start( // two links are separate from the `link_many` chains above. link_tee_branch(&tee, &video_queue).context("could not attach the video branch to the tee")?; link_tee_branch(&tee, &raw_queue).context("could not attach the raw branch to the tee")?; + if let Some(branch) = stream_branch.as_ref() { + link_tee_branch(&tee, branch.head()) + .context("could not attach the H.264 branch to the tee")?; + } // **Watch the bus, or every media failure is silent.** // @@ -669,7 +725,341 @@ pub fn start( fps, "signalling server listening" ); - Ok((pipeline, channels_rx, frames)) + Ok((pipeline, channels_rx, frames, stream_branch)) +} + +/// Build the valved H.264 branch: `queue ! valve ! videorate ! videoscale ! videoconvert ! enc ! +/// parse ! appsink`. +/// +/// Fails rather than degrades when there is no encoder to use, and the caller carries on without a +/// branch: a robot that cannot stream to a Space is still a robot that walks, and `media.stream` +/// then refuses with a reason instead of accepting and sending nothing. +fn build_stream_branch( + pipeline: &gst::Pipeline, + width: u32, + height: u32, + fps: u32, +) -> Result { + // Rate and size for the streamed copy, independent of what the video track carries. Five a + // second at 640 is what a model wants and a fraction of the encode the WebRTC branch does. + const STREAM_FPS: u32 = 5; + const STREAM_LONGEST: u32 = 640; + + let make = |name: &str| -> Result { + gst::ElementFactory::make(name) + .build() + .map_err(|_| anyhow!("no {name} element")) + }; + + // Leaky like the raw branch: a stalled encoder must never become the video track's problem. + let queue = gst::ElementFactory::make("queue") + .property("max-size-buffers", 2u32) + .property("max-size-bytes", 0u32) + .property("max-size-time", 0u64) + .property_from_str("leaky", "downstream") + .build() + .map_err(|_| anyhow!("no queue element"))?; + + let valve = gst::ElementFactory::make("valve") + // **Shut until asked.** This is what makes a second encoder free while nobody streams. + .property("drop", true) + .build() + .map_err(|_| anyhow!("no valve element"))?; + + // `drop-only` so it never duplicates a frame to hit a rate — a repeated frame costs the + // encoder a whole access unit to say nothing happened. + let rate = gst::ElementFactory::make("videorate") + .property("drop-only", true) + .property("max-rate", STREAM_FPS as i32) + .build() + .map_err(|_| anyhow!("no videorate element"))?; + + let scale = make("videoscale")?; + + // The turn already happened before the tee, so this branch's input is upright and the aspect + // ratio here is the upright one. + let scale = (scale, { + let longest = width.max(height) as f32; + let factor = (STREAM_LONGEST as f32 / longest).min(1.0); + // Even dimensions: H.264 chroma is subsampled, and an odd width is a negotiation failure + // on some encoders and a green column on others. + let even = |value: f32| ((value.round() as u32).max(2) / 2) * 2; + gst::Caps::builder("video/x-raw") + .field("width", even(width as f32 * factor) as i32) + .field("height", even(height as f32 * factor) as i32) + .build() + }); + let caps = gst::ElementFactory::make("capsfilter") + .property("caps", &scale.1) + .build() + .map_err(|_| anyhow!("no capsfilter element"))?; + let scale = scale.0; + + // The tee carries `UYVY` (see the capture caps above) and neither encoder takes it: `x264enc` + // wants planar YUV, `mpph264enc` NV12. Without this the branch **fails to link at build time**, + // `start` returns the error, and mediad does not start at all on a machine without `mpph264enc` + // — which is every laptop, and the sim twin with it. A passthrough when formats already agree, + // so it costs the board nothing. + let convert = make("videoconvert")?; + + // `mpph264enc` is the board's hardware encoder — the same one `webrtcsink` uses through the + // patched plugin the header describes. `x264enc` is the fallback for a laptop and for a board + // whose MPP is missing, which is a slow encode rather than a broken one. + let encoder = make("mpph264enc") + .or_else(|_| { + tracing::info!("no mpph264enc; falling back to x264enc for the frame stream"); + make("x264enc") + }) + .context("neither mpph264enc nor x264enc is available")?; + + // `config-interval=-1` repeats SPS and PPS in front of every keyframe, which is what lets a + // receiver that connects mid-stream decode from the next one without having been sent + // anything it missed. Without it a reconnecting Space needs the parameter sets it never saw. + let parse = gst::ElementFactory::make("h264parse") + .property("config-interval", -1i32) + .build() + .map_err(|_| anyhow!("no h264parse element"))?; + + let encoded = Encoded::default(); + let appsink = gst_app::AppSink::builder() + .caps( + &gst::Caps::builder("video/x-h264") + .field("stream-format", "byte-stream") + .field("alignment", "au") + .build(), + ) + // One WebSocket message is one access unit, which is what `alignment=au` above buys. + .sync(false) + // **`async=false`, or this sink holds the whole pipeline in PAUSED.** A sink prerolls on + // its first buffer, and the bin does not finish going to PLAYING until every async sink + // has. This branch's first buffer only arrives once somebody opens the valve — so with the + // default the pipeline never completed its state change, and the *raw* appsink one branch + // over, which had prerolled, waited for PLAYING for ever: no callbacks, no frames, and the + // duck detector and the auto-exposure loop starved on a camera that was capturing at 30 + // fps. The video track kept working because `webrtcsink` is live and does not preroll, + // which is what made this invisible from the console. + .async_(false) + .max_buffers(ENCODED_DEPTH as u32) + .drop(false) + .build(); + + { + let encoded = encoded.clone(); + appsink.set_callbacks( + gst_app::AppSinkCallbacks::builder() + .new_sample(move |sink| { + let sample = sink.pull_sample().map_err(|_| gst::FlowError::Eos)?; + let buffer = sample.buffer().ok_or(gst::FlowError::Error)?; + let map = buffer.map_readable().map_err(|_| gst::FlowError::Error)?; + // A keyframe is a buffer *without* the delta-unit flag. Reading it the other + // way round would mark every P-frame a keyframe and defeat the whole queue. + let keyframe = !buffer.flags().contains(gst::BufferFlags::DELTA_UNIT); + encoded.push(map.as_slice().to_vec(), keyframe); + Ok(gst::FlowSuccess::Ok) + }) + .build(), + ); + } + + pipeline + .add_many([ + &queue, + &valve, + &rate, + &scale, + &caps, + &convert, + &encoder, + &parse, + appsink.upcast_ref(), + ]) + .context("could not add the H.264 branch to the pipeline")?; + gst::Element::link_many([ + &queue, + &valve, + &rate, + &scale, + &caps, + &convert, + &encoder, + &parse, + appsink.upcast_ref(), + ]) + .context("could not link the H.264 branch")?; + + tracing::info!( + encoder = %encoder.factory().map(|f| f.name().to_string()).unwrap_or_default(), + fps = STREAM_FPS, + "an H.264 branch is available for streaming, shut until something asks" + ); + let _ = fps; + Ok(StreamBranch { + head: queue, + valve, + encoder, + encoded, + }) +} + +/// The H.264 branch's output: access units in order, with gaps closed to the next keyframe. +/// +/// **Not [`Frames`], and the difference is the whole point.** `Frames` is last-value-wins: a slow +/// reader gets the newest picture and the older ones are discarded, which is exactly right for an +/// independent frame and exactly wrong for a predicted one. A P-frame whose reference was dropped +/// decodes to garbage that looks like a broken camera rather than a broken transport, so a reader +/// that falls behind here is given the next *keyframe* and nothing between. +/// +/// A short queue rather than one slot for the same reason: a group of pictures has to arrive whole. +#[derive(Clone, Default)] +pub struct Encoded(Arc); + +#[derive(Default)] +struct EncodedShared { + units: Mutex, + arrived: std::sync::Condvar, +} + +#[derive(Default)] +struct EncodedQueue { + /// `(bytes, keyframe)`, oldest first. + ready: std::collections::VecDeque<(Vec, bool)>, + /// Set when the queue overflowed: everything until the next keyframe is unusable to a reader + /// that missed what came before it. + awaiting_key: bool, + /// How many units were discarded, for the stream's own counters. + dropped: u64, +} + +/// How many access units to hold. Two seconds at 15 fps, which is longer than any keyframe +/// interval worth setting — so a reader that stalls briefly recovers without waiting for one. +const ENCODED_DEPTH: usize = 32; + +impl Encoded { + /// Take the next unit, waiting up to `FRAME_TIMEOUT` for one. + /// + /// `None` means the encoder produced nothing in that time, which on a valved branch is the + /// ordinary state: nobody is streaming, so nothing is being encoded. + pub fn next_unit(&self) -> Option<(Vec, bool)> { + let mut units = self.0.units.lock().expect("not poisoned"); + loop { + while let Some((bytes, keyframe)) = units.ready.pop_front() { + if units.awaiting_key && !keyframe { + units.dropped += 1; + continue; + } + units.awaiting_key = false; + return Some((bytes, keyframe)); + } + let (again, timed_out) = self + .0 + .arrived + .wait_timeout(units, FRAME_TIMEOUT) + .expect("not poisoned"); + units = again; + if timed_out.timed_out() && units.ready.is_empty() { + return None; + } + } + } + + /// Units discarded because a reader was behind. Cumulative. + pub fn dropped(&self) -> u64 { + self.0.units.lock().expect("not poisoned").dropped + } + + fn push(&self, bytes: Vec, keyframe: bool) { + let mut units = self.0.units.lock().expect("not poisoned"); + if units.ready.len() >= ENCODED_DEPTH { + // Behind by two seconds of video. Throwing away one unit would leave a hole that + // corrupts everything referring across it, so the queue is emptied and the stream + // resumes at the next keyframe. + units.dropped += units.ready.len() as u64; + units.ready.clear(); + units.awaiting_key = true; + } + units.ready.push_back((bytes, keyframe)); + drop(units); + self.0.arrived.notify_one(); + } +} + +/// Turn the H.264 branch on or off, and ask its encoder for a keyframe. +/// +/// **A valve rather than adding and removing elements.** The branch is built once and gated: with +/// `drop=true` no buffer reaches the encoder, so the second encode costs nothing while nobody is +/// streaming — and turning it on is a property write rather than pad-blocking surgery on a live +/// pipeline. `remote-webrtc.md`'s note that the encoder is this board's budget is why it is gated +/// at all, and `pipeline.rs`'s own history with a `videoflip` is why it is not rebuilt. +#[derive(Clone)] +pub struct StreamBranch { + /// The branch's first element — its `queue`, which is what the tee's request pad links to. + /// + /// **Separate from the valve, and it cost a panic on the board to learn why.** Both were one + /// field called `valve`, holding the queue because that is what has to be linked; `open()` + /// then set `drop` on it and glib panicked with `property 'drop' of type 'GstQueue' not + /// found`. It killed the task handling the call rather than the daemon, so the symptom was a + /// `media.stream` that never answered — silence, from a robot that was otherwise fine. + head: gst::Element, + valve: gst::Element, + encoder: gst::Element, + pub encoded: Encoded, +} + +impl StreamBranch { + /// What the tee links to: the head of the branch, not the valve behind it. + pub fn head(&self) -> &gst::Element { + &self.head + } + + /// Open the valve, and ask for a keyframe so a receiver has something to start on. + /// + /// Without the request a receiver waits for the encoder's own keyframe interval before its + /// first decodable picture — seconds of nothing, indistinguishable from a stream that is not + /// working. `h264parse config-interval=-1` puts SPS and PPS in front of it, so that keyframe + /// is enough on its own. + pub fn open(&self) { + self.gate(false); + self.request_keyframe(); + } + + pub fn close(&self) { + self.gate(true); + } + + /// Set the valve's `drop`, and **do not panic if it is the wrong element**. + /// + /// `set_property` panics on a name the element does not have, and this one ran on a tokio + /// worker inside the task answering `media.stream` — so the first version of this file killed + /// that task and the call simply never came back. A robot that is otherwise healthy, silent + /// on one method, is a much worse failure than a refusal. The element is right now; the guard + /// is for the next time somebody moves a field. + fn gate(&self, drop: bool) { + if self + .valve + .has_property_with_type("drop", bool::static_type()) + { + self.valve.set_property("drop", drop); + } else { + tracing::error!( + element = %self.valve.factory().map(|f| f.name().to_string()).unwrap_or_default(), + "the frame stream's valve has no `drop` property, so it cannot be gated" + ); + } + } + + /// Ask the encoder for a keyframe now. + pub fn request_keyframe(&self) { + if let Some(pad) = self.encoder.static_pad("sink") { + // Upstream, on the encoder's sink pad: the event travels to the encoder, which is the + // element that can honour it. `all_headers` is what repeats SPS/PPS with it. + let event = gst_video::UpstreamForceKeyUnitEvent::builder() + .all_headers(true) + .build(); + if !pad.send_event(event) { + tracing::debug!("the encoder would not take a keyframe request"); + } + } + } } /// Request a source pad from the tee and link it to a branch's sink pad. @@ -720,6 +1110,9 @@ fn wire_frames(appsink: &gst_app::AppSink, frames: Frames, width: u32, height: u width, height, format: CAPTURE_FORMAT, + // Taken here rather than at delivery: this is the moment the buffer existed, + // and it costs one clock read on a frame someone already asked for. + captured_at: SystemTime::now(), data: map.as_slice().to_vec(), }); @@ -894,6 +1287,102 @@ fn camera_source(camera: &Camera, fps: u32) -> Result { Ok(src) } +/// A simulated camera as an `appsrc`, fed by a thread reading frames off a socket. +/// +/// **`is-live` and `do-timestamp`, both of them.** A camera is live by construction; an `appsrc` is +/// not, and without saying so the pipeline races ahead of the clock and `webrtcsink` sees a source +/// that can be pulled faster than real time. And without timestamps every downstream element has to +/// invent them, which shows up as a stream that plays at the wrong speed rather than as an error. +/// +/// The reader owns the reconnect: MuJoCo restarts whenever the number of ducks changes, and a +/// camera that goes away must not take the pipeline with it — the encoder simply has no new frames +/// until it comes back, which is what a real camera being unplugged looks like too. +fn sim_source(addr: &str, width: u32, height: u32, fps: u32) -> Result { + use gst_app::prelude::*; + + let caps = gst::Caps::builder("video/x-raw") + .field("format", CAPTURE_FORMAT) + .field("width", width as i32) + .field("height", height as i32) + .field("framerate", gst::Fraction::new(fps as i32, 1)) + .build(); + + let src = gst_app::AppSrc::builder() + .caps(&caps) + .is_live(true) + .do_timestamp(true) + .format(gst::Format::Time) + .build(); + + let expected = (width as usize) * (height as usize) * 2; + let announce = addr.to_owned(); + let addr = addr.to_owned(); + let pushable = src.clone(); + std::thread::Builder::new() + .name("sim-camera".into()) + .spawn(move || { + let mut complained = false; + loop { + match read_frames(&addr, expected, &pushable) { + // A clean close means it had connected and streamed; clear the flag so the + // *next* failure is logged, as `tofd`'s `sim_loop` and `RemoteIo` both do. + Ok(()) => { + complained = false; + tracing::warn!(%addr, "the simulated camera closed"); + } + Err(e) if !complained => { + complained = true; + tracing::warn!(%addr, error = %e, "no simulated camera; retrying"); + } + Err(_) => {} + } + std::thread::sleep(std::time::Duration::from_secs(1)); + } + }) + .map(|_| ()) + .unwrap_or_else( + |e| tracing::error!(error = %e, "no reader thread for the simulated camera"), + ); + + tracing::info!(addr = %announce, width, height, fps, "simulated head camera"); + Ok(src.upcast()) +} + +/// Length-prefixed frames from the simulator into an `appsrc`, until it stops or the frames stop. +fn read_frames(addr: &str, expected: usize, src: &gst_app::AppSrc) -> std::io::Result<()> { + use std::io::Read; + + let stream = std::net::TcpStream::connect(addr)?; + stream.set_nodelay(true)?; + let mut reader = std::io::BufReader::new(stream); + let mut header = [0u8; 4]; + let mut frame = vec![0u8; expected]; + tracing::info!(%addr, "the simulated camera is feeding the pipeline"); + + loop { + reader.read_exact(&mut header)?; + let len = u32::from_le_bytes(header) as usize; + // A frame of the wrong size means the two ends disagree about the geometry, and pushing it + // would be a picture nobody can read. Said once, loudly, rather than a stream of noise. + if len != expected { + return Err(std::io::Error::other(format!( + "the simulator sent a {len}-byte frame and this pipeline expects {expected} — the simulator's camera must match `[media] quality`" + ))); + } + reader.read_exact(&mut frame)?; + let mut buffer = gst::Buffer::with_size(len).map_err(std::io::Error::other)?; + buffer + .get_mut() + .expect("a fresh buffer is writable") + .map_writable() + .map_err(std::io::Error::other)? + .copy_from_slice(&frame); + if src.push_buffer(buffer).is_err() { + return Ok(()); // the pipeline is gone + } + } +} + /// What the tee carries, and what both branches therefore see. /// /// Single-plane on purpose: `v4l2src` cannot drive rkisp's two-plane `NM12` at full rate, and @@ -1020,6 +1509,20 @@ fn raise_capture_buffers(src: &gst::Element) -> Result<()> { Ok(()) } +/// Which sensor mode this process managed to put the camera in, once it has tried. +/// +/// A `OnceLock` rather than a value threaded up through the pipeline builder, because that is what +/// it is: one fact about this process's camera, established while the pipeline is built and read +/// afterwards by whatever answers `media.video`. `None` — never set, or set after a failed switch +/// — means the geometry is unknown, and `crate::camera` publishes no intrinsics for it. +static SENSOR_MODE: std::sync::OnceLock> = + std::sync::OnceLock::new(); + +/// The sensor mode in force, or `None` when there is no camera or the switch did not take. +pub fn sensor_mode() -> Option { + *SENSOR_MODE.get().unwrap_or(&None) +} + /// Switch the IMX219 out of its boot mode, which caps capture at 21 fps. /// /// The sensor boots in 3280x2464 and the rkisp scaler will happily give us 1280x720 from it — at @@ -1046,9 +1549,13 @@ fn pin_sensor_mode(fps: u32) -> Result<()> { %media, %entity, why = %String::from_utf8_lossy(&output.stderr).trim(), "media-ctl would not set the 1920x1080 sensor mode — capture stays in the boot \ - mode, which caps it at 21 fps" + mode, which caps it at 21 fps, and `media.video` publishes no camera intrinsics \ + because the exact framing is then the boot mode's (same ~62 deg field, different \ + 4:3->16:9 crop) rather than the pinned mode the calibration is for" ); + let _ = SENSOR_MODE.set(None); } else { + let _ = SENSOR_MODE.set(Some(crate::camera::SensorMode::PINNED)); tracing::info!(%media, %entity, target_fps = fps, "sensor mode 1920x1080"); } Ok(()) @@ -1360,6 +1867,7 @@ fn wire_consumers( channels: mpsc::Sender, runtime: tokio::runtime::Handle, consumers: Consumers, + relays: Arc, ) -> Result<()> { // Counted here rather than inferred from the log, so `robotctl health` can say whether anyone // is actually watching. `consumer-removed` is guarded the same way `consumer-added` is: a @@ -1400,6 +1908,10 @@ fn wire_consumers( .and_then(|v| v.get::().ok()) .unwrap_or_else(|| "?".into()); + // Before the datachannel, because this is what the *offer* needs and the offer is + // generated as soon as this handler returns. §6 of `remote-access-design.md`. + offer_relay_candidates(&webrtcbin, &peer, &relays); + match open_control_channel(&webrtcbin, &peer, &runtime) { Ok(channel) => { // A full queue means nobody is accepting sessions, which is a bug rather than @@ -1415,6 +1927,47 @@ fn wire_consumers( Ok(()) } +/// Add this robot's TURN servers to one consumer's `webrtcbin`, so its offer carries a `relay`. +/// +/// **Runs on the thread that builds the offer, and must not block it.** `Relays::uris` reads a +/// cache and never does I/O for exactly this reason: a fetch here would delay every consumer's +/// connection, including the LAN ones that will never use a relay. An empty list is the ordinary +/// state right after boot and on a robot nobody has signed in — host and srflx candidates are +/// enough for anything on the same network. +/// +/// Nothing here is fatal. A robot that cannot offer a relay is reachable from most places; one +/// whose negotiation broke because a credential was malformed is reachable from none. +fn offer_relay_candidates(webrtcbin: &gst::Element, peer: &str, relays: &Arc) { + let uris = relays.uris(); + if uris.is_empty() { + tracing::debug!(peer, "no relay servers held; offering host and srflx only"); + return; + } + // Checked before it is emitted, for the reason `open_control_channel` checks its own signal: + // `emit_by_name` panics when a signal is absent or its signature has changed, and a panic in + // a C closure aborts the process instead of unwinding. + if glib::subclass::signal::SignalId::lookup("add-turn-server", webrtcbin.type_()).is_none() { + tracing::warn!( + peer, + "webrtcbin has no add-turn-server signal; this consumer gets no relay candidate" + ); + return; + } + let mut added = 0; + for uri in uris.iter() { + // The return is whether the server was accepted; a rejected URI is worth a line and not + // an abandoned session. + if webrtcbin.emit_by_name::("add-turn-server", &[&uri.as_str()]) { + added += 1; + } else { + // The host only. **A TURN URI carries a password**, and a log line is the one place + // it must never appear. + tracing::warn!(peer, "a relay server was refused by webrtcbin"); + } + } + tracing::info!(peer, relays = added, "offering relay candidates"); +} + /// Create the `control` datachannel on one peer's `webrtcbin` and bridge it to channels. fn open_control_channel( webrtcbin: &gst::Element, @@ -1489,12 +2042,71 @@ fn open_control_channel( mod tests { use super::*; + /// **A reader gets a frame out of the real pipeline.** The rendezvous tests below stand in + /// for the appsink; this one runs the appsink, on the test pattern, with every branch the + /// robot has — because the bug this guards was between branches. The valved H.264 sink + /// prerolled nothing, so the bin never finished going to PLAYING, and the raw appsink sat in + /// preroll holding its first buffer for ever: `starved=20` on every detector report and no + /// `metering` line from auto-exposure, on a robot whose console video was fine. + /// + /// Skipped, and loudly, where the plugins are not installed: CI has GStreamer's base and bad + /// sets but neither `webrtcsink` nor an H.264 encoder, and a test that fails for want of a + /// plugin says nothing about the pipeline. A machine set up to run `mediad` has them. + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn a_reader_gets_a_frame_from_the_running_pipeline() { + gst::init().expect("gstreamer"); + let missing: Vec<&str> = ["videotestsrc", "webrtcsink", "valve", "h264parse"] + .into_iter() + .filter(|name| gst::ElementFactory::find(name).is_none()) + .collect(); + let no_encoder = ["mpph264enc", "x264enc"] + .iter() + .all(|name| gst::ElementFactory::find(name).is_none()); + if !missing.is_empty() || no_encoder { + eprintln!("skipping: no {missing:?} / no H.264 encoder on this machine"); + return; + } + + let producer = crate::producer::Producer::local(duck_ipc_proto::build_info!()); + let settings = Settings { + host: "127.0.0.1".into(), + // Not 8443, so a `mediad` already running on this machine is left alone. + port: 18_443, + bitrate: 500_000, + congestion_control: robotd_params::CongestionControl::default(), + width: 320, + height: 240, + fps: 15, + rotation: Rotation::None, + }; + let (pipeline, _channels, frames, _stream) = start( + Source::Test, + &producer, + &settings, + crate::turn::Relays::empty(), + ) + .expect("the pipeline starts on the test pattern"); + + // Several asks, because the first can legitimately land before the source has produced + // anything; what must not happen is every one of them timing out. + let frame = tokio::task::spawn_blocking(move || (0..10).find_map(|_| frames.next_frame())) + .await + .expect("the reader thread"); + let _ = pipeline.set_state(gst::State::Null); + + let frame = frame.expect("a frame within five seconds; the raw branch is starved"); + assert_eq!((frame.width, frame.height), (320, 240)); + assert_eq!(frame.format, CAPTURE_FORMAT); + assert_eq!(frame.data.len(), 320 * 240 * 2, "packed UYVY"); + } + /// A frame whose every byte is `tag`, so a test can say *which* capture came back. fn frame(tag: u8) -> Frame { Frame { width: 4, height: 2, format: CAPTURE_FORMAT, + captured_at: SystemTime::UNIX_EPOCH + std::time::Duration::from_secs(tag as u64), data: vec![tag; 16], } } diff --git a/mediad/src/relay.rs b/mediad/src/relay.rs new file mode 100644 index 00000000..68964a39 --- /dev/null +++ b/mediad/src/relay.rs @@ -0,0 +1,2506 @@ +//! The robot's half of the bridge to the rendezvous service. +//! +//! A LAN client reaches `webrtcsink`'s signalling server directly and nothing here is involved. +//! This is what makes a duck reachable from *outside* its network: it connects **outward** to +//! `reachy_mini_central` holding the account token, registers as a **producer**, and the service +//! shows a client only the robots its own account owns. `docs/design/remote-access-design.md` §3 +//! owns the argument; this module owns the connection. +//! +//! # What it does +//! +//! **Registration and liveness** (§8, slice 2): the robot appears in the service's listing and +//! stays there, for as long as the account token on disk is the one it connected with. +//! +//! **And it carries a session** (slice 4). A remote consumer cannot reach `webrtcsink`'s +//! signalling server — it is on a loopback address behind somebody's router — so when one asks +//! the rendezvous for a session, [`bridge`] opens that socket *as the consumer on its behalf* and +//! carries envelopes between the two, rewriting the `sessionId` each hop names and reading none +//! of the payload. One session at a time; a second is refused by name. +//! +//! # A control lane that needs no candidate pair, and why there is one +//! +//! The bridge above carries a *negotiation*: SDP and ICE, so that a consumer and the robot can +//! find a path between them and speak WebRTC over it. When they cannot find one, everything built +//! on it is gone. A relay candidate is what keeps that from happening (§6) — and it is a +//! dependency rather than a guarantee: it is somebody else's service, metered per account, and +//! the control channel is SCTP over whatever pair ICE settled on, so a relay that stops being +//! available takes a JSON-RPC call of a few hundred bytes down with the video. +//! +//! So the JSON-RPC a consumer wants to send does not have to go through WebRTC at all, and the +//! rendezvous turns out to already carry it: `handle_peer_message` in their `app.py` relays +//! **every key of a `peer` envelope except `type` and `sessionId`** verbatim to the session +//! partner, without looking at `sdp` or `ice`. A `peer` message carrying an `rpc` key is therefore +//! a control call, relayed opaquely, with no change to a service the mini fleet also depends on. +//! +//! ```text +//! consumer ──POST /send {type:peer, sessionId, rpc:{…}}──► rendezvous ──SSE──► this relay +//! ◄─────────── SSE {type:peer, sessionId, rpc:{…}} ◄──POST /send────────┘ +//! ``` +//! +//! `session::run` is what answers it, unchanged: its own header says it is transport-agnostic so +//! that "a WebSocket surface could reuse it unchanged", and this is that surface. Same routing +//! table, same per-lane sockets, same refusal to parse a reply. No ICE, no DTLS, no TURN. +//! +//! **What it is not is a teleop lane.** The rendezvous allows 1200 requests per 60 s per peer, so +//! roughly twenty a second shared with the heartbeat — four calls to install and run a policy is +//! nothing, and a 50 Hz intent stream is over budget in a second. Worse, exceeding it earns a +//! `429` on the *whole peer*, which would take the robot's own lease down with it: a client could +//! knock a robot off the rendezvous by subscribing to telemetry. Hence [`Budget`], which bounds +//! notifications and never a reply — replies are one-per-request and so already bounded by +//! whatever the client itself can afford to send. +//! +//! # Why the transport is HTTP, which is not what `remote-webrtc.md` §7 assumed +//! +//! The envelopes are the gst signalling protocol's — the same messages a LAN client exchanges — +//! but they arrive over **SSE** and are sent with **`POST /send`**, with per-hop peer and session +//! ids. So the payload stays opaque and the envelope does not: this is a translator with an +//! opaque payload rather than a relay. §3.2 has the two sides side by side. +//! +//! # Three things read out of their source that shape the code below +//! +//! - **`POST /send` before `GET /events` is a 400.** The peer does not exist until the stream +//! does — identity comes from the bearer token, and the token is bound to a peer by the +//! `/events` connection. So the stream is opened *first* and registration follows the welcome, +//! which happens to be the order §3.4 wanted anyway for a different reason. +//! - **The lease is refreshed by inbound `POST`, not by a healthy stream.** Thirty seconds, and a +//! half-open TCP connection absorbs server-pushed keepalives silently for minutes — during +//! which the robot believes it is reachable and is not. Hence [`heartbeat`-cadence] re-posts of +//! `setPeerStatus`, and hence the split-brain poll. +//! - **Only producers carrying `meta.hardware_id` are swept.** A producer without it is never +//! evicted, so a crashed daemon would leave a ghost in somebody's robot list forever. This +//! always sends one — the SoC serial, or `/etc/machine-id` where there is no serial to read. +//! +//! [`heartbeat`-cadence]: Welcome::heartbeat +//! +//! # It is a task, not a daemon +//! +//! In `mediad` rather than a `relayd` for §3.5's reasons: a separate unit would need its own copy +//! of the producer identity, its own config and its own restart story, and would still be useless +//! without `mediad` running. Nothing here touches GStreamer — the boundary `pipeline.rs`'s +//! no-panic rule lives on — and nothing here is `cfg(target_os)`-gated, so the whole of it is +//! testable on a laptop against a fake service. + +use std::path::{Path, PathBuf}; +use std::time::Duration; + +use eventsource_stream::Eventsource as _; +use futures_util::StreamExt; +use serde::{Deserialize, Serialize}; + +/// The Space the mini's fleet already registers against. §4. +pub const DEFAULT_RENDEZVOUS: &str = "https://pollen-robotics-reachy-mini-central.hf.space"; + +/// Where `updaterd` keeps the account credential. +/// +/// **A cross-daemon file format, and `hf_robot_account` owns it.** `updaterd` performs the login +/// and that crate writes the file; a test there pins the one key `read_access_token` takes out of +/// it, because the writer is what can break the contract. +/// Read on every connect attempt rather than cached: a login that happens while this task is +/// waiting has to take effect without a restart, and re-reading a small file on a path that +/// already sleeps for thirty seconds costs nothing. +pub const DEFAULT_TOKEN_PATH: &str = "/etc/robot/hf-token"; + +/// How long to wait between looks at a token file that is not there yet. +/// +/// This is the `waiting for token` state, and it is the ordinary state of a robot nobody has +/// signed in — so it must be quiet in the journal and cheap on the board. +const NO_TOKEN_POLL: Duration = Duration::from_secs(30); + +/// How long a read from the event stream may go quiet before the connection is presumed dead. +/// +/// The service emits `event: ping` after 30 s of idle, whose only job is to keep the proxy in +/// front of the Space from killing the connection. Sixty seconds is two missed pings, which is +/// what `reachy_mini`'s relay uses. §3.3. +const READ_TIMEOUT: Duration = Duration::from_secs(60); + +/// How long the welcome gets to arrive before the connection is abandoned. +const WELCOME_TIMEOUT: Duration = Duration::from_secs(20); + +/// The fallback heartbeat cadence, when the welcome names none. +/// +/// The service publishes `recommended_heartbeat_interval_seconds: 10.0` and **no `lease_seconds`** +/// — so `reachy_mini`'s middle rung, `lease_seconds / 3`, is unreachable here and is not +/// reproduced. Five seconds is a sixth of the lease, which survives a missed post. +const HEARTBEAT_FALLBACK: Duration = Duration::from_secs(5); + +/// The cadence is clamped, so a misconfigured service can neither ask for a request storm nor +/// talk us into a cadence slower than our own eviction. +const HEARTBEAT_BOUNDS: (Duration, Duration) = (Duration::from_secs(1), Duration::from_secs(60)); + +/// How often to ask the service whether it still lists this robot. §3.4, split-brain. +const STATUS_POLL: Duration = Duration::from_secs(30); + +/// How many consecutive times the service may fail to list this robot before reconnecting. +/// +/// Two rather than one: `/api/robot-status` is a separate request from the stream, and one lost +/// answer is not evidence of anything. +const MISSES_BEFORE_RECONNECT: u32 = 2; + +/// Reconnect backoff: where it starts, where it stops, and how much noise goes on top. +const BACKOFF_START: Duration = Duration::from_secs(5); +const BACKOFF_MAX: Duration = Duration::from_secs(60); +const BACKOFF_JITTER: f64 = 0.10; + +/// How long a request that is not the event stream gets. +const REQUEST_TIMEOUT: Duration = Duration::from_secs(20); + +/// How long the robot's own signalling server gets to hand over a session. +/// +/// It is in this process — `webrtcsink` runs it — so this is generous rather than tuned. What it +/// guards against is a pipeline that never reached PLAYING, which produces a server with no +/// producer and a handshake that would otherwise wait for one forever. +const LOCAL_HANDSHAKE_TIMEOUT: Duration = Duration::from_secs(10); + +/// Where the robot's own signalling server listens. +/// +/// `webrtcsink` runs it in this process with `signalling-server-host` and `-port`, so this is the +/// same 8443 `mediad`'s `--port` defaults to. Loopback, because the bridge and the server are one +/// process — a remote peer reaches this robot through the rendezvous, never through this socket. +pub const DEFAULT_LOCAL_SIGNALLING: &str = "ws://127.0.0.1:8443"; + +/// Every interval this task runs on, in one place. +/// +/// A value rather than the constants directly, and it exists for the tests: §3.4's four failure +/// modes are all *timing* failures — a lease that stops being refreshed, a service that goes on +/// answering while it has forgotten us, a fleet reconnecting in lockstep — and none of them can be +/// reproduced on demand by hand on a board. With the intervals injectable, each one is a test that +/// runs in under a second. Production always uses [`Timings::default`], which is the constants +/// above. +#[derive(Debug, Clone, Copy)] +pub struct Timings { + pub no_token_poll: Duration, + pub read_timeout: Duration, + pub welcome_timeout: Duration, + pub heartbeat_fallback: Duration, + pub heartbeat_bounds: (Duration, Duration), + pub status_poll: Duration, + pub backoff_start: Duration, + pub backoff_max: Duration, +} + +impl Default for Timings { + fn default() -> Self { + Self { + no_token_poll: NO_TOKEN_POLL, + read_timeout: READ_TIMEOUT, + welcome_timeout: WELCOME_TIMEOUT, + heartbeat_fallback: HEARTBEAT_FALLBACK, + heartbeat_bounds: HEARTBEAT_BOUNDS, + status_poll: STATUS_POLL, + backoff_start: BACKOFF_START, + backoff_max: BACKOFF_MAX, + } + } +} + +// ── what a client sees in the listing ──────────────────────────────────────── + +/// What this robot calls itself to the service. +/// +/// Free-form to the protocol and **not to the server**, which reads three of these keys — see the +/// module header on `hardware_id`. The rest are for whoever is looking at a list of robots. +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +pub struct Meta { + /// The stable-identity key: same physical robot across reinstalls, renames and new tokens. + /// + /// The server evicts an older producer of the same user carrying the same value, which is how + /// a re-flashed board or a restarted daemon stops showing up as a second robot. It is also + /// what makes this robot sweepable at all. + pub hardware_id: String, + /// What a person sees. Absent when `configd` did not answer in time, as elsewhere. + #[serde(skip_serializing_if = "Option::is_none")] + pub name: Option, + /// `microduck`, which is what lets one client list two families of robot without opening a + /// session to ask what it found. + pub kind: &'static str, + /// The release this robot is running, for the reason the local `meta` carries it. + pub release: String, + pub api_version: u32, +} + +impl Meta { + /// This robot's `meta`, from what the local producer already learned. + /// + /// `hardware_id` falls back to `/etc/machine-id` when there is no SoC serial to read — a + /// developer's laptop, or a board whose device tree has no `serial-number`. Stable per + /// install rather than per robot, which is weaker and still correct for the purpose: it keeps + /// one machine from being listed twice, and it keeps the producer sweepable. `sounds` makes + /// exactly this substitution for exactly this reason. + pub fn of(producer: &crate::producer::Producer, machine_id: Option) -> Option { + let hardware_id = producer + .serial + .clone() + .or(machine_id) + .or_else(|| read_machine_id(Path::new("/etc/machine-id")))?; + Some(Self { + hardware_id, + name: producer.name.clone(), + kind: "microduck", + release: producer.release.clone(), + api_version: producer.api_version, + }) + } +} + +fn read_machine_id(path: &Path) -> Option { + let id = std::fs::read_to_string(path).ok()?.trim().to_owned(); + (!id.is_empty()).then_some(id) +} + +// ── the wire ───────────────────────────────────────────────────────────────── + +/// What the service sends down the event stream. +/// +/// Unknown types are a variant rather than an error: this is somebody else's service and it is +/// allowed to grow messages we do not handle. The ones named here are the ones acted on. +/// +/// **`Peer` carries the whole envelope rather than its fields**, and that is the design rather +/// than laziness: an SDP or an ICE candidate passes through this process untouched, and the only +/// thing rewritten is the `sessionId` around it. Deserialising the payload would mean owning a +/// copy of a schema that belongs to WebRTC, and re-serialising it would mean re-encoding SDP that +/// arrived perfectly good. §3.2 — a translator with an opaque payload. +#[derive(Debug, Clone, PartialEq)] +enum Inbound { + Welcome(Welcome), + /// A consumer wants a session, which is what the bridge exists to serve. + StartSession { + session_id: String, + }, + /// The other side gave up, or the service ended it. + EndSession { + session_id: Option, + }, + /// SDP or ICE for a session in flight. + Peer(serde_json::Value), + Other, +} + +/// Read a message off the wire far enough to route it, and no further. +fn classify(raw: &str) -> Result { + let value: serde_json::Value = + serde_json::from_str(raw).map_err(|e| format!("unparseable message: {e}"))?; + let session_id = |value: &serde_json::Value| value["sessionId"].as_str().map(str::to_owned); + Ok(match value["type"].as_str().unwrap_or_default() { + "welcome" => Inbound::Welcome( + serde_json::from_value(value) + .map_err(|e| format!("a welcome this page cannot read: {e}"))?, + ), + "startSession" => match session_id(&value) { + Some(session_id) => Inbound::StartSession { session_id }, + // A `startSession` with no id is not something to answer: there is nothing to answer + // *about*, and inventing one would open a session the service cannot route. + None => return Err("a startSession with no sessionId".to_owned()), + }, + "endSession" => Inbound::EndSession { + session_id: session_id(&value), + }, + "peer" => Inbound::Peer(value), + _ => Inbound::Other, + }) +} + +/// The first message on a healthy stream, and the only one that has to arrive. +/// +/// **Two casings in one object**, which is the service's and not a mistake here: `peerId` is +/// camelCase like every other envelope field, and `recommended_heartbeat_interval_seconds` is +/// snake_case like every `meta` key. A blanket `rename_all` silently reads the cadence as absent +/// and falls back to five seconds — a robot that works while posting twice as often as asked, and +/// nothing anywhere says why. So that one field is named outright. +#[derive(Debug, Clone, PartialEq, Deserialize)] +#[serde(rename_all = "camelCase")] +struct Welcome { + /// The id this connection is known by, which `/api/robot-status` reports back. Kept so the + /// split-brain poll can look for *this* robot rather than for any robot. + peer_id: String, + /// The account the token belongs to, as the service resolved it. Logged once: it is the + /// answer to "whose robot does the service think this is". + #[serde(default)] + username: Option, + /// What the service asks for, in seconds. Absent on a service that does not say. + #[serde(default, rename = "recommended_heartbeat_interval_seconds")] + recommended_heartbeat_interval_seconds: Option, +} + +impl Welcome { + /// The cadence to post at: what was asked for, clamped, or the fallback. + fn heartbeat(&self, timings: &Timings) -> Duration { + let (min, max) = timings.heartbeat_bounds; + match self.recommended_heartbeat_interval_seconds { + Some(seconds) if seconds.is_finite() && seconds > 0.0 => { + Duration::from_secs_f64(seconds).clamp(min, max) + } + _ => timings.heartbeat_fallback, + } + } +} + +/// What this robot sends. `POST /send`, one object per request. +#[derive(Debug, Serialize)] +#[serde(tag = "type", rename_all = "camelCase")] +enum Outbound<'a> { + /// Registration, and every heartbeat after it: the same message, which is why the lease is + /// keyed on the request rather than on its contents. + SetPeerStatus { + roles: [&'static str; 1], + meta: &'a Meta, + }, + /// How this slice refuses a session it cannot serve yet. + #[serde(rename_all = "camelCase")] + EndSession { + session_id: &'a str, + reason: &'a str, + }, +} + +/// What `/api/robot-status` answers. Only the ids are read. +#[derive(Debug, Deserialize)] +struct RobotStatus { + #[serde(default)] + robots: Vec, +} + +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +struct RobotStatusEntry { + peer_id: String, +} + +// ── the task ───────────────────────────────────────────────────────────────── + +/// Why a connection ended, which is what decides how long to wait before the next one. +#[derive(Debug)] +enum Ended { + /// The stream closed or went quiet, or a post failed. Ordinary; back off and reconnect. + Reconnect(String), + /// The service accepted the token and stopped listing this robot anyway. §3.4. + SplitBrain, + /// The service refused the token. Backing off does not fix this — a login does — so this + /// waits on the token file instead of on a timer. + Unauthorised, + /// The credential on disk is not the one this connection is using: a `logout`, or a + /// `login --force` onto another account. + /// + /// **This is what makes `account.logout` mean anything here.** The token is read once per + /// connection, so without this the relay would go on refreshing the lease with a credential + /// its owner had deleted — a robot signed out of an account and still listed under it, which + /// is precisely the claim `remote-access-design.md` §2.6 makes about being revocable. + /// Dropping the stream is the deregistration: a clean disconnect evicts the peer at once, and + /// the 30 s sweep is only there for sockets that never report closing. + CredentialChanged, +} + +/// The relay, as the task that owns the outward connection. +pub struct Relay { + base: String, + token_path: PathBuf, + meta: Meta, + client: reqwest::Client, + timings: Timings, + local_signalling: String, + /// What the pipeline knows, for the two calls [`crate::session::run`] answers itself. + /// + /// **A watch rather than a value, because the relay starts before the answer exists.** It is + /// spawned deliberately early — a robot that appears in its owner's list and cannot stream is + /// still a robot somebody can reach to find out why — while the frame geometry is only + /// truthful *after* the pipeline is up, since which sensor mode is in force is not known until + /// something has tried to set it. So `main` hands over a receiver and fills it in later, and a + /// lane reads whatever is current when it opens. Empty means `media.video` is refused rather + /// than answered with zeros. + video: Option>>, + /// Where each service listens, for the control lane's pool. Overridable for the same reason + /// `--rendezvous-url` is: the whole of this module is meant to be exercisable on a laptop, + /// and a lane whose sockets were hardcoded to `/run/robot` could only be tested on a board. + sockets: crate::upstream::Sockets, +} + +impl Relay { + /// Build one. Fails only if the HTTP client will not build, which means no TLS stack. + pub fn new( + base: impl Into, + token_path: impl Into, + meta: Meta, + ) -> Option { + let client = reqwest::Client::builder() + .connect_timeout(Duration::from_secs(10)) + .user_agent(concat!("mediad/", env!("CARGO_PKG_VERSION"))) + .build() + .map_err(|e| { + tracing::error!( + error = %e, + "no HTTP client, so this robot cannot be reached from outside its network" + ); + }) + .ok()?; + Some(Self { + base: base.into().trim_end_matches('/').to_owned(), + token_path: token_path.into(), + meta, + client, + timings: Timings::default(), + local_signalling: DEFAULT_LOCAL_SIGNALLING.to_owned(), + video: None, + sockets: Default::default(), + }) + } + + /// Where the services this lane routes to are listening. + pub fn with_sockets(mut self, sockets: crate::upstream::Sockets) -> Self { + self.sockets = sockets; + self + } + + /// Where to read the video's geometry when a control lane opens. + /// + /// A builder rather than a fourth argument to [`Relay::new`] because the video is discovered + /// later than the relay is built, and because every test here is about the wire rather than + /// about the camera. + pub fn with_video( + mut self, + video: tokio::sync::watch::Receiver>, + ) -> Self { + self.video = Some(video); + self + } + + /// Point the bridge at a signalling server other than `webrtcsink`'s own. + /// + /// For tests, and for a `mediad` whose `--port` is not the default: the bridge connects to + /// the server this same process is running, so the two numbers have to agree. + pub fn with_local_signalling(mut self, url: impl Into) -> Self { + self.local_signalling = url.into(); + self + } + + /// Run on intervals other than the shipped ones. See [`Timings`]; tests only. + #[doc(hidden)] + pub fn with_timings(mut self, timings: Timings) -> Self { + self.timings = timings; + self + } + + /// Stay registered for as long as this process runs. + /// + /// Never returns. Every failure is a reconnect, because there is no state here worth keeping + /// across one: the service's view of this robot is rebuilt by the next `setPeerStatus`. + pub async fn run(self) { + let mut backoff = self.timings.backoff_start; + loop { + let Some(token) = self.token() else { + // At `debug`: a robot nobody has signed in is not a robot with a problem, and + // this is every thirty seconds forever. + tracing::debug!( + path = %self.token_path.display(), + "no account token yet; this robot is reachable on its own network only" + ); + tokio::time::sleep(self.timings.no_token_poll).await; + continue; + }; + + match self.session(&token).await { + Ended::Unauthorised => { + tracing::warn!( + "the rendezvous service refused this robot's account token; a new login \ + is what fixes it" + ); + tokio::time::sleep(self.timings.no_token_poll).await; + } + Ended::CredentialChanged => { + // No backoff: either there is a new token to use immediately, or there is + // none and the loop above is about to wait on the file anyway. + tracing::info!( + "this robot's account credential changed; the rendezvous connection is \ + dropped, which takes the robot out of the service's listing" + ); + backoff = self.timings.backoff_start; + } + Ended::SplitBrain => { + // Reconnect immediately rather than backing off: the connection looked + // healthy, so there is nothing to wait for, and every second here is a + // second the robot is not reachable while believing it is. + tracing::warn!( + "the service no longer lists this robot although the stream was healthy; \ + reconnecting" + ); + backoff = self.timings.backoff_start; + } + Ended::Reconnect(why) => { + tracing::info!(%why, retry_in = ?backoff, "remote access is off; will retry"); + tokio::time::sleep(jittered(backoff)).await; + backoff = (backoff * 2).min(self.timings.backoff_max); + } + } + } + } + + /// The access token, or `None` when this robot belongs to nobody. + /// + /// `hf_robot_account` owns the reading of it: it is the crate that writes the file, and the + /// TURN credentials need the same token out of the same place. + fn token(&self) -> Option { + hf_robot_account::read_access_token(&self.token_path) + } + + /// One connection: open the stream, register, then hold the lease until something breaks. + async fn session(&self, token: &str) -> Ended { + let mut events = match self.open_stream(token).await { + Ok(events) => events, + Err(ended) => return ended, + }; + + let welcome = match self.await_welcome(&mut events).await { + Ok(welcome) => welcome, + Err(ended) => return ended, + }; + + // Registered *before* anything reports this robot as reachable, so no observer can see + // "remote access enabled" while the service does not yet know the robot exists. §3.4. + if let Err(ended) = self.set_peer_status(token).await { + return ended; + } + let heartbeat = welcome.heartbeat(&self.timings); + tracing::info!( + peer_id = %welcome.peer_id, + account = welcome.username.as_deref().unwrap_or("unknown"), + ?heartbeat, + "registered with the rendezvous service; this robot is reachable from outside its \ + network" + ); + + // At most one at a time. The service gates this too (`sessionRejected` with `robot_busy`), + // so this is belt-and-braces — and it stays, because two remote peers writing into one + // intent slot is `remote-webrtc.md` §9's interleaving bug with the pad replaced by a + // second continent. §3.4. + let mut bridged: Option = None; + + // The control lane, which is independent of the one above: a consumer may hold one, the + // other, or both. Built on the first `rpc` envelope of a session and dropped with it. + let mut control: Option = None; + + let mut heartbeats = tokio::time::interval(heartbeat); + heartbeats.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay); + heartbeats.tick().await; // the first tick is immediate, and registration just happened + let mut polls = tokio::time::interval(self.timings.status_poll); + polls.tick().await; + let mut misses = 0; + + loop { + tokio::select! { + _ = heartbeats.tick() => { + // Checked here rather than on its own timer: the heartbeat is already the + // fastest thing in this loop, so a `logout` takes effect within one cadence + // — ten seconds — and re-reading a small file that often costs nothing. + if self.token().as_deref() != Some(token) { + return Ended::CredentialChanged; + } + if let Err(ended) = self.set_peer_status(token).await { + return ended; + } + } + _ = polls.tick() => { + match self.lists_us(token, &welcome.peer_id).await { + Ok(true) => misses = 0, + Ok(false) => { + misses += 1; + tracing::warn!( + misses, + peer_id = %welcome.peer_id, + "the service did not list this robot" + ); + if misses >= MISSES_BEFORE_RECONNECT { + return Ended::SplitBrain; + } + } + // A failed poll is not a miss: it says nothing about whether the service + // lists us, and treating it as one would reconnect a healthy stream + // every time the network hiccuped twice. + Err(why) => tracing::debug!(%why, "could not ask whether we are listed"), + } + } + // A session that ended on its own — the peer left, the pipeline stopped — is + // reaped here rather than being noticed the next time one is asked for, so + // `account.status` and the service agree about whether this robot is busy. + _ = async { + match bridged.as_mut() { + Some(session) => (&mut session.task).await.ok(), + // Nothing to wait for; this branch must never be the one that fires. + None => std::future::pending().await, + } + }, if bridged.is_some() => { + tracing::debug!("the bridged session finished"); + bridged = None; + } + event = tokio::time::timeout(self.timings.read_timeout, events.next()) => { + match event { + Err(_) => return Ended::Reconnect(format!( + "nothing arrived on the event stream for {:?}, which is two missed \ + pings", + self.timings.read_timeout + )), + Ok(None) => return Ended::Reconnect( + "the service closed the event stream".to_owned(), + ), + Ok(Some(Err(e))) => return Ended::Reconnect( + format!("the event stream failed: {e}"), + ), + Ok(Some(Ok(message))) => { + if let Some(ended) = + self.handle(token, message, &mut bridged, &mut control).await + { + return ended; + } + } + } + } + } + } + } + + /// `GET /events`, as a stream of parsed messages. + async fn open_stream( + &self, + token: &str, + ) -> Result> + Unpin, Ended> { + let url = format!("{}/events", self.base); + let response = self + .client + .get(&url) + .bearer_auth(token) + .header("accept", "text/event-stream") + .send() + .await + .map_err(|e| Ended::Reconnect(format!("GET {url}: {e}")))?; + + if response.status() == reqwest::StatusCode::UNAUTHORIZED { + return Err(Ended::Unauthorised); + } + if !response.status().is_success() { + return Err(Ended::Reconnect(format!( + "GET {url}: HTTP {}", + response.status() + ))); + } + + // `eventsource-stream` owns the framing: `data:` split across TCP reads, multi-line + // payloads, comments and the fields we do not use. What is left here is JSON. + let events = response + .bytes_stream() + .eventsource() + .filter_map(|event| async move { + match event { + Err(e) => Some(Err(format!("{e}"))), + // The service's keepalive carries no data and means only "still here". + Ok(event) if event.data.trim().is_empty() => None, + Ok(event) => Some(classify(&event.data)), + } + }); + Ok(Box::pin(events)) + } + + /// Read until the welcome, which is the message that says the peer now exists. + async fn await_welcome( + &self, + events: &mut (impl futures_util::Stream> + Unpin), + ) -> Result { + let deadline = tokio::time::Instant::now() + self.timings.welcome_timeout; + loop { + let event = tokio::time::timeout_at(deadline, events.next()) + .await + .map_err(|_| { + Ended::Reconnect(format!( + "no welcome within {:?}", + self.timings.welcome_timeout + )) + })?; + match event { + None => { + return Err(Ended::Reconnect( + "the stream closed before the welcome".to_owned(), + )); + } + Some(Err(why)) => { + // A message we cannot read is not a reason to drop a stream that is otherwise + // working; the welcome may be the next one. + tracing::debug!(%why, "skipping a message while waiting for the welcome"); + } + Some(Ok(Inbound::Welcome(welcome))) => return Ok(welcome), + Some(Ok(_)) => {} + } + } + } + + /// Register, and refresh the lease. The same request does both. + async fn set_peer_status(&self, token: &str) -> Result<(), Ended> { + self.send( + token, + &Outbound::SetPeerStatus { + roles: ["producer"], + meta: &self.meta, + }, + ) + .await + } + + /// One `POST /send`. + async fn send(&self, token: &str, message: &Outbound<'_>) -> Result<(), Ended> { + let url = format!("{}/send", self.base); + let response = self + .client + .post(&url) + .bearer_auth(token) + .timeout(REQUEST_TIMEOUT) + .json(message) + .send() + .await + .map_err(|e| Ended::Reconnect(format!("POST {url}: {e}")))?; + + match response.status() { + status if status.is_success() => Ok(()), + reqwest::StatusCode::UNAUTHORIZED => Err(Ended::Unauthorised), + // 400 here means the peer does not exist — the stream this token was bound to is + // gone. Reconnecting is what rebuilds it, and it is the whole reason the stream is + // opened before anything is posted. + status => Err(Ended::Reconnect(format!("POST {url}: HTTP {status}"))), + } + } + + /// Whether the service still lists this robot. §3.4, split-brain. + async fn lists_us(&self, token: &str, peer_id: &str) -> Result { + let url = format!("{}/api/robot-status", self.base); + let response = self + .client + .get(&url) + .bearer_auth(token) + .timeout(REQUEST_TIMEOUT) + .send() + .await + .map_err(|e| format!("GET {url}: {e}"))?; + if !response.status().is_success() { + return Err(format!("GET {url}: HTTP {}", response.status())); + } + let status: RobotStatus = response + .json() + .await + .map_err(|e| format!("GET {url}: {e}"))?; + Ok(status.robots.iter().any(|robot| robot.peer_id == peer_id)) + } + + /// One message from the service. `Some` ends the connection. + async fn handle( + &self, + token: &str, + message: Inbound, + bridged: &mut Option, + control: &mut Option, + ) -> Option { + match message { + // A second welcome on one stream would mean the service rebound this token, which is + // what happens when another process registers with it. Reconnecting is how we find + // out whose peer we are now. + Inbound::Welcome(_) => Some(Ended::Reconnect( + "the service sent a second welcome on the same stream".to_owned(), + )), + Inbound::StartSession { session_id } => { + if bridged.as_ref().is_some_and(Bridged::live) { + // Refused by name rather than dropped: an unanswered `startSession` leaves a + // peer waiting on a robot that is never going to answer, and the owner + // looking at a robot that reads as broken rather than busy. + tracing::info!(%session_id, "refusing a second remote session"); + return self + .send( + token, + &Outbound::EndSession { + session_id: &session_id, + reason: "this robot is already in a remote session", + }, + ) + .await + .err(); + } + + let (to_local, from_remote) = tokio::sync::mpsc::channel(64); + let hub = Hub { + client: self.client.clone(), + base: self.base.clone(), + token: token.to_owned(), + }; + let local_url = self.local_signalling.clone(); + let remote_id = session_id.clone(); + let task = tokio::spawn(async move { + if let Err(why) = bridge(hub, local_url, remote_id, from_remote).await { + tracing::warn!(%why, "a remote session could not be bridged"); + } + }); + *bridged = Some(Bridged { + remote_id: session_id, + to_local, + task, + }); + None + } + Inbound::Peer(envelope) => { + // **`rpc` first, because it needs no session to have been negotiated.** A + // consumer that never intends to speak WebRTC still had to `startSession` to get + // a session id — `handle_peer_message` drops an envelope naming a session the + // service does not know — but it has no offer to send and none to answer. So a + // control call is dispatched before the bridge is consulted, and a robot whose + // media never connected still answers. + if !envelope["rpc"].is_null() { + return self.control(token, &envelope, control).await; + } + + // The envelope is forwarded whole and the payload is never read. What decides + // where it goes is the session it names — and a `peer` for a session this robot + // is not in is dropped rather than guessed at. + let Some(session) = bridged.as_ref() else { + tracing::debug!("a peer message arrived with no session to carry it"); + return None; + }; + let names_it = envelope["sessionId"].as_str() == Some(session.remote_id.as_str()); + if !names_it { + tracing::debug!( + session = ?envelope["sessionId"].as_str(), + "a peer message for another session; dropping it" + ); + return None; + } + if session.to_local.send(envelope).await.is_err() { + tracing::debug!("the bridged session went away before its message arrived"); + *bridged = None; + } + None + } + Inbound::EndSession { session_id } => { + // Dropping `Bridged` aborts the task, which closes the local socket — that is + // what tells `webrtcsink` the consumer is gone. + if control + .as_ref() + .is_some_and(|c| session_id.as_deref().is_none_or(|id| id == c.remote_id)) + { + tracing::info!(?session_id, "the service ended the control lane"); + *control = None; + } + if bridged + .as_ref() + .is_some_and(|s| session_id.as_deref().is_none_or(|id| id == s.remote_id)) + { + tracing::info!(?session_id, "the service ended the bridged session"); + *bridged = None; + } else if control.is_none() { + tracing::debug!(?session_id, "the service ended a session we do not have"); + } + None + } + Inbound::Other => None, + } + } +} + +// ── the control lane: JSON-RPC over the rendezvous, with no candidate pair ─── + +/// How many notifications a control lane may post per window. +/// +/// The rendezvous allows 1200 requests per 60 s **per peer**, and exceeding it earns a `429` on +/// everything that token does — including the heartbeat that holds this robot's lease. So a +/// consumer subscribing to 50 Hz telemetry would not merely get a slow stream, it would take the +/// robot off its owner's listing. Half the allowance is left for the heartbeat, the status poll +/// and whatever a person is actually doing. +const NOTIFICATIONS_PER_WINDOW: u32 = 400; +const NOTIFICATION_WINDOW: Duration = Duration::from_secs(60); + +/// A sliding allowance for lines nobody asked for. +/// +/// **Replies are deliberately not subject to it.** One reply answers one request, and a request +/// cost the consumer a `POST` of its own, so replies are already bounded by whatever the consumer +/// can afford — throttling them would only break callers. Notifications have no such bound: a +/// subscription is one request and then an unbounded stream. +struct Budget { + allowance: u32, + window: Duration, + spent: u32, + dropped: u64, + since: tokio::time::Instant, +} + +impl Budget { + fn new(allowance: u32, window: Duration) -> Self { + Self { + allowance, + window, + spent: 0, + dropped: 0, + since: tokio::time::Instant::now(), + } + } + + /// Whether one notification may go out now. + fn take(&mut self) -> bool { + let now = tokio::time::Instant::now(); + if now.duration_since(self.since) >= self.window { + if self.dropped > 0 { + tracing::info!( + dropped = self.dropped, + "notifications dropped on the control lane: it is not a telemetry transport, \ + and the alternative is a rate limit that would end this robot's lease" + ); + self.dropped = 0; + } + self.spent = 0; + self.since = now; + } + if self.spent < self.allowance { + self.spent += 1; + true + } else { + self.dropped += 1; + false + } + } +} + +/// One consumer's control lane, and the task that owns both its halves. +struct Control { + /// The id the *service* knows this session by, which is what every envelope has to name. + remote_id: String, + /// Lines on their way into [`crate::session::run`]. + to_session: tokio::sync::mpsc::Sender, + task: tokio::task::JoinHandle<()>, +} + +impl Control { + fn live(&self) -> bool { + !self.task.is_finished() + } +} + +impl Drop for Control { + fn drop(&mut self) { + // The task owns the session and its pool of unix sockets; aborting it closes every one, + // which is what ends a subscription a consumer walked away from. + self.task.abort(); + } +} + +impl Relay { + /// One `peer` envelope carrying `rpc`. + /// + /// Opens the lane if this session has none yet, which is what makes a control-only consumer + /// need no cooperation from the media path: it says `startSession`, the service gives it an + /// id, and its first call is what builds everything on this side. + async fn control( + &self, + token: &str, + envelope: &serde_json::Value, + control: &mut Option, + ) -> Option { + let Some(session_id) = envelope["sessionId"].as_str() else { + tracing::debug!("an rpc envelope with no sessionId; dropping it"); + return None; + }; + + // A new session supersedes an old lane rather than being refused. Unlike a media session + // — where two consumers writing into one intent slot is `remote-webrtc.md` §9's + // interleaving bug — a lane holds no hardware: it is a socket per service, and the + // service's own concurrency gate already means one consumer at a time. + let reusable = control + .as_ref() + .is_some_and(|c| c.remote_id == session_id && c.live()); + if !reusable { + *control = Some(self.open_control(token, session_id)); + } + + // The payload goes in as the line `session::run` expects, which is the object itself + // rather than a string containing one: this transport is JSON all the way down, so + // encoding a JSON-RPC object *inside* a JSON string would be an escaping bug waiting for + // its first apostrophe. + let line = envelope["rpc"].to_string(); + let lane = control.as_ref().expect("just built"); + if lane.to_session.send(line).await.is_err() { + tracing::debug!("the control lane went away before its call arrived"); + *control = None; + } + None + } + + /// Build a lane: a session, a pool of upstream sockets, and the task that posts its answers. + fn open_control(&self, token: &str, session_id: &str) -> Control { + let (to_session, from_consumer) = tokio::sync::mpsc::channel::(64); + // Deeper than the inbound half on purpose: one call can answer with a stream, and the + // budget below would rather drop a notification than have a service block writing it. + let (to_consumer, mut from_session) = tokio::sync::mpsc::channel::(256); + + let hub = Hub { + client: self.client.clone(), + base: self.base.clone(), + token: token.to_owned(), + }; + let pool = crate::upstream::Pool::new(self.sockets.clone(), to_consumer.clone()); + // Read now rather than held as a receiver: a lane's answer to `media.video` should be + // whatever was true when the consumer connected, and a picture that changed shape + // mid-session is a `media.video` notification's job on the media path. + let video = self.video.as_ref().and_then(|watch| watch.borrow().clone()); + let remote_id = session_id.to_owned(); + let session_id = session_id.to_owned(); + + let task = tokio::spawn(async move { + let session = tokio::spawn(crate::session::run( + from_consumer, + to_consumer, + pool, + // `None` on a board with no camera, and on a lane opened before the pipeline has + // said what the video is — which is possible because the relay starts first, on + // purpose. `session::run` refuses `media.video` rather than answering with zeros. + video, + )); + + let mut budget = Budget::new(NOTIFICATIONS_PER_WINDOW, NOTIFICATION_WINDOW); + while let Some(line) = from_session.recv().await { + let Ok(payload) = serde_json::from_str::(&line) else { + // `session::run` builds every line through `duck-ipc-proto`, and a service's + // own output is forwarded verbatim — so this is a daemon emitting something + // that is not JSON, which is worth a line rather than a silent drop. + tracing::warn!(line = %line.chars().take(120).collect::(), + "a control lane answer that is not JSON; dropping it"); + continue; + }; + // An answer has an id because the call it answers had one. Everything else is a + // notification, and only notifications are rationed. + let answers = !payload["id"].is_null(); + if !answers && !budget.take() { + continue; + } + let envelope = serde_json::json!({ + "type": "peer", + "sessionId": session_id, + "rpc": payload, + }); + if let Err(why) = hub.send(&envelope).await { + // The stream this token is bound to has gone, or the service refused. Either + // way the connection loop is about to find out on its own; this lane just + // stops. + tracing::info!(%why, "a control lane could not reach the service"); + break; + } + } + session.abort(); + }); + + tracing::info!(%remote_id, "a control lane is open: JSON-RPC without a candidate pair"); + Control { + remote_id, + to_session, + task, + } + } +} + +// ── the local half: one bridged session ────────────────────────────────────── +// +// A remote consumer wants a session with this robot. On the LAN that consumer would open a +// WebSocket to `webrtcsink`'s own signalling server and ask it for one; from off the LAN it cannot +// reach that socket at all, so **this task plays the consumer on its behalf**: it opens the local +// WebSocket, asks for a session with the local producer, and then carries envelopes between the +// two sides, rewriting the one field whose value differs per hop. +// +// The roles are inverted on the two sides, and that is the whole shape of it (§3.1): to the +// rendezvous this process *is* the robot, and to `webrtcsink` it is a peer asking for a session. +// +// **One local connection per bridged session**, opened when the session starts and dropped when it +// ends. A long-lived local socket multiplexing several sessions would need the session table §3.2 +// describes; one connection per session makes that table a single pair of ids, and the concurrent +// session the table would have existed for is refused anyway — the service gates it, and §3.4 says +// to keep gating it here too. + +/// What the bridge needs to talk to the rendezvous while a session is in flight. +#[derive(Clone)] +struct Hub { + client: reqwest::Client, + base: String, + token: String, +} + +impl Hub { + /// `POST /send`, for a task that has no `Relay` to hand. + async fn send(&self, message: &serde_json::Value) -> Result<(), String> { + let url = format!("{}/send", self.base); + let response = self + .client + .post(&url) + .bearer_auth(&self.token) + .timeout(REQUEST_TIMEOUT) + .json(message) + .send() + .await + .map_err(|e| format!("POST {url}: {e}"))?; + if !response.status().is_success() { + return Err(format!("POST {url}: HTTP {}", response.status())); + } + Ok(()) + } + + /// Tell the service a session is over. Best effort: a failure here is a session that is + /// already gone, and the peer finds out when the media stops either way. + async fn end(&self, session_id: &str, reason: &str) { + let message = serde_json::json!({ + "type": "endSession", "sessionId": session_id, "reason": reason, + }); + if let Err(why) = self.send(&message).await { + tracing::debug!(%why, "could not tell the service the session ended"); + } + } +} + +/// A bridged session, from the rendezvous side's point of view. +struct Bridged { + /// The id the *service* knows this session by. + remote_id: String, + /// Envelopes from the service, on their way to the local signalling server. + to_local: tokio::sync::mpsc::Sender, + task: tokio::task::JoinHandle<()>, +} + +impl Bridged { + /// Whether the task carrying this session is still running. + fn live(&self) -> bool { + !self.task.is_finished() + } +} + +impl Drop for Bridged { + fn drop(&mut self) { + // The task owns a WebSocket and a channel; aborting it closes both, which is what tells + // `webrtcsink` the consumer went away. + self.task.abort(); + } +} + +/// Carry one session, and **tell the service when it is over however that happens**. +/// +/// The wrapper exists for that second half. A consumer whose session ends without being told sits +/// looking at a robot it believes is connecting, forever — and the failure that produces it is +/// never the ordinary path, it is an early return from somewhere in the middle. So there is one +/// place that posts `endSession` and every exit goes through it. +async fn bridge( + hub: Hub, + local_url: String, + remote_id: String, + from_remote: tokio::sync::mpsc::Receiver, +) -> Result<(), String> { + let outcome = carry(&hub, &local_url, &remote_id, from_remote).await; + match &outcome { + Ok(why) => { + hub.end(&remote_id, why).await; + tracing::info!(remote = %remote_id, %why, "the bridged session ended"); + } + Err(why) => { + tracing::warn!(remote = %remote_id, %why, "the bridged session failed"); + hub.end(&remote_id, why).await; + } + } + outcome.map(|_| ()) +} + +/// One session, from the local handshake to whichever side stops first. +/// +/// `Ok` carries why it ended, which is what the consumer is told. +async fn carry( + hub: &Hub, + local_url: &str, + remote_id: &str, + mut from_remote: tokio::sync::mpsc::Receiver, +) -> Result { + use futures_util::SinkExt; + use tokio_tungstenite::tungstenite::Message; + + let (mut socket, _) = tokio_tungstenite::connect_async(local_url) + .await + .map_err(|e| format!("{local_url}: {e}"))?; + + // The consumer's own handshake against `webrtcsink`'s server: a welcome, then ask what is + // producing, then ask that producer for a session. Exactly what the console page does over a + // LAN, which is why the page was the reference for this rather than the protocol document. + let mut producer = None; + let mut local_id = None; + let deadline = tokio::time::Instant::now() + LOCAL_HANDSHAKE_TIMEOUT; + while local_id.is_none() { + let message = tokio::time::timeout_at(deadline, socket.next()) + .await + .map_err(|_| { + format!("{local_url} did not start a session within {LOCAL_HANDSHAKE_TIMEOUT:?}") + })? + .ok_or_else(|| format!("{local_url} closed during the handshake"))? + .map_err(|e| format!("{local_url}: {e}"))?; + let Message::Text(text) = message else { + continue; + }; + + // Read as the envelope it is rather than through `classify`: two of these three types + // exist only on this side of the bridge, and giving them variants in the rendezvous's + // vocabulary would put local-only messages in a remote-only enum. + let value: serde_json::Value = + serde_json::from_str(&text).map_err(|e| format!("{local_url}: {e}"))?; + match value["type"].as_str().unwrap_or_default() { + "welcome" => { + socket + .send(Message::text(r#"{"type":"list"}"#)) + .await + .map_err(|e| format!("{local_url}: {e}"))?; + } + "list" => { + let first = value["producers"].get(0).and_then(|p| p["id"].as_str()); + let Some(id) = first else { + return Err( + "the robot's own signalling server lists no producer, so there is \ + nothing to bridge — its pipeline has not reached PLAYING" + .to_owned(), + ); + }; + producer = Some(id.to_owned()); + let request = serde_json::json!({ "type": "startSession", "peerId": id }); + socket + .send(Message::text(request.to_string())) + .await + .map_err(|e| format!("{local_url}: {e}"))?; + } + "sessionStarted" => { + local_id = value["sessionId"].as_str().map(str::to_owned); + } + _ => {} + } + } + + let local_id = local_id.expect("the loop above does not end until it is set"); + tracing::info!( + remote = %remote_id, + local = %local_id, + producer = producer.as_deref().unwrap_or("unknown"), + "a remote session is bridged to this robot's own signalling server" + ); + + // From here it is two directions and one rewritten field. + let outcome = loop { + tokio::select! { + // The local producer: SDP, ICE, or the end of the session. + message = socket.next() => { + let Some(message) = message else { + break Err("the robot's signalling server closed the session".to_owned()); + }; + let message = message.map_err(|e| format!("{local_url}: {e}"))?; + let Message::Text(text) = message else { continue }; + match classify(&text)? { + Inbound::Peer(mut envelope) => { + envelope["sessionId"] = serde_json::Value::String(remote_id.to_owned()); + hub.send(&envelope).await?; + } + Inbound::EndSession { .. } => { + break Ok("the robot ended the session".to_owned()); + } + _ => {} + } + } + // The remote consumer, by way of the event stream this session arrived on. + envelope = from_remote.recv() => { + let Some(mut envelope) = envelope else { + break Ok("the rendezvous connection went away".to_owned()); + }; + envelope["sessionId"] = serde_json::Value::String(local_id.clone()); + socket + .send(Message::text(envelope.to_string())) + .await + .map_err(|e| format!("{local_url}: {e}"))?; + } + } + }; + + // The local side is told with a message rather than a dropped socket, so `webrtcsink` frees + // its consumer immediately instead of on a timeout. The remote side is told by the caller, + // which is the only place that does it. + let farewell = serde_json::json!({ "type": "endSession", "sessionId": local_id }); + let _ = socket.send(Message::text(farewell.to_string())).await; + let _ = socket.close(None).await; + outcome +} + +/// A duration plus up to [`BACKOFF_JITTER`] of itself, so a fleet does not reconnect in lockstep. +fn jittered(base: Duration) -> Duration { + let spread = base.mul_f64(BACKOFF_JITTER); + base + spread.mul_f64(rand::random::()) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn meta() -> Meta { + Meta { + hardware_id: "3fa1c51b".to_owned(), + name: Some("olducky".to_owned()), + kind: "microduck", + release: "0.10.0".to_owned(), + api_version: duck_ipc_proto::API_VERSION, + } + } + + /// The cadence the welcome asks for is used, and a service that asks for something absurd is + /// clamped rather than obeyed. + #[test] + fn the_heartbeat_cadence_is_the_services_within_reason() { + let welcome = |seconds: Option| Welcome { + peer_id: "peer-1".to_owned(), + username: None, + recommended_heartbeat_interval_seconds: seconds, + }; + + let timings = Timings::default(); + assert_eq!( + welcome(Some(10.0)).heartbeat(&timings), + Duration::from_secs(10), + "what this service actually publishes" + ); + assert_eq!( + welcome(None).heartbeat(&timings), + HEARTBEAT_FALLBACK, + "a service that says nothing gets a sixth of the lease" + ); + assert_eq!( + welcome(Some(0.001)).heartbeat(&timings), + HEARTBEAT_BOUNDS.0, + "a request storm is refused" + ); + assert_eq!( + welcome(Some(600.0)).heartbeat(&timings), + HEARTBEAT_BOUNDS.1, + "and so is a cadence slower than the lease it is meant to refresh" + ); + assert_eq!( + welcome(Some(f64::NAN)).heartbeat(&timings), + HEARTBEAT_FALLBACK + ); + assert_eq!(welcome(Some(-1.0)).heartbeat(&timings), HEARTBEAT_FALLBACK); + } + + /// The messages this robot has to recognise, as the service spells them. + #[test] + fn the_wire_is_read_as_the_service_writes_it() { + let welcome = classify( + r#"{"type":"welcome","peerId":"p-1","username":"PierreRouanet", + "recommended_heartbeat_interval_seconds":10.0}"#, + ) + .unwrap(); + let Inbound::Welcome(welcome) = welcome else { + panic!("{welcome:?}"); + }; + assert_eq!(welcome.peer_id, "p-1"); + assert_eq!(welcome.username.as_deref(), Some("PierreRouanet")); + assert_eq!( + welcome.heartbeat(&Timings::default()), + Duration::from_secs(10) + ); + + assert_eq!( + classify(r#"{"type":"startSession","peerId":"p-2","sessionId":"s-1"}"#).unwrap(), + Inbound::StartSession { + session_id: "s-1".to_owned() + }, + ); + // A `peer` keeps its whole envelope, payload included — that is what makes this a + // translator rather than a parser. + let peer = + classify(r#"{"type":"peer","sessionId":"s-1","sdp":{"type":"offer","sdp":"v=0\r\n"}}"#) + .unwrap(); + let Inbound::Peer(envelope) = &peer else { + panic!("{peer:?}") + }; + assert_eq!(envelope["sdp"]["sdp"], "v=0\r\n"); + + // Messages this slice does not act on must not be errors: it is somebody else's service + // and it is allowed to grow. + for other in [ + r#"{"type":"list","producers":[]}"#, + r#"{"type":"peerStatusChanged","peerId":"p-1","roles":["producer"],"meta":{}}"#, + r#"{"type":"sessionRejected","reason":"robot_busy","activeApp":"whatever"}"#, + r#"{"type":"somethingAddedNextYear"}"#, + ] { + assert_eq!(classify(other).unwrap(), Inbound::Other, "{other}"); + } + } + + /// What is posted, spelled the way the server reads it. + #[test] + fn registration_says_producer_and_carries_the_stable_id() { + let meta = meta(); + let json = serde_json::to_value(Outbound::SetPeerStatus { + roles: ["producer"], + meta: &meta, + }) + .unwrap(); + + assert_eq!(json["type"], "setPeerStatus"); + assert_eq!(json["roles"][0], "producer"); + assert_eq!( + json["meta"]["hardware_id"], "3fa1c51b", + "the key the server sweeps and evicts on — snake_case, as it reads it" + ); + assert_eq!(json["meta"]["kind"], "microduck"); + assert_eq!(json["meta"]["name"], "olducky"); + + let json = serde_json::to_value(Outbound::EndSession { + session_id: "s-1", + reason: "nope", + }) + .unwrap(); + assert_eq!(json["type"], "endSession"); + assert_eq!( + json["sessionId"], "s-1", + "camelCase, as the server sends it" + ); + } + + /// A robot with no serial still gets a stable id, because a producer without one is never + /// swept — it would haunt its owner's robot list after a crash. + #[test] + fn a_board_with_no_serial_falls_back_to_the_machine_id() { + let producer = crate::producer::Producer { + name: Some("olducky".to_owned()), + serial: None, + release: "0.10.0".to_owned(), + api_version: duck_ipc_proto::API_VERSION, + }; + + let meta = Meta::of(&producer, Some("machine-1".to_owned())).expect("a stable id"); + assert_eq!(meta.hardware_id, "machine-1"); + + let with_serial = crate::producer::Producer { + serial: Some("3fa1c51b".to_owned()), + ..producer + }; + assert_eq!( + Meta::of(&with_serial, Some("machine-1".to_owned())) + .unwrap() + .hardware_id, + "3fa1c51b", + "the serial wins: it survives a reinstall, and the machine id does not" + ); + } + + #[test] + fn an_empty_machine_id_file_is_no_machine_id() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("machine-id"); + std::fs::write(&path, "\n").unwrap(); + assert_eq!(read_machine_id(&path), None); + std::fs::write(&path, " abc123 \n").unwrap(); + assert_eq!(read_machine_id(&path), Some("abc123".to_owned())); + } + + /// The token file is `updaterd`'s, and this reads one field out of it — including when the + /// record grows fields this does not know. + #[test] + fn the_credential_is_read_for_the_one_field_that_matters() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("hf-token"); + let relay = Relay::new("http://127.0.0.1:1", &path, meta()).unwrap(); + + assert_eq!( + relay.token(), + None, + "no file is a robot signed in to nobody" + ); + + std::fs::write( + &path, + r#"{"access_token":"hf_abc","refresh_token":"r","expires_at":1,"username":"x", + "something_added_later":true}"#, + ) + .unwrap(); + assert_eq!(relay.token().as_deref(), Some("hf_abc")); + + std::fs::write(&path, r#"{"refresh_token":"only"}"#).unwrap(); + assert_eq!( + relay.token(), + None, + "a record with no access token is no use" + ); + + std::fs::write(&path, "{ not json").unwrap(); + assert_eq!( + relay.token(), + None, + "and a corrupt one is signed out, not fatal" + ); + } + + // ── against a fake rendezvous service ─────────────────────────────────── + // + // The four failure modes §3.4 names are all timing failures, and this is where they are + // reproduced on demand: a service that stops listing a robot whose stream is fine, a lease + // that has to be refreshed by traffic rather than by a socket looking healthy, a session + // arriving that this slice cannot serve, and a token that is not there yet. + + /// A stand-in for `reachy_mini_central`, holding what it was told and what it will say next. + struct FakeService { + base: String, + state: std::sync::Arc, + _task: tokio::task::JoinHandle<()>, + } + + #[derive(Default)] + struct Fake { + /// Every `POST /send` body, in order. + posts: std::sync::Mutex>, + /// How many times the event stream has been opened. + streams: std::sync::atomic::AtomicUsize, + /// How many times an event stream has been dropped by the client, which is what a clean + /// disconnect looks like from here — and what evicts a producer on the real service. + closed: std::sync::atomic::AtomicUsize, + /// The bearer token each stream was opened with, in order. + bearers: std::sync::Mutex>, + /// Whether `/api/robot-status` admits this robot exists. + lists_us: std::sync::atomic::AtomicBool, + /// What the welcome asks for, and messages to push after it. + heartbeat_seconds: std::sync::Mutex>, + push: std::sync::Mutex>>, + /// A status to answer `POST /send` with instead of 200. + refuse_posts: std::sync::Mutex>, + } + + impl Fake { + fn posts(&self) -> Vec { + self.posts.lock().unwrap().clone() + } + + fn bearers(&self) -> Vec { + self.bearers.lock().unwrap().clone() + } + + fn closed(&self) -> usize { + self.closed.load(std::sync::atomic::Ordering::SeqCst) + } + + fn streams(&self) -> usize { + self.streams.load(std::sync::atomic::Ordering::SeqCst) + } + + fn of_type(&self, kind: &str) -> Vec { + self.posts() + .into_iter() + .filter(|post| post["type"] == kind) + .collect() + } + + /// Push a message down the open stream, as the service would. + fn push(&self, message: serde_json::Value) { + let sender = self.push.lock().unwrap().clone().expect("a stream is open"); + sender + .send(message.to_string()) + .expect("the stream is live"); + } + } + + /// Increments the fake's `closed` count when the stream it lives in is dropped. + struct Closing(std::sync::Arc); + + impl Drop for Closing { + fn drop(&mut self) { + self.0 + .closed + .fetch_add(1, std::sync::atomic::Ordering::SeqCst); + } + } + + const PEER_ID: &str = "peer-under-test"; + + async fn fake_service() -> FakeService { + use axum::extract::State; + use axum::routing::{get, post}; + + let state = std::sync::Arc::new(Fake::default()); + state + .lists_us + .store(true, std::sync::atomic::Ordering::SeqCst); + + let app = axum::Router::new() + .route( + "/events", + get(|State(fake): State>, + headers: axum::http::HeaderMap| async move { + fake.streams + .fetch_add(1, std::sync::atomic::Ordering::SeqCst); + fake.bearers.lock().unwrap().push( + headers + .get("authorization") + .and_then(|value| value.to_str().ok()) + .unwrap_or("") + .to_owned(), + ); + let closing = Closing(std::sync::Arc::clone(&fake)); + let (tx, mut rx) = tokio::sync::mpsc::unbounded_channel::(); + *fake.push.lock().unwrap() = Some(tx); + + let cadence = *fake.heartbeat_seconds.lock().unwrap(); + let welcome = match cadence { + Some(seconds) => serde_json::json!({ + "type": "welcome", + "peerId": PEER_ID, + "username": "PierreRouanet", + "recommended_heartbeat_interval_seconds": seconds, + }), + None => serde_json::json!({ + "type": "welcome", "peerId": PEER_ID, "username": "PierreRouanet", + }), + }; + + // The framing the real service uses: `data:` lines, and a comment-only ping + // when there is nothing to say. + let events = async_stream::stream! { + // Moved in, so it is dropped when the client hangs up — which is how the + // real service learns to evict a producer. + let _closing = closing; + yield Ok::<_, std::io::Error>(format!("data: {welcome}\n\n")); + while let Some(message) = rx.recv().await { + yield Ok(format!("data: {message}\n\n")); + } + }; + ( + [("content-type", "text/event-stream")], + axum::body::Body::from_stream(events), + ) + }), + ) + .route( + "/send", + post( + |State(fake): State>, body: String| async move { + let message: serde_json::Value = + serde_json::from_str(&body).expect("the relay posts JSON"); + fake.posts.lock().unwrap().push(message); + match *fake.refuse_posts.lock().unwrap() { + None => axum::http::StatusCode::OK, + Some(status) => axum::http::StatusCode::from_u16(status).unwrap(), + } + }, + ), + ) + .route( + "/api/robot-status", + get(|State(fake): State>| async move { + let robots = if fake.lists_us.load(std::sync::atomic::Ordering::SeqCst) { + serde_json::json!([{ "peerId": PEER_ID, "busy": false }]) + } else { + serde_json::json!([]) + }; + axum::Json(serde_json::json!({ "robots": robots })) + }), + ) + .with_state(std::sync::Arc::clone(&state)); + + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let base = format!("http://{}", listener.local_addr().unwrap()); + let task = tokio::spawn(async move { + let _ = axum::serve(listener, app).await; + }); + FakeService { + base, + state, + _task: task, + } + } + + /// Intervals small enough that a test does not wait for a robot's afternoon. + fn brisk() -> Timings { + Timings { + no_token_poll: Duration::from_millis(50), + read_timeout: Duration::from_secs(5), + welcome_timeout: Duration::from_secs(2), + heartbeat_fallback: Duration::from_millis(50), + heartbeat_bounds: (Duration::from_millis(20), Duration::from_secs(60)), + status_poll: Duration::from_millis(50), + backoff_start: Duration::from_millis(20), + backoff_max: Duration::from_millis(50), + } + } + + fn signed_in(dir: &tempfile::TempDir) -> PathBuf { + let path = dir.path().join("hf-token"); + std::fs::write( + &path, + r#"{"access_token":"hf_abc","username":"PierreRouanet"}"#, + ) + .unwrap(); + path + } + + /// Wait for something to become true, or fail saying what never happened. + async fn until(what: &str, mut ready: impl FnMut() -> bool) { + for _ in 0..200 { + if ready() { + return; + } + tokio::time::sleep(Duration::from_millis(25)).await; + } + panic!("{what} did not happen"); + } + + /// The whole of slice 2: the stream opens, the welcome arrives, the robot registers as a + /// producer, and the lease keeps being refreshed after that. + /// + /// The ordering assertion is the one worth having: **nothing is posted before the welcome**. + /// `POST /send` on this service is a 400 until the token has been bound to a peer by the + /// event stream, so a relay that registered first would work only by accident of scheduling. + #[tokio::test] + async fn it_registers_as_a_producer_and_holds_the_lease() { + let dir = tempfile::tempdir().unwrap(); + let service = fake_service().await; + *service.state.heartbeat_seconds.lock().unwrap() = Some(0.05); + + let relay = Relay::new(&service.base, signed_in(&dir), meta()) + .unwrap() + .with_timings(brisk()); + let task = tokio::spawn(relay.run()); + + until("registration", || { + !service.state.of_type("setPeerStatus").is_empty() + }) + .await; + + let first = service.state.of_type("setPeerStatus")[0].clone(); + assert_eq!(first["roles"][0], "producer"); + assert_eq!(first["meta"]["hardware_id"], "3fa1c51b"); + assert_eq!(first["meta"]["kind"], "microduck"); + assert_eq!( + service + .state + .streams + .load(std::sync::atomic::Ordering::SeqCst), + 1, + "one stream, and it was opened before anything was posted" + ); + + // The lease is refreshed by traffic, not by a socket that looks healthy: a service that + // saw one post and then silence would evict this robot after thirty seconds. + until("a second heartbeat", || { + service.state.of_type("setPeerStatus").len() >= 3 + }) + .await; + task.abort(); + } + + /// Split-brain: the stream is healthy and the service has forgotten us. + /// + /// Nothing in the connection notices, which is the entire problem — so the poll is what + /// notices, and two consecutive misses force a reconnect. One miss must not: a single lost + /// answer says nothing about whether we are listed. + #[tokio::test] + async fn a_service_that_stops_listing_this_robot_is_reconnected() { + let dir = tempfile::tempdir().unwrap(); + let service = fake_service().await; + + let relay = Relay::new(&service.base, signed_in(&dir), meta()) + .unwrap() + .with_timings(brisk()); + let task = tokio::spawn(relay.run()); + until("the first stream", || { + service + .state + .streams + .load(std::sync::atomic::Ordering::SeqCst) + >= 1 + }) + .await; + + service + .state + .lists_us + .store(false, std::sync::atomic::Ordering::SeqCst); + + until("a reconnect", || { + service + .state + .streams + .load(std::sync::atomic::Ordering::SeqCst) + >= 2 + }) + .await; + task.abort(); + } + + /// A robot nobody has signed in does not talk to anybody, and starts as soon as it is. + /// + /// This is the ordinary state of a duck out of a box, so it must be quiet — and the token has + /// to be picked up without a restart, because the login that writes it happens over BLE while + /// this task is already running. + #[tokio::test] + async fn nothing_happens_until_the_robot_belongs_to_somebody() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("hf-token"); + let service = fake_service().await; + + let relay = Relay::new(&service.base, &path, meta()) + .unwrap() + .with_timings(brisk()); + let task = tokio::spawn(relay.run()); + + tokio::time::sleep(Duration::from_millis(200)).await; + assert_eq!( + service + .state + .streams + .load(std::sync::atomic::Ordering::SeqCst), + 0, + "a robot with no account must not reach the service at all" + ); + + // The login lands, over BLE, while this is running. + std::fs::write(&path, r#"{"access_token":"hf_abc"}"#).unwrap(); + until("registration after a login", || { + !service.state.of_type("setPeerStatus").is_empty() + }) + .await; + task.abort(); + } + + /// Signing the robot out takes it out of the service's listing, promptly. + /// + /// The token is read once per connection, so nothing about a `logout` reaches this task on its + /// own: without the check on the heartbeat tick the relay would go on refreshing the lease + /// with a credential its owner had deleted — a robot signed out of an account and still + /// listed under it. §2.6 claims `account.logout` is one of the three things that make a LAN + /// peer's login acceptable, so it has to be true here and not only in `updaterd`. + /// + /// Dropping the stream *is* the deregistration: a clean disconnect evicts the peer at once on + /// the real service, and the 30 s sweep exists only for sockets that never report closing. + #[tokio::test] + async fn signing_the_robot_out_drops_the_connection() { + let dir = tempfile::tempdir().unwrap(); + let path = signed_in(&dir); + let service = fake_service().await; + *service.state.heartbeat_seconds.lock().unwrap() = Some(0.05); + + let relay = Relay::new(&service.base, &path, meta()) + .unwrap() + .with_timings(brisk()); + let task = tokio::spawn(relay.run()); + until("registration", || { + !service.state.of_type("setPeerStatus").is_empty() + }) + .await; + + // `account logout` on the robot is exactly this. + std::fs::remove_file(&path).unwrap(); + + until("the stream to be dropped", || service.state.closed() >= 1).await; + let posts_when_signed_out = service.state.of_type("setPeerStatus").len(); + tokio::time::sleep(Duration::from_millis(300)).await; + assert_eq!( + service.state.of_type("setPeerStatus").len(), + posts_when_signed_out, + "a robot signed out must stop refreshing its lease" + ); + assert_eq!( + service.state.streams(), + 1, + "and must not reconnect: there is nothing to connect with" + ); + task.abort(); + } + + /// Signing in to a *different* account moves the robot, rather than leaving it where it was. + /// + /// `login --force` is how a robot changes hands, and the connection it changes hands over is + /// authenticated by the old owner's token. So the same check that notices a logout has to + /// notice a replacement, and reconnect with the new credential. + #[tokio::test] + async fn a_relogin_reconnects_with_the_new_token() { + let dir = tempfile::tempdir().unwrap(); + let path = signed_in(&dir); + let service = fake_service().await; + *service.state.heartbeat_seconds.lock().unwrap() = Some(0.05); + + let relay = Relay::new(&service.base, &path, meta()) + .unwrap() + .with_timings(brisk()); + let task = tokio::spawn(relay.run()); + until("registration", || { + !service.state.of_type("setPeerStatus").is_empty() + }) + .await; + assert_eq!(service.state.bearers()[0], "Bearer hf_abc"); + + std::fs::write(&path, r#"{"access_token":"hf_somebody_else"}"#).unwrap(); + + until("a reconnect on the new token", || { + service.state.bearers().len() >= 2 + }) + .await; + assert_eq!( + service.state.bearers()[1], + "Bearer hf_somebody_else", + "the second connection belongs to whoever the robot belongs to now" + ); + assert!( + service.state.closed() >= 1, + "and the first one was closed, which is what un-lists the robot for the old account" + ); + task.abort(); + } + + /// A token the service refuses is not something backing off can fix. + /// + /// It waits on the file instead: the remedy is a login, and hammering a 401 every five + /// seconds until somebody performs one is a request storm against somebody else's Space. + #[tokio::test] + async fn a_refused_token_waits_for_a_new_one_rather_than_hammering() { + let dir = tempfile::tempdir().unwrap(); + let service = fake_service().await; + *service.state.refuse_posts.lock().unwrap() = Some(401); + + let relay = Relay::new(&service.base, signed_in(&dir), meta()) + .unwrap() + .with_timings(Timings { + no_token_poll: Duration::from_secs(30), + ..brisk() + }); + let task = tokio::spawn(relay.run()); + + until("the first attempt", || { + !service.state.of_type("setPeerStatus").is_empty() + }) + .await; + tokio::time::sleep(Duration::from_millis(300)).await; + assert_eq!( + service.state.of_type("setPeerStatus").len(), + 1, + "a 401 must not be retried on the reconnect timer" + ); + task.abort(); + } + + // ── the local half ────────────────────────────────────────────────────── + // + // A stand-in for `webrtcsink`'s own signalling server, which is the one thing about this + // module that cannot be a channel: the bridge is a WebSocket client, and the framing is part + // of what it gets wrong if it gets anything wrong. + + // ── the control lane ──────────────────────────────────────────────────── + // + // The property under test is the one the lane exists for: **a call crosses with no candidate + // pair, no offer and no answer.** Every test here pushes an `rpc` envelope at a relay whose + // media path was never negotiated, which is exactly the state a consumer behind a NAT is left + // in while §6's TURN endpoint has no DNS. + + /// A daemon that reads one line and replies with the next canned one, remembering what it saw. + /// + /// Lifted from `session.rs`'s tests rather than shared, for now: two fakes of five lines are + /// cheaper than a test-support surface that both have to agree on. + fn fake_daemon( + path: &Path, + replies: Vec, + ) -> tokio::sync::mpsc::UnboundedReceiver { + use tokio::io::{AsyncBufReadExt as _, AsyncWriteExt as _, BufReader}; + + let listener = std::os::unix::net::UnixListener::bind(path).unwrap(); + listener.set_nonblocking(true).unwrap(); + let listener = tokio::net::UnixListener::from_std(listener).unwrap(); + let (seen_tx, seen_rx) = tokio::sync::mpsc::unbounded_channel(); + + tokio::spawn(async move { + while let Ok((stream, _)) = listener.accept().await { + let seen = seen_tx.clone(); + let mut replies = replies.clone().into_iter(); + tokio::spawn(async move { + let (read, mut write) = stream.into_split(); + let mut lines = BufReader::new(read).lines(); + while let Ok(Some(line)) = lines.next_line().await { + let _ = seen.send(line); + if let Some(reply) = replies.next() { + let _ = write.write_all(format!("{reply}\n").as_bytes()).await; + let _ = write.flush().await; + } + } + }); + } + }); + seen_rx + } + + /// Sockets pointing at a directory, so a lane can be driven on a laptop. + fn sockets_in(dir: &Path) -> crate::upstream::Sockets { + crate::upstream::Sockets { + updater: dir.join("updater.sock"), + robot: dir.join("robot.sock"), + config: dir.join("config.sock"), + pad: dir.join("pad.sock"), + tof: dir.join("tof.sock"), + } + } + + /// **A call crosses the rendezvous and its answer comes back, with no WebRTC anywhere.** + /// + /// This is the whole of the lane: `POST /send {type:peer, rpc}` in, a JSON-RPC line out to + /// the service that owns the answer, and the answer back out as another `peer` envelope. No + /// `startSession` is bridged, no offer is exchanged, and nothing in the path can be defeated + /// by a NAT — which is the point, because the media path currently can be. + #[tokio::test] + async fn a_call_crosses_the_rendezvous_with_no_candidate_pair() { + let dir = tempfile::tempdir().unwrap(); + let service = fake_service().await; + let sockets = sockets_in(dir.path()); + + // `robot.policies` is routed to `robotd`, and this is what a duck answers with. + let answer = serde_json::json!({ + "jsonrpc": "2.0", "id": 1, + "result": { "mode": "walk", "enabled": true, "slots": [] }, + }) + .to_string(); + let mut robotd = fake_daemon(&sockets.robot, vec![answer.clone()]); + + let relay = Relay::new(&service.base, signed_in(&dir), meta()) + .unwrap() + .with_timings(brisk()) + .with_sockets(sockets); + let task = tokio::spawn(relay.run()); + until("registration", || { + !service.state.of_type("setPeerStatus").is_empty() + }) + .await; + + service.state.push(serde_json::json!({ + "type": "peer", + "sessionId": "remote-session-1", + "rpc": { "jsonrpc": "2.0", "id": 1, "method": "robot.policies", "params": {} }, + })); + + // The daemon sees the line the consumer wrote, verbatim: this transport routes and does + // not rewrite. + let asked = tokio::time::timeout(Duration::from_secs(5), robotd.recv()) + .await + .expect("robotd was asked within five seconds") + .expect("a line"); + let asked: serde_json::Value = serde_json::from_str(&asked).unwrap(); + assert_eq!(asked["method"], "robot.policies"); + assert_eq!(asked["id"], 1); + + // And the answer comes back as a `peer` envelope naming the same session. + until("the answer to reach the service", || { + service + .state + .of_type("peer") + .iter() + .any(|posted| !posted["rpc"].is_null()) + }) + .await; + let posted = service + .state + .of_type("peer") + .into_iter() + .find(|posted| !posted["rpc"].is_null()) + .unwrap(); + assert_eq!( + posted["sessionId"], "remote-session-1", + "the answer names the session the service routes on" + ); + assert_eq!( + posted["rpc"], + serde_json::from_str::(&answer).unwrap(), + "and the payload is the daemon's own line, unparsed and unwrapped" + ); + task.abort(); + } + + /// A method this transport refuses is refused *here*, without reaching a daemon. + /// + /// `route::permits` is what says so, and it is the same table the datachannel uses — so this + /// asserts the lane consults it rather than that the answer is what it is. A lane that + /// forwarded everything would make the rendezvous a way around a per-transport rule, and the + /// pairing PIN is the one that would matter: a peer that can rewrite it locks a phone out of + /// the recovery path. + #[tokio::test] + async fn the_lane_refuses_what_the_route_table_refuses() { + let dir = tempfile::tempdir().unwrap(); + let service = fake_service().await; + let sockets = sockets_in(dir.path()); + // Deliberately no daemon at all: a refusal must not depend on one being there. + + let relay = Relay::new(&service.base, signed_in(&dir), meta()) + .unwrap() + .with_timings(brisk()) + .with_sockets(sockets); + let task = tokio::spawn(relay.run()); + until("registration", || { + !service.state.of_type("setPeerStatus").is_empty() + }) + .await; + + service.state.push(serde_json::json!({ + "type": "peer", + "sessionId": "remote-session-1", + "rpc": { "jsonrpc": "2.0", "id": 7, "method": "system.pairingPin", "params": {} }, + })); + + until("the refusal", || { + service + .state + .of_type("peer") + .iter() + .any(|posted| !posted["rpc"]["error"].is_null()) + }) + .await; + let posted = service + .state + .of_type("peer") + .into_iter() + .find(|posted| !posted["rpc"]["error"].is_null()) + .unwrap(); + assert_eq!( + posted["rpc"]["id"], 7, + "a refusal answers the call that earned it" + ); + task.abort(); + } + + /// `media.video` is refused rather than answered with zeros when there is no picture. + /// + /// The lane can open before the pipeline has said what the video is — the relay starts first, + /// on purpose — and it carries no media of its own in any case. A consumer told the frame is + /// 0×0 and upright has been handed a wrong number that looks like a right one; told there is + /// no video, it can act. + #[tokio::test] + async fn a_lane_with_no_picture_says_so() { + let dir = tempfile::tempdir().unwrap(); + let service = fake_service().await; + let relay = Relay::new(&service.base, signed_in(&dir), meta()) + .unwrap() + .with_timings(brisk()) + .with_sockets(sockets_in(dir.path())); + let task = tokio::spawn(relay.run()); + until("registration", || { + !service.state.of_type("setPeerStatus").is_empty() + }) + .await; + + service.state.push(serde_json::json!({ + "type": "peer", + "sessionId": "remote-session-1", + "rpc": { "jsonrpc": "2.0", "id": 3, "method": "media.video", "params": {} }, + })); + + until("the answer", || { + service + .state + .of_type("peer") + .iter() + .any(|posted| !posted["rpc"].is_null()) + }) + .await; + let posted = service + .state + .of_type("peer") + .into_iter() + .find(|posted| !posted["rpc"].is_null()) + .unwrap(); + assert!( + posted["rpc"]["error"]["message"] + .as_str() + .unwrap_or_default() + .contains("not publishing video"), + "said {:?}", + posted["rpc"] + ); + task.abort(); + } + + /// What the fake local server saw and said. + #[derive(Default)] + struct Local { + /// Everything the bridge sent it, in order. + seen: std::sync::Mutex>, + /// A sender for pushing messages *to* the bridge, once it has connected. + push: std::sync::Mutex>>, + } + + impl Local { + fn seen(&self) -> Vec { + self.seen.lock().unwrap().clone() + } + + fn of_type(&self, kind: &str) -> Vec { + self.seen() + .into_iter() + .filter(|m| m["type"] == kind) + .collect() + } + + fn push(&self, message: serde_json::Value) { + let sender = self + .push + .lock() + .unwrap() + .clone() + .expect("the bridge has connected"); + sender + .send(message.to_string()) + .expect("the bridge is listening"); + } + } + + const LOCAL_PRODUCER: &str = "the-robots-own-producer"; + const LOCAL_SESSION: &str = "local-session-1"; + + /// A signalling server that answers the consumer handshake and then relays. + /// + /// `producers` controls the one interesting failure: a server with none, which is what a + /// pipeline that never reached PLAYING looks like from here. + async fn fake_signalling(producers: bool) -> (String, std::sync::Arc) { + use axum::extract::State; + use axum::extract::ws::{Message, WebSocketUpgrade}; + use axum::routing::any; + + let local = std::sync::Arc::new(Local::default()); + + let app = axum::Router::new() + .route( + "/", + any(move |upgrade: WebSocketUpgrade, + State(local): State>| async move { + upgrade.on_upgrade(move |mut socket| async move { + let (tx, mut rx) = tokio::sync::mpsc::unbounded_channel::(); + *local.push.lock().unwrap() = Some(tx); + + // The welcome, unprompted, as the real server sends it. + let welcome = serde_json::json!({ "type": "welcome", "peerId": "the-bridge" }); + if socket.send(Message::text(welcome.to_string())).await.is_err() { + return; + } + + loop { + tokio::select! { + incoming = socket.recv() => { + let Some(Ok(Message::Text(text))) = incoming else { return }; + let message: serde_json::Value = + serde_json::from_str(&text).expect("the bridge sends JSON"); + let kind = message["type"].as_str().unwrap_or("").to_owned(); + local.seen.lock().unwrap().push(message); + + let answer = match kind.as_str() { + "list" => Some(serde_json::json!({ + "type": "list", + "producers": if producers { + serde_json::json!([{ "id": LOCAL_PRODUCER, "meta": {} }]) + } else { + serde_json::json!([]) + }, + })), + "startSession" => Some(serde_json::json!({ + "type": "sessionStarted", + "peerId": LOCAL_PRODUCER, + "sessionId": LOCAL_SESSION, + })), + _ => None, + }; + if let Some(answer) = answer + && socket.send(Message::text(answer.to_string())).await.is_err() + { + return; + } + } + pushed = rx.recv() => { + let Some(pushed) = pushed else { return }; + if socket.send(Message::text(pushed)).await.is_err() { + return; + } + } + } + } + }) + }), + ) + .with_state(std::sync::Arc::clone(&local)); + + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let url = format!("ws://{}/", listener.local_addr().unwrap()); + tokio::spawn(async move { + let _ = axum::serve(listener, app).await; + }); + (url, local) + } + + /// **The whole of slice 4: a session's envelopes cross, and their ids are rewritten.** + /// + /// The two assertions that matter are the two rewrites. An offer from the robot's own + /// producer must reach the service carrying the *remote* session id, because that is the id + /// the service routes on — and the consumer's answer must reach the local server carrying the + /// *local* one, for the same reason on the other side. Getting either backwards produces a + /// session where signalling is exchanged and no media ever flows, with nothing in any log to + /// say why, which is why this is pinned rather than tried by hand. + #[tokio::test] + async fn a_remote_session_is_bridged_to_the_local_signalling_server() { + let dir = tempfile::tempdir().unwrap(); + let service = fake_service().await; + let (local_url, local) = fake_signalling(true).await; + + let relay = Relay::new(&service.base, signed_in(&dir), meta()) + .unwrap() + .with_timings(brisk()) + .with_local_signalling(&local_url); + let task = tokio::spawn(relay.run()); + until("registration", || { + !service.state.of_type("setPeerStatus").is_empty() + }) + .await; + + // A consumer asks the service for a session with this robot. + service.state.push(serde_json::json!({ + "type": "startSession", "peerId": "a-consumer", "sessionId": "remote-session-1", + })); + + // The bridge plays the consumer on the local side: welcome, list, startSession. + until("the local handshake", || { + !local.of_type("startSession").is_empty() + }) + .await; + assert_eq!( + local.of_type("startSession")[0]["peerId"], + LOCAL_PRODUCER, + "the session is asked of the robot's own producer" + ); + + // The producer offers. `webrtcsink` knows what it is sending, so the offer comes from it. + local.push(serde_json::json!({ + "type": "peer", + "sessionId": LOCAL_SESSION, + "sdp": { "type": "offer", "sdp": "v=0\r\no=- 1 2 IN IP4 127.0.0.1\r\n" }, + })); + + until("the offer to reach the service", || { + !service.state.of_type("peer").is_empty() + }) + .await; + let forwarded = service.state.of_type("peer")[0].clone(); + assert_eq!( + forwarded["sessionId"], "remote-session-1", + "outbound envelopes carry the id the *service* routes on" + ); + assert_eq!( + forwarded["sdp"]["sdp"], "v=0\r\no=- 1 2 IN IP4 127.0.0.1\r\n", + "and the payload is untouched — this process never parses SDP" + ); + + // The consumer answers, by way of the service. + service.state.push(serde_json::json!({ + "type": "peer", + "sessionId": "remote-session-1", + "sdp": { "type": "answer", "sdp": "v=0\r\nanswer\r\n" }, + })); + + until("the answer to reach the robot", || { + !local.of_type("peer").is_empty() + }) + .await; + let delivered = local.of_type("peer")[0].clone(); + assert_eq!( + delivered["sessionId"], LOCAL_SESSION, + "inbound envelopes carry the id the robot's own server routes on" + ); + assert_eq!(delivered["sdp"]["sdp"], "v=0\r\nanswer\r\n"); + + task.abort(); + } + + /// A second consumer is refused by name while one is being carried. + /// + /// The service gates this itself, so this is the belt-and-braces §3.4 asks for — and the + /// reason it stays is that two remote peers writing into one intent slot is the interleaving + /// bug `remote-webrtc.md` §9 defers, with a second continent instead of a gamepad. + #[tokio::test] + async fn a_second_remote_session_is_refused_while_one_is_live() { + let dir = tempfile::tempdir().unwrap(); + let service = fake_service().await; + let (local_url, local) = fake_signalling(true).await; + + let relay = Relay::new(&service.base, signed_in(&dir), meta()) + .unwrap() + .with_timings(brisk()) + .with_local_signalling(&local_url); + let task = tokio::spawn(relay.run()); + until("registration", || { + !service.state.of_type("setPeerStatus").is_empty() + }) + .await; + + service.state.push(serde_json::json!({ + "type": "startSession", "peerId": "first", "sessionId": "remote-1", + })); + until("the first session", || { + !local.of_type("startSession").is_empty() + }) + .await; + + service.state.push(serde_json::json!({ + "type": "startSession", "peerId": "second", "sessionId": "remote-2", + })); + until("a refusal", || { + !service.state.of_type("endSession").is_empty() + }) + .await; + + let refusal = service.state.of_type("endSession")[0].clone(); + assert_eq!( + refusal["sessionId"], "remote-2", + "the second one is what is refused" + ); + assert!( + refusal["reason"].as_str().unwrap().contains("already"), + "and the reason says the robot is busy rather than broken: {refusal}" + ); + assert_eq!( + local.of_type("startSession").len(), + 1, + "the robot's own server was asked for one session, not two" + ); + task.abort(); + } + + /// A robot whose pipeline never started has no producer to bridge to, and says so. + /// + /// The peer is told the session is over rather than left waiting on media that is never + /// coming — the failure this shape has to avoid is a client that looks connected forever. + #[tokio::test] + async fn a_robot_with_no_producer_ends_the_session_rather_than_hanging() { + let dir = tempfile::tempdir().unwrap(); + let service = fake_service().await; + let (local_url, _local) = fake_signalling(false).await; + + let relay = Relay::new(&service.base, signed_in(&dir), meta()) + .unwrap() + .with_timings(brisk()) + .with_local_signalling(&local_url); + let task = tokio::spawn(relay.run()); + until("registration", || { + !service.state.of_type("setPeerStatus").is_empty() + }) + .await; + + service.state.push(serde_json::json!({ + "type": "startSession", "peerId": "a-consumer", "sessionId": "remote-session-1", + })); + + until("the session to be ended", || { + !service.state.of_type("endSession").is_empty() + }) + .await; + assert_eq!( + service.state.of_type("endSession")[0]["sessionId"], + "remote-session-1" + ); + task.abort(); + } + + /// Jitter is added, and it never shortens the wait. + #[test] + fn backoff_jitter_only_ever_adds() { + for _ in 0..100 { + let waited = jittered(Duration::from_secs(10)); + assert!(waited >= Duration::from_secs(10), "{waited:?}"); + assert!(waited <= Duration::from_secs(11), "{waited:?}"); + } + } +} diff --git a/mediad/src/route.rs b/mediad/src/route.rs index 47fa7421..977cc11a 100644 --- a/mediad/src/route.rs +++ b/mediad/src/route.rs @@ -82,7 +82,7 @@ fn permits(call: &proto::Call) -> bool { // camera: it is looking at the robot, which is precisely what a phone in the room over // Bluetooth was not. Permitted for that reason, and it would be worth revisiting if a // control-only session without video ever becomes a normal thing. - RobotEnable(_) | RobotInit | RobotRelax => true, + RobotEnable(_) | RobotInit | RobotRelax | RobotRebootMotors(_) => true, // `robot.stop` is permitted here and refused over BLE, and the difference is honesty // rather than authority. BLE's objection was that a stop button over "an unbonded, @@ -141,6 +141,9 @@ fn permits(call: &proto::Call) -> bool { // reads below, and a remote client watching a gait misbehave has an obvious use for it. RobotPolicies => true, + // The robot's static geometry, for a mapper on the other end of the video: a read. + RobotModel => true, + // Re-reading the slots goes with loading one: a client that can change what drives the // robot wants the case where something else changed it too. RobotReloadPolicies => true, @@ -159,6 +162,10 @@ fn permits(call: &proto::Call) -> bool { // neighbour can already replace with `robot.loadPolicy`. PolicyFetch(_) | PolicyInstall(_) => true, + // The detector's set, the same way. Installing one restarts *this* daemon, which ends the + // session that asked — the answer is sent before the restart, and the peer reconnects. + DetectorCheck | DetectorInstall(_) => true, + // ── the account, permitted, and this one is worth reading ──────────── // // The console is the obvious place to sign a robot in from: it is a page with the robot @@ -204,6 +211,8 @@ fn permits(call: &proto::Call) -> bool { // "it will be through `mediad`'s video path, where depth belongs next to the frame it // annotates". TofStream => true, + // The head IMU rides the same video path, for the same reason: it annotates the frames. + HeadImuStream => true, // ── reading the robot's software ───────────────────────────────────── // @@ -214,6 +223,11 @@ fn permits(call: &proto::Call) -> bool { // ── identity and status ───────────────────────────────────────────── SystemInfo | SystemServices | SystemSetName(_) => true, + // A daemon's journal tail. Read-only, and the question that follows a unit reported as + // `failed` — which `SystemServices` above can now say and could not explain. Permitted + // here for the reason `Show` is: a datachannel has room for a reply BLE has to trim, + // so the console is the transport where a whole screenful is cheap. + SystemLogs(_) => true, // Drops this session, and unlike an update leaves nothing mid-transition: the robot comes // back and the client reconnects. It is what you offer a confused robot. SystemReboot => true, @@ -379,6 +393,9 @@ mod tests { // a decision. proto::method::POLICY_INSTALL, proto::method::POLICY_FETCH, + // Replacing the detector, which restarts this daemon. Permitted for the policy + // set's reason; the session ends and the peer reconnects. + proto::method::DETECTOR_INSTALL, // Binding this robot to a Hugging Face account, and unbinding it. The argument // is in the table above — briefly: the console is where somebody would sign a // robot in, a robot that already belongs to somebody refuses without `force`, @@ -432,6 +449,7 @@ mod tests { | proto::Call::RobotStop | proto::Call::RobotSubscribe(_) | proto::Call::TofStream + | proto::Call::HeadImuStream | proto::Call::PadInput ); if wanted { @@ -497,6 +515,8 @@ mod tests { query: "microduck".to_owned(), }), proto::Call::PolicyInstall(proto::PolicyInstallParams::default()), + proto::Call::DetectorCheck, + proto::Call::DetectorInstall(proto::PolicyInstallParams::default()), ] { assert!( matches!(route_for(&call), Route::To(..)), diff --git a/mediad/src/session.rs b/mediad/src/session.rs index 2b013b42..6c65132e 100644 --- a/mediad/src/session.rs +++ b/mediad/src/session.rs @@ -40,16 +40,34 @@ use crate::upstream::Pool; /// and the console showed a sideways picture with nothing in the log to say why. The push is kept /// as a courtesy for a client that only listens; `media.video` as a *call* is what the console uses, /// because a question it asks when it is ready cannot arrive too early. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[derive(Debug, Clone, PartialEq)] pub struct Video { pub width: u32, pub height: u32, /// Degrees clockwise the camera is mounted from upright. pub rotate: u32, + /// The camera's geometry, for a consumer that has to turn pixels into directions — SLAM, + /// visual odometry, "how far is that". `None` when there is no camera or its mode is not one + /// `crate::camera` knows the field of view of, which is a robot that should not be believed + /// rather than one to guess about. + pub intrinsics: Option, } -/// The method a peer asks with. Answered here rather than routed: no service owns it. +/// The methods a peer asks with. Answered here rather than routed: no service owns them. const VIDEO_METHOD: &str = "media.video"; +const STREAM_METHOD: &str = "media.stream"; + +/// What this daemon can answer about its own media, when it has a pipeline to answer from. +/// +/// One handle rather than two Options, because the two arrive together: both are known only once +/// the pipeline is up — `sensor_mode()` is not truthful before then, and there are no frames to +/// encode either — and both are absent for the same reasons, a board with no camera or a lane +/// opened before the pipeline started. A peer asking either gets a refusal that says which. +#[derive(Clone)] +pub struct Media { + pub video: Video, + pub streamer: std::sync::Arc, +} /// What the video is, told to a peer once when its channel opens. /// @@ -61,29 +79,59 @@ const VIDEO_METHOD: &str = "media.video"; /// /// A notification, with no id, because the page already treats an id-less line as something that /// streams (`robot.state` is the other one) — no new mechanism at either end. -pub fn video_notification(video: Video) -> String { - let Video { - width, - height, - rotate, - } = video; - format!( - r#"{{"jsonrpc":"2.0","method":"{VIDEO_METHOD}","params":{{"width":{width},"height":{height},"rotate":{rotate}}}}}"# - ) +pub fn video_notification(video: &Video) -> String { + // Built with `serde_json` rather than `format!` since it gained a nested object: one escaping + // mistake in a hand-written line is a peer that cannot parse anything this daemon says. + serde_json::json!({ + "jsonrpc": "2.0", + "method": VIDEO_METHOD, + "params": video_params(video), + }) + .to_string() +} + +/// The video's description, which the notification pushes and the call answers with. +/// +/// One function for both, because a peer that asks and a peer that listens must be told the same +/// thing — and the console does both, a push when the channel opens and a call when it is ready. +fn video_params(video: &Video) -> serde_json::Value { + let mut params = serde_json::json!({ + "width": video.width, + "height": video.height, + "rotate": video.rotate, + // The two clocks at one instant: RTCP sender reports state RTP time in wall-clock + // (`real_ns`), `robot.state`/`tof.frame` stamp with `mono_ns`'s clock. A peer that has both + // can put the picture on the robot's axis. + "mono_ns": proto::clock::monotonic_ns(), + "real_ns": proto::clock::realtime_ns(), + }); + // Absent rather than null when the geometry is unknown: a consumer reading a missing key knows + // it must calibrate, where one reading `null` has to be told what that meant. + if let Some(intrinsics) = &video.intrinsics { + params["intrinsics"] = serde_json::to_value(intrinsics).expect("intrinsics serialise"); + } + params } +/// `media` is `None` when there is no pipeline to answer for. +/// +/// A datachannel always has one — it exists because `webrtcsink` handed over a consumer — but the +/// rendezvous control lane (`relay::Relay::control`) is JSON-RPC with no media beside it, and it +/// can open before the pipeline has started. Answering `media.video` with zeros in that case would +/// be a consumer told the picture is 0×0 and upright, which is worse than being told there is no +/// picture: one is actionable, the other is a wrong number that looks like a right one. pub async fn run( mut inbound: mpsc::Receiver, outbound: mpsc::Sender, mut pool: Pool, - video: Video, + media: Option, ) { while let Some(line) = inbound.recv().await { let line = line.trim().to_string(); if line.is_empty() { continue; } - if let Some(reply) = handle(&line, &mut pool, video).await { + if let Some(reply) = handle(&line, &mut pool, media.as_ref()).await { // A closed outbound means the peer is gone; there is nothing left to do for it. if outbound.send(reply).await.is_err() { break; @@ -95,7 +143,7 @@ pub async fn run( /// Route one line. Returns a reply to send back only when this transport answers it itself — /// which is to say, only when it refuses. -async fn handle(line: &str, pool: &mut Pool, video: Video) -> Option { +async fn handle(line: &str, pool: &mut Pool, media: Option<&Media>) -> Option { let request: proto::Request = match serde_json::from_str(line) { Ok(request) => request, Err(e) => { @@ -117,17 +165,52 @@ async fn handle(line: &str, pool: &mut Pool, video: Video) -> Option { // Answered here, before anything tries to make a `Call` of it: this is `mediad`'s own question // about `mediad`'s own pipeline, and there is no service to route it to. if request.method == VIDEO_METHOD { - return Some( - serde_json::to_string(&proto::Response::ok( + return Some(match media { + Some(media) => { + serde_json::to_string(&proto::Response::ok(id, &video_params(&media.video))) + .expect("Response serialises") + } + None => error_line( id, - &serde_json::json!({ - "width": video.width, - "height": video.height, - "rotate": video.rotate, - }), - )) - .expect("Response serialises"), - ); + proto::Error::new( + proto::code::INTERNAL_ERROR, + "this robot is not publishing video, so there is no geometry to describe", + ), + ), + }); + } + + // **The one call that makes this robot send its camera somewhere it was told about.** + // + // Answered here rather than routed for the same reason `media.video` is — the pipeline is + // `mediad`'s and no service owns it — and it is the reason a frame stream needs no relay + // candidate: the robot dials out, so a NAT is not a participant. `stream.rs`'s header has the + // argument; this is where a peer asks for it. + if request.method == STREAM_METHOD { + let Some(media) = media else { + return Some(error_line( + id, + proto::Error::new( + proto::code::INTERNAL_ERROR, + "this robot has no camera to stream", + ), + )); + }; + return Some(match stream_request(&request.params) { + // No `url` key at all is a question rather than an instruction, which is what lets a + // client show what is streaming without having to remember what it asked for. + Ask::Status => { + serde_json::to_string(&proto::Response::ok(id, &media.streamer.status())) + .expect("Response serialises") + } + Ask::Stop => serde_json::to_string(&proto::Response::ok(id, &media.streamer.stop())) + .expect("Response serialises"), + Ask::Start(config) => match media.streamer.start(config) { + Ok(answer) => serde_json::to_string(&proto::Response::ok(id, &answer)) + .expect("Response serialises"), + Err(why) => error_line(id, proto::Error::new(proto::code::INVALID_PARAMS, why)), + }, + }); } let call = match request.as_call() { @@ -169,6 +252,59 @@ async fn handle(line: &str, pool: &mut Pool, video: Video) -> Option { /// One refusal, as a line. Built through [`proto::Response`] rather than by hand so the envelope /// has exactly one definition — the same reason `duck-ipc-proto` exists. +/// What a `media.stream` call is asking for. +/// +/// `url` present is start, `url: null` is stop, no `url` key is a question. The same three-way +/// reading `robot.loadPolicy` gives `slot` and `path`, and for the same reason: one method that a +/// client can use to look, to change and to put back is one method to route and one to permit. +enum Ask { + Status, + Stop, + Start(crate::stream::Config), +} + +fn stream_request(params: &Option) -> Ask { + use crate::stream::Config; + + // Absent params is the same question as params with no `url`: a client asking what is + // streaming should not have to send `{}` to be understood. + let params = match params { + None => return Ask::Status, + Some(params) => params, + }; + match params.get("url") { + None => Ask::Status, + Some(serde_json::Value::Null) => Ask::Stop, + Some(url) => Ask::Start(Config { + url: url.as_str().unwrap_or_default().to_owned(), + // Every number is optional: what a caller almost always wants is "stream to here", + // and a default that is cheap enough to ignore is better than four required fields. + fps: params + .get("fps") + .and_then(serde_json::Value::as_f64) + .unwrap_or(Config::DEFAULT_FPS), + longest: params + .get("longest") + .and_then(serde_json::Value::as_u64) + .unwrap_or(Config::DEFAULT_LONGEST as u64) as u32, + quality: params + .get("quality") + .and_then(serde_json::Value::as_u64) + .unwrap_or(Config::DEFAULT_QUALITY as u64) + .min(100) as u8, + // H.264 unless asked otherwise: the encode is the VPU's rather than a core's, and + // inter-frame prediction is worth five to fifteen times the bytes over the same + // wifi. JPEG stays reachable because it needs no keyframe to start and no decoder + // state to keep, which is what a receiver that reconnects constantly wants. + encoding: params + .get("encoding") + .and_then(serde_json::Value::as_str) + .and_then(crate::stream::Encoding::parse) + .unwrap_or_default(), + }), + } +} + fn error_line(id: Option, error: proto::Error) -> String { // A `Response` cannot fail to serialise: every field is a `String`, an `Id` or an `Error`. serde_json::to_string(&proto::Response::err(id, error)).expect("Response serialises") @@ -238,11 +374,36 @@ mod tests { inbound, outbound, pool, - Video { - width: 1280, - height: 720, - rotate: 90, - }, + Some(Media { + video: Video { + width: 1280, + height: 720, + rotate: 90, + intrinsics: crate::camera::Intrinsics::nominal( + Some(crate::camera::SensorMode::PINNED), + 1280, + 720, + ), + }, + // A streamer whose encoder produces nothing, which is all this needs: the tests + // here are about which method is answered by whom, and an encoder that touched a + // pipeline would make the whole file unbuildable off a board. + streamer: std::sync::Arc::new(crate::stream::Streamer::new( + crate::stream::Encoders { + jpeg: std::sync::Arc::new(|_| None), + h264: Some(std::sync::Arc::new(|_| None)), + gate: None, + }, + crate::producer::Producer { + name: Some("olducky".to_owned()), + serial: Some("3fa1c51b".to_owned()), + release: "0.10.0".to_owned(), + api_version: proto::API_VERSION, + }, + 90, + dir.path().join("hf-token"), + )), + }), )); Harness { to_peer, @@ -406,16 +567,22 @@ mod tests { assert!(reply.contains("robot.teleport"), "{reply}"); } - /// The line that tells a page how the camera is mounted. + /// The line that tells a page how the camera is mounted, and a consumer what its geometry is. /// - /// Hand-built JSON, so this is the only thing between a console that rotates the picture and one - /// that shows it sideways and says nothing. + /// This is the only thing between a console that rotates the picture and one that shows it + /// sideways and says nothing — and now also between a perception consumer that can turn a + /// pixel into a direction and one that has to guess a focal length. #[test] - fn the_video_notification_carries_the_mount_rotation() { - let line = video_notification(Video { + fn the_video_notification_carries_the_mount_rotation_and_the_geometry() { + let line = video_notification(&Video { width: 1280, height: 720, rotate: 90, + intrinsics: crate::camera::Intrinsics::nominal( + Some(crate::camera::SensorMode::PINNED), + 1280, + 720, + ), }); let parsed: serde_json::Value = serde_json::from_str(&line).expect("valid json"); assert_eq!(parsed["method"], "media.video"); @@ -423,6 +590,35 @@ mod tests { assert_eq!(parsed["params"]["width"], 1280); assert_eq!(parsed["params"]["height"], 720); assert_eq!(parsed["params"]["rotate"], 90); + + let intrinsics = &parsed["params"]["intrinsics"]; + assert!( + (intrinsics["fx"].as_f64().unwrap() - 1065.14).abs() < 0.1, + "{intrinsics}" + ); + assert_eq!(intrinsics["cx"], 640.0); + assert_eq!( + intrinsics["calibrated"], false, + "the module's design figures, and a consumer has to be able to tell" + ); + } + + /// A robot whose camera geometry is unknown omits the key rather than sending a null. + /// + /// Which is every robot streaming a test pattern, and any board where `media-ctl` would not + /// set the sensor mode. A consumer reading a missing key knows it has to calibrate; one + /// reading `null` has to be told what that meant. + #[test] + fn an_unknown_geometry_is_absent_from_the_line() { + let line = video_notification(&Video { + width: 1280, + height: 720, + rotate: 0, + intrinsics: None, + }); + let parsed: serde_json::Value = serde_json::from_str(&line).expect("valid json"); + assert!(parsed["params"].get("intrinsics").is_none(), "{line}"); + assert_eq!(parsed["params"]["width"], 1280); } /// `media.video` is answered by the session, not routed to a service. @@ -430,6 +626,59 @@ mod tests { /// This is the path the console uses, and it exists because pushing the same information when /// the channel appears races the browser's datachannel and loses — a sideways picture with /// nothing in the log. A question the page asks when it is ready cannot arrive too early. + /// `media.stream` reads three ways off one key, and the validation is here rather than later. + /// + /// No `url` is a question, so a client can show what is streaming without remembering what it + /// asked for. And a url that is not a WebSocket is refused *before* a thread and a socket are + /// spawned for it — a robot that accepted `http://` would sit in a reconnect loop against + /// something that can never upgrade, having told the caller yes. + #[tokio::test] + async fn media_stream_answers_the_question_and_refuses_a_bad_url() { + let dir = tempfile::tempdir().unwrap(); + let mut h = harness(sockets_in(dir.path()), dir); + + async fn ask(h: &mut Harness, line: &str) -> serde_json::Value { + h.from_peer.send(line.into()).await.unwrap(); + serde_json::from_str(&h.to_peer.recv().await.unwrap()).expect("valid json") + } + + let answer = ask( + &mut h, + r#"{"jsonrpc":"2.0","id":1,"method":"media.stream"}"#, + ) + .await; + assert_eq!(answer["result"]["streaming"], false, "{answer}"); + assert_eq!(answer["result"]["sent"], 0); + + let answer = ask( + &mut h, + r#"{"jsonrpc":"2.0","id":2,"method":"media.stream","params":{"url":"http://a.b"}}"#, + ) + .await; + assert_eq!( + answer["error"]["code"], + proto::code::INVALID_PARAMS, + "{answer}" + ); + assert!( + answer["error"]["message"] + .as_str() + .unwrap() + .contains("ws://"), + "the refusal says what a url has to be: {answer}" + ); + + // Stopping something that was never started is not an error: a client tidying up after + // itself should not have to know whether there was anything to tidy. + let answer = ask( + &mut h, + r#"{"jsonrpc":"2.0","id":3,"method":"media.stream","params":{"url":null}}"#, + ) + .await; + assert_eq!(answer["result"]["streaming"], false, "{answer}"); + assert_eq!(answer["result"]["was"], serde_json::Value::Null); + } + #[tokio::test] async fn the_page_can_ask_what_the_video_is() { let dir = tempfile::tempdir().unwrap(); diff --git a/mediad/src/snapshot.rs b/mediad/src/snapshot.rs new file mode 100644 index 00000000..d19fa25e --- /dev/null +++ b/mediad/src/snapshot.rs @@ -0,0 +1,134 @@ +//! A bounded local snapshot client for the HTTP console; camera bytes never enter control routing. +use anyhow::{Context, Result, ensure}; +use duck_ipc_proto as proto; +use std::path::Path; +use tokio::io::{AsyncBufReadExt, AsyncReadExt, AsyncWriteExt, BufReader}; +use tokio::net::UnixStream; + +pub(crate) async fn fetch(socket: &Path) -> Result<(proto::MediaFrameHeader, Vec)> { + tokio::time::timeout(std::time::Duration::from_secs(3), async { + let stream = UnixStream::connect(socket).await?; + let (read, mut write) = stream.into_split(); + let mut reader = BufReader::new(read); + // A normal version handshake, followed by the deliberately unrouted binary method. + let request = proto::Request::call( + proto::Id::Number(1), + &proto::Call::Hello(proto::HelloParams { + api_version: proto::API_VERSION, + }), + ); + let mut line = serde_json::to_vec(&request)?; + line.push(b'\n'); + write.write_all(&line).await?; + let hello = response(&mut reader, 1).await?; + let hello: proto::HelloResult = serde_json::from_value(hello)?; + ensure!( + hello.api_version == proto::API_VERSION, + "camera API version mismatch" + ); + let request = proto::Request { + jsonrpc: "2.0".into(), + id: Some(proto::Id::Number(2)), + method: proto::method::MEDIA_FRAME.into(), + params: None, + }; + let mut line = serde_json::to_vec(&request)?; + line.push(b'\n'); + write.write_all(&line).await?; + let header: proto::MediaFrameHeader = + serde_json::from_value(response(&mut reader, 2).await?)?; + ensure!( + header.valid_uyvy(), + "invalid camera geometry or payload size" + ); + let mut data = vec![0; header.bytes]; + reader.read_exact(&mut data).await?; + Ok((header, data)) + }) + .await + .context("camera snapshot timed out")? +} + +async fn response( + reader: &mut (impl tokio::io::AsyncBufRead + Unpin), + id: u64, +) -> Result { + let mut line = Vec::new(); + reader.take(4097).read_until(b'\n', &mut line).await?; + ensure!( + line.len() <= 4096 && line.ends_with(b"\n"), + "invalid camera response length" + ); + let response: proto::Response = serde_json::from_slice(&line)?; + ensure!( + response.jsonrpc == "2.0" && response.id == Some(proto::Id::Number(id)), + "camera response ID mismatch" + ); + if let Some(error) = response.error { + anyhow::bail!("camera: {}", error.message); + } + response.result.context("camera response has no result") +} + +/// Encode the frame as a PNG, upright. +/// +/// **This is the one consumer that has nowhere to put the rotation.** A PNG opened in a browser +/// or piped into a viewer carries no `rotate` alongside it, so a snapshot route that returned the +/// sensor's own orientation would hand every human a sideways picture and no way to know why. The +/// raw UYVY path keeps its "told, not applied" contract — the header names the angle and the +/// recorder turns it — but here the header *is* the thing being thrown away. +/// +/// It costs nothing the hot path would notice: this runs once per request on a blocking thread, +/// not thirty times a second in front of the encoder, which is what made `videoflip` expensive. +pub(crate) fn png(header: proto::MediaFrameHeader, data: Vec) -> Result> { + use image::ImageEncoder; + let turn = duck_detect::Turn::from_degrees(header.rotate) + .context("camera reported a mount that is not a quarter turn")?; + let mut rgb = Vec::new(); + // The turned dimensions, not the header's: a quarter turn swaps the axes, and encoding the + // capture geometry over rotated pixels is a diagonally sheared image rather than an error. + let (width, height) = duck_detect::rgb_from_uyvy( + &data, + header.width as usize, + header.height as usize, + // No downscale. `max` is the same either side of a quarter turn, so this stays the + // longest edge whichever way the frame is about to go. + header.width.max(header.height) as usize, + turn, + &mut rgb, + ); + let mut encoded = Vec::new(); + image::codecs::png::PngEncoder::new(&mut encoded).write_image( + &rgb, + width as u32, + height as u32, + image::ExtendedColorType::Rgb8, + )?; + Ok(encoded) +} + +#[cfg(test)] +mod tests { + use super::*; + #[tokio::test] + async fn malformed_or_unmatched_headers_fail_before_pixels_are_allocated() { + for bytes in [ + vec![b'x'; 4097], + vec![255, b'\n'], + b"{\"jsonrpc\":\"2.0\",\"id\":99,\"result\":{}}\n".to_vec(), + ] { + let mut reader = BufReader::new(bytes.as_slice()); + assert!(response(&mut reader, 2).await.is_err()); + } + } + #[tokio::test] + async fn silent_camera_has_a_deadline() { + let dir = tempfile::tempdir().unwrap(); + let socket = dir.path().join("media.sock"); + let _listener = tokio::net::UnixListener::bind(&socket).unwrap(); + let result = tokio::time::timeout(std::time::Duration::from_secs(5), fetch(&socket)) + .await + .unwrap(); + assert!(result.unwrap_err().to_string().contains("timed out")); + } +} diff --git a/mediad/src/stream.rs b/mediad/src/stream.rs new file mode 100644 index 00000000..168c5061 --- /dev/null +++ b/mediad/src/stream.rs @@ -0,0 +1,983 @@ +//! Frames out to a WebSocket **this robot dials**, for a Space that runs a model on them. +//! +//! # This is the fallback, and WebRTC is the default +//! +//! **A consumer should use WebRTC**, which carries encrypted media, a control channel on the same +//! session and a return path, and which reaches a data centre because the robot offers a relay +//! candidate (`remote-access-design.md` §6). `docs/faq.md` is the decision, in the shape somebody +//! arrives at it. This module is for the narrow case WebRTC serves badly: a **program** consuming +//! **frames only** on a **long-running** stream, where a relay's metered bandwidth is the cost +//! that matters. +//! +//! In that case an outbound WebSocket needs nobody's relay. **The robot already proves this every +//! second it is reachable**: `relay.rs` holds an outbound HTTPS stream to a Space right now, and +//! nothing about a home router objects. So the frames go the same way the registration does — +//! outward — and NAT stops being a participant. +//! +//! ```text +//! Space ──media.stream {url: "wss://…/frames"}──► rendezvous ──► this robot +//! robot ═══════════ JPEG frames, outbound wss, direct ═══════════► Space +//! ``` +//! +//! The rendezvous carries **the instruction and not the pixels**, which is the property that makes +//! this scale where relaying payload through a shared service would not: one small envelope per +//! session, on a service the mini fleet also depends on, and the bytes go point to point. +//! +//! **This was written when the relay endpoint was dead and WebRTC could not connect from a data +//! centre at all.** That is fixed (§6), so the reason this exists is now the narrow one above and +//! not "the alternative does not work". What survives of the original argument is the cost: a +//! relay is metered per Hugging Face account at 10 GB a month, and a stream that runs all day +//! spends an allowance its owner also needs for being *watched*. +//! +//! # What it is not +//! +//! **Not a replacement for WebRTC, and not the path to reach for first.** There is no return +//! media path and no control channel, so nothing here helps a browser *watch* a robot, carries +//! audio, or closes a teleop loop; driving means a separate JSON-RPC call over the rendezvous. +//! Encryption is the receiver's TLS rather than DTLS-SRTP, terminating at a server instead of at +//! the peer. A consumer that is not all three of program, frames-only and long-running wants §6. +//! +//! # This half is portable, and that is deliberate +//! +//! [`pump`] takes a channel of already-encoded frames and knows nothing about GStreamer, so the +//! reconnect, the backoff, the framing and the counters are all exercised on a laptop against a +//! fake server. What touches [`crate::pipeline::Frames`] is [`encode_frames`], which is the thin +//! linux-only half: read a frame, turn it upright, JPEG it, hand it over. The same split +//! `session.rs` has, for the same reason — every failure worth testing here is a timing failure. + +use std::sync::Arc; +use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; +use std::time::Duration; + +use futures_util::{SinkExt as _, StreamExt as _}; +#[cfg(target_os = "linux")] +use image::ImageEncoder as _; + +/// What the frames are, on the wire. +/// +/// The hello carries this, so a receiver branches on it rather than sniffing bytes — and so that +/// adding the second one was a field rather than a protocol. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +pub enum Encoding { + /// One JPEG per message, every one independently decodable. + Jpeg, + /// H.264 access units, SPS/PPS in front of every keyframe. + /// + /// **Cheaper on the board and on the wire, and not free.** The encode is the VPU's rather than + /// the CPU's and inter-frame prediction is worth five to fifteen times the bytes — but a + /// receiver joining mid-stream can decode nothing until a keyframe arrives, and a dropped + /// frame corrupts every frame after it until the next one. Both of those are handled here + /// rather than wished away: see [`Streamer::start`]'s drop policy and + /// `pipeline::force_keyframe`. + #[default] + H264, +} + +impl Encoding { + pub fn wire_name(self) -> &'static str { + match self { + Self::Jpeg => "jpeg", + Self::H264 => "h264", + } + } + + pub fn parse(name: &str) -> Option { + match name.to_ascii_lowercase().as_str() { + "jpeg" | "jpg" | "mjpeg" => Some(Self::Jpeg), + "h264" | "avc" => Some(Self::H264), + _ => None, + } + } +} + +/// One encoded thing to send: a JPEG, or an H.264 access unit. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Unit { + pub bytes: Vec, + /// Whether a receiver could start decoding here. Always true for JPEG; true for an H.264 IDR. + /// + /// **This is what makes the drop policy correct.** Discarding the oldest and keeping the + /// newest is right for independent frames and wrong for a predicted stream: the newest P-frame + /// refers to one that was thrown away, so a receiver decodes garbage until the next keyframe. + /// Knowing which is which is the difference between dropping a frame and corrupting a second. + pub keyframe: bool, +} + +/// What to send, and where. +#[derive(Debug, Clone, PartialEq)] +pub struct Config { + /// `wss://…` for anything that is not on this network. `ws://` is accepted because a laptop + /// on the bench serving a receiver over TLS is a certificate nobody wants to make, and it is + /// logged either way — the destination of this robot's camera is worth a line in the journal. + pub url: String, + /// Frames a second. Not the capture rate: every frame costs a colour conversion and a JPEG on + /// the CPU, and what receives them is a model that does not want thirty. + pub fps: f64, + /// The longest side, in pixels. Downscale only. + pub longest: u32, + /// JPEG quality, 1..=100. Ignored for H.264, whose bitrate is the pipeline's. + pub quality: u8, + pub encoding: Encoding, +} + +impl Config { + /// What a caller gets for asking for nothing: enough for a model, cheap enough to ignore. + pub const DEFAULT_FPS: f64 = 5.0; + pub const DEFAULT_LONGEST: u32 = 640; + pub const DEFAULT_QUALITY: u8 = 70; + + /// How long to wait between pulls, which is nothing when the source paces itself. + /// + /// A JPEG is made on demand from whatever the camera last captured, so the rate is this + /// thread's to keep. An H.264 unit arrives when the encoder emits one, and the rate was + /// already imposed by a `videorate` in the pipeline — so waiting here would delay a frame + /// that exists rather than avoid making one. + pub fn interval(&self) -> Duration { + if self.encoding == Encoding::H264 { + return Duration::ZERO; + } + let fps = if self.fps.is_finite() { + self.fps + } else { + Self::DEFAULT_FPS + }; + Duration::from_secs_f64(1.0 / fps.clamp(0.2, 15.0)) + } +} + +/// What a stream has done so far, for `media.stream`'s answer. +#[derive(Debug, Default)] +pub struct Counters { + pub connected: AtomicBool, + /// Frames handed to the socket. + pub sent: AtomicU64, + /// Frames encoded and thrown away because the socket was not keeping up. + /// + /// **Dropping is correct, and what to drop depends on the encoding.** For JPEG the newest + /// frame is what a model wants, so the oldest goes. For H.264 a gap corrupts everything until + /// the next keyframe, so the stream is abandoned to the next one instead — which is why this + /// can jump by a whole group of pictures at a time. + pub dropped: AtomicU64, + /// Text frames the far end sent back — a model's answers, if it sends any. + pub replies: AtomicU64, + pub reconnects: AtomicU64, +} + +/// How long to wait before redialling, and how much of itself to add as jitter. +/// +/// A struct rather than constants for the reason `relay::Timings` is one: every failure worth +/// testing here is a timing failure, and a test that waited two real seconds per reconnect would +/// be a test nobody runs. `tokio`'s paused clock would do it too, at the cost of a feature on the +/// dependency for the whole crate. +#[derive(Debug, Clone, Copy)] +pub struct Timings { + pub backoff_start: Duration, + pub backoff_max: Duration, + pub jitter: f64, +} + +impl Default for Timings { + fn default() -> Self { + Self { + backoff_start: Duration::from_secs(2), + backoff_max: Duration::from_secs(30), + jitter: 0.2, + } + } +} + +/// Dial `url`, send `hello`, then forward every frame until told to stop. +/// +/// Reconnects for as long as `running` says so: a Space restarts on every push and sleeps when +/// nobody is looking at it, so a receiver going away is the ordinary case rather than the end of +/// the stream. Frames encoded while there is no socket are dropped, not queued — see +/// [`Counters::dropped`]. +pub async fn pump( + url: String, + token: Option, + hello: String, + mut frames: tokio::sync::mpsc::Receiver>, + counters: Arc, + running: Arc, + timings: Timings, +) { + let mut backoff = timings.backoff_start; + while running.load(Ordering::Relaxed) { + match connect(&url, token.as_deref()).await { + Err(why) => { + counters.connected.store(false, Ordering::Relaxed); + tracing::warn!(%url, %why, "the frame receiver could not be reached"); + } + Ok(mut socket) => { + counters.connected.store(true, Ordering::Relaxed); + backoff = timings.backoff_start; + tracing::info!(%url, "streaming frames to a receiver this robot dialled"); + + let outcome = carry(&mut socket, &hello, &mut frames, &counters, &running).await; + counters.connected.store(false, Ordering::Relaxed); + // Closed politely, so the far end knows this was not a crash. + let _ = socket.close(None).await; + match outcome { + Carried::Stopped => break, + Carried::Lost(why) => tracing::info!(%url, %why, "the frame stream dropped"), + } + } + } + if !running.load(Ordering::Relaxed) { + break; + } + counters.reconnects.fetch_add(1, Ordering::Relaxed); + tokio::time::sleep(jittered(backoff, timings.jitter)).await; + backoff = (backoff * 2).min(timings.backoff_max); + } + counters.connected.store(false, Ordering::Relaxed); + tracing::info!(%url, "the frame stream ended"); +} + +type Socket = + tokio_tungstenite::WebSocketStream>; + +/// Why [`carry`] returned, which decides whether to redial. +enum Carried { + /// Asked to stop. Nothing to redial. + Stopped, + Lost(String), +} + +async fn connect(url: &str, token: Option<&str>) -> Result { + use tokio_tungstenite::tungstenite::client::IntoClientRequest as _; + + let mut request = url + .into_client_request() + .map_err(|e| format!("{url} is not a WebSocket url: {e}"))?; + if let Some(token) = token { + // **The robot's own credential, as the handshake header.** The receiver is a public + // endpoint, so it has to be able to say whose camera this is — the same `whoami-v2` + // resolution the rendezvous does with this token. Without it a Space would take frames + // from anybody and show them to anybody. + let value = format!("Bearer {token}") + .parse() + .map_err(|_| "the account token is not a header value".to_owned())?; + request.headers_mut().insert("authorization", value); + } + let (socket, _) = tokio_tungstenite::connect_async(request) + .await + .map_err(|e| e.to_string())?; + Ok(socket) +} + +async fn carry( + socket: &mut Socket, + hello: &str, + frames: &mut tokio::sync::mpsc::Receiver>, + counters: &Counters, + running: &AtomicBool, +) -> Carried { + use tokio_tungstenite::tungstenite::Message; + + // **What is coming, before any of it arrives.** A receiver reading binary frames has no way to + // know their size, their rate or which robot's camera they are, and guessing from the first + // JPEG is a decoder's job rather than a protocol. + if let Err(e) = socket.send(Message::Text(hello.into())).await { + return Carried::Lost(format!("the hello would not send: {e}")); + } + + loop { + tokio::select! { + frame = frames.recv() => match frame { + None => return Carried::Stopped, + Some(frame) => { + if !running.load(Ordering::Relaxed) { + return Carried::Stopped; + } + if let Err(e) = socket.send(Message::Binary(frame.into())).await { + return Carried::Lost(format!("a frame would not send: {e}")); + } + counters.sent.fetch_add(1, Ordering::Relaxed); + } + }, + // **Read, or the connection dies of politeness.** tungstenite answers pings while the + // stream is polled and not otherwise, so a sender that only ever writes stops + // answering keepalives and the far end hangs up. Whatever the receiver says back — a + // model's answer — is counted and logged rather than acted on: nothing on this robot + // has asked to be driven by it. + inbound = socket.next() => match inbound { + None => return Carried::Lost("the receiver closed the socket".to_owned()), + Some(Err(e)) => return Carried::Lost(e.to_string()), + Some(Ok(Message::Close(_))) => { + return Carried::Lost("the receiver said goodbye".to_owned()); + } + Some(Ok(Message::Text(text))) => { + counters.replies.fetch_add(1, Ordering::Relaxed); + tracing::debug!(reply = %text.chars().take(200).collect::(), + "the frame receiver answered"); + } + Some(Ok(_)) => {} + }, + } + } +} + +fn jittered(base: Duration, jitter: f64) -> Duration { + base + base.mul_f64(jitter).mul_f64(rand::random::()) +} + +/// One encoded frame, on demand, at the size and quality asked for. +/// +/// A closure rather than a `Frames` handle, and that is what keeps this module portable: +/// everything that touches GStreamer lives on the far side of it, and the linux-only half is +/// [`jpeg_encoder`] alone. On a laptop a test supplies a closure that returns bytes. +pub type Encode = Arc Option + Send + Sync>; + +/// The ways this robot can encode what it streams, and the gate on the expensive one. +/// +/// Both are held rather than one chosen at startup, because `media.stream` names an encoding per +/// call: H.264 by default, and JPEG for a receiver that reconnects constantly enough to care more +/// about starting instantly than about bytes. +pub struct Encoders { + pub jpeg: Encode, + /// `None` on a board with no H.264 encoder at all, where asking for it is refused with that + /// as the reason rather than accepted and silently served as JPEG. + pub h264: Option, + /// Opens and shuts the H.264 branch's valve. `true` on start, `false` on stop. + pub gate: Option>, +} + +/// The robot's frame stream: at most one, started and stopped by `media.stream`. +/// +/// **One at a time**, and unlike the media path that is a resource decision rather than a +/// protocol one: every frame costs a colour conversion and a JPEG on a CPU that is also running a +/// control loop, so two receivers would be two of that. A second `media.stream` replaces the +/// first, which is also the only way to change the rate without a stop. +pub struct Streamer { + encoders: Encoders, + producer: crate::producer::Producer, + /// The mount angle, for the hello — these frames are already upright, and a receiver has to be + /// told that rather than left to apply it twice. + rotate: u32, + token_path: std::path::PathBuf, + timings: Timings, + live: std::sync::Mutex>, +} + +struct Live { + config: Config, + running: Arc, + counters: Arc, +} + +impl Drop for Live { + fn drop(&mut self) { + // Both halves watch this: the encoder thread stops reading frames, and the pump stops + // redialling. Dropping the channel is what actually wakes the pump. + self.running.store(false, Ordering::Relaxed); + } +} + +impl Streamer { + pub fn new( + encoders: Encoders, + producer: crate::producer::Producer, + rotate: u32, + token_path: impl Into, + ) -> Self { + Self { + encoders, + producer, + rotate, + token_path: token_path.into(), + timings: Timings::default(), + live: std::sync::Mutex::new(None), + } + } + + pub fn with_timings(mut self, timings: Timings) -> Self { + self.timings = timings; + self + } + + /// Start streaming, replacing whatever was streaming before. + pub fn start(&self, config: Config) -> Result { + if config.url.is_empty() { + return Err("no url to stream to".to_owned()); + } + if !(config.url.starts_with("ws://") || config.url.starts_with("wss://")) { + return Err(format!("{} is not a ws:// or wss:// url", config.url)); + } + if !(1..=100).contains(&config.quality) { + return Err("quality is 1..=100".to_owned()); + } + + // **Logged before anything is sent, at info.** This is the one call that makes a robot + // hand its camera to somewhere it was told about rather than somewhere it knows, so where + // that was has to be in the journal whether or not anybody was watching the page that + // asked. `remote-webrtc.md` §4 leaves this transport ungated on purpose; a line in the + // log is what makes that decision auditable rather than invisible. + tracing::info!( + url = %config.url, fps = config.fps, longest = config.longest, + "asked to stream frames out" + ); + + let encode = match config.encoding { + Encoding::Jpeg => Arc::clone(&self.encoders.jpeg), + Encoding::H264 => match self.encoders.h264.as_ref() { + Some(encode) => Arc::clone(encode), + None => { + return Err( + "this robot has no H.264 encoder, so it can only stream jpeg — ask for \ + `\"encoding\": \"jpeg\"`" + .to_owned(), + ); + } + }, + }; + // Opened before the encoder thread reads, or its first pull waits a whole frame timeout + // for a branch nothing is flowing through. + if let Some(gate) = self.encoders.gate.as_ref() { + gate(config.encoding == Encoding::H264); + } + + let running = Arc::new(AtomicBool::new(true)); + let counters = Arc::new(Counters::default()); + // Two deep: the newest frame and one in flight. A model wants the freshest picture, so a + // frame encoded while the socket is behind is dropped rather than queued — a backlog is + // latency that never comes back, and `Counters::dropped` is how a caller sees it happening. + let (to_socket, from_encoder) = tokio::sync::mpsc::channel::>(2); + + let interval = config.interval(); + let for_encoder = config.clone(); + let encoding = Arc::clone(&running); + let counted = Arc::clone(&counters); + std::thread::Builder::new() + .name("frame-stream".to_owned()) + .spawn(move || { + // Set after a drop on a predicted stream: everything until the next keyframe + // would decode against a frame the receiver never got, so it is skipped rather + // than sent. For JPEG this never becomes true — every unit is a keyframe. + let mut awaiting_key = false; + + while encoding.load(Ordering::Relaxed) { + let began = std::time::Instant::now(); + if let Some(unit) = encode(&for_encoder) { + if awaiting_key && !unit.keyframe { + counted.dropped.fetch_add(1, Ordering::Relaxed); + continue; + } + awaiting_key = false; + match to_socket.try_send(unit.bytes) { + Ok(()) => {} + Err(tokio::sync::mpsc::error::TrySendError::Full(_)) => { + counted.dropped.fetch_add(1, Ordering::Relaxed); + // A gap has happened. On a predicted stream the only safe thing + // to send next is a keyframe. + awaiting_key = !unit.keyframe || awaiting_key; + if for_encoder.encoding == Encoding::H264 { + awaiting_key = true; + } + } + // The pump is gone, so there is nowhere left to send. + Err(tokio::sync::mpsc::error::TrySendError::Closed(_)) => return, + } + } + // Zero for a source that paces itself: an H.264 appsink blocks until the + // encoder produces, and `videorate` upstream is what limits the rate. Sleeping + // on top of that would only add latency to frames that already exist. + if let Some(rest) = interval.checked_sub(began.elapsed()) { + std::thread::sleep(rest); + } + } + }) + .map_err(|e| format!("no thread for the frame stream: {e}"))?; + + tokio::spawn(pump( + config.url.clone(), + hf_robot_account::read_access_token(&self.token_path), + hello(&config, &self.producer, self.rotate), + from_encoder, + Arc::clone(&counters), + Arc::clone(&running), + self.timings, + )); + + let answer = describe(Some(&config), &counters); + *self.live.lock().expect("not poisoned") = Some(Live { + config, + running, + counters, + }); + Ok(answer) + } + + /// Stop streaming. Answers whether there was anything to stop. + pub fn stop(&self) -> serde_json::Value { + let was = self.live.lock().expect("not poisoned").take(); + // Shut the branch whatever it was streaming: a valve left open is a second encode for + // nobody, which is the cost this whole arrangement exists to avoid. + if let Some(gate) = self.encoders.gate.as_ref() { + gate(false); + } + match &was { + Some(live) => tracing::info!(url = %live.config.url, "the frame stream was stopped"), + None => tracing::debug!("nothing was streaming"), + } + serde_json::json!({ "streaming": false, "was": was.as_ref().map(|l| l.config.url.clone()) }) + } + + /// What is streaming and how it is going. + pub fn status(&self) -> serde_json::Value { + let live = self.live.lock().expect("not poisoned"); + match live.as_ref() { + None => describe(None, &Counters::default()), + Some(live) => describe(Some(&live.config), &live.counters), + } + } +} + +fn describe(config: Option<&Config>, counters: &Counters) -> serde_json::Value { + let mut answer = serde_json::json!({ + "streaming": config.is_some(), + "sent": counters.sent.load(Ordering::Relaxed), + "dropped": counters.dropped.load(Ordering::Relaxed), + "replies": counters.replies.load(Ordering::Relaxed), + "reconnects": counters.reconnects.load(Ordering::Relaxed), + "connected": counters.connected.load(Ordering::Relaxed), + }); + if let Some(config) = config { + answer["url"] = config.url.clone().into(); + answer["fps"] = config.fps.into(); + answer["longest"] = config.longest.into(); + answer["quality"] = config.quality.into(); + } + answer +} + +/// The line that opens a stream, so a receiver knows what it is about to be sent. +pub fn hello(config: &Config, producer: &crate::producer::Producer, rotate: u32) -> String { + serde_json::json!({ + "type": "hello", + "robot": { + "name": producer.name, + "serial": producer.serial, + "release": producer.release, + "api_version": producer.api_version, + "kind": "microduck", + }, + "frames": { + "encoding": config.encoding.wire_name(), + "longest": config.longest, + "fps": config.fps, + "quality": config.quality, + // Zero, always, and said out loud: unlike the WebRTC path, these frames are turned + // upright before they are encoded. A receiver that honoured a mount angle here would + // rotate an already-upright picture. + "rotate": 0, + "mount_rotate": rotate, + // H.264 only, and stated because it decides how a receiver frames what arrives: one + // WebSocket message is one access unit, with SPS and PPS repeated in front of every + // keyframe (`h264parse config-interval=-1`), so a receiver that joins mid-stream needs + // nothing from the messages it missed. + "annexb": config.encoding == Encoding::H264, + }, + }) + .to_string() +} + +/// The linux half for H.264: take the next access unit the branch's encoder produced. +/// +/// Nothing is converted or copied here beyond the unit itself — the VPU did the work, upstream of +/// an appsink — which is the whole reason to prefer this over JPEG on a board. The rate was +/// imposed by a `videorate` in the branch, so this blocks until there is something and never +/// paces anything itself ([`Config::interval`] returns zero for it). +/// +/// Opening the valve is [`Encoders::gate`]'s job rather than this closure's, so that a stream +/// which stops shuts it again: a second encoder running for nobody is what the valve exists to +/// prevent. +#[cfg(target_os = "linux")] +pub fn h264_encoder(branch: crate::pipeline::StreamBranch) -> Encode { + Arc::new(move |_config: &Config| { + let (bytes, keyframe) = branch.encoded.next_unit()?; + Some(Unit { bytes, keyframe }) + }) +} + +/// The linux half: read a frame off the tee, turn it upright, and JPEG it. +/// +/// **The only part of this module that knows a pipeline exists.** Everything above takes bytes +/// from a channel, which is what lets the reconnect and the framing be tested on a laptop; this is +/// four lines of pixels and a call into `duck-detect`, where the UYVY arithmetic already lives +/// because the detector needed exactly the same conversion. +/// +/// `next_frame` blocks on a condvar until the *next* capture, so this runs on its own thread — +/// [`Streamer::start`] gives it one, the same way the detector and the exposure loop have theirs. +/// A frame is asked for rather than published, which is the whole design of `Frames`: at 5 fps +/// this copies five of thirty rather than all thirty. +#[cfg(target_os = "linux")] +pub fn jpeg_encoder(frames: crate::pipeline::Frames, turn: duck_detect::Turn) -> Encode { + // Reused across frames: at 640×480 the RGB buffer is 920 KB, and allocating that five times a + // second forever is a page fault storm for no reason. A `Mutex` because `Encode` is `Fn` — one + // thread ever takes it, so it is uncontended by construction. + let scratch = std::sync::Mutex::new((Vec::::new(), Vec::::new())); + + Arc::new(move |config: &Config| { + let frame = frames.next_frame()?; + if frame.format != crate::pipeline::CAPTURE_FORMAT { + // The caps changed under us. Refusing beats encoding one format as another, which + // produces a picture that is wrong in a way a receiver cannot detect. + tracing::warn!( + format = frame.format, + expected = crate::pipeline::CAPTURE_FORMAT, + "not streaming a frame in a format this encoder does not know" + ); + return None; + } + + let mut held = scratch.lock().expect("not poisoned"); + let (rgb, jpeg) = &mut *held; + let (width, height) = duck_detect::rgb_from_uyvy( + &frame.data, + frame.width as usize, + frame.height as usize, + config.longest as usize, + turn, + rgb, + ); + + jpeg.clear(); + let encoder = + image::codecs::jpeg::JpegEncoder::new_with_quality(&mut *jpeg, config.quality); + match encoder.write_image( + rgb, + width as u32, + height as u32, + image::ExtendedColorType::Rgb8, + ) { + // Every JPEG is a keyframe by construction, which is the property that makes this + // encoding worth keeping: a receiver that reconnects can decode the very next + // message, where H.264 has to wait for one. + Ok(()) => Some(Unit { + bytes: jpeg.clone(), + keyframe: true, + }), + Err(e) => { + tracing::warn!(error = %e, "a frame would not encode"); + None + } + } + }) +} +#[cfg(test)] +mod tests { + use super::*; + + /// What a fake receiver saw. + #[derive(Default)] + struct Seen { + hello: std::sync::Mutex>, + frames: std::sync::Mutex>>, + bearers: std::sync::Mutex>>, + /// Sockets to hang up on rather than serve, for the reconnect test. + hang_up: std::sync::atomic::AtomicU32, + } + + /// A receiver that records the hello and the frames, and can be told to drop the first N. + async fn receiver(seen: Arc) -> String { + use axum::extract::{State, WebSocketUpgrade, ws}; + + let app = axum::Router::new() + .route( + "/frames", + axum::routing::get( + |State(seen): State>, + headers: axum::http::HeaderMap, + upgrade: WebSocketUpgrade| async move { + seen.bearers.lock().unwrap().push( + headers + .get("authorization") + .and_then(|v| v.to_str().ok()) + .map(str::to_owned), + ); + upgrade.on_upgrade(move |mut socket| async move { + if seen.hang_up.fetch_saturating_sub() { + return; + } + while let Some(Ok(message)) = socket.recv().await { + match message { + ws::Message::Text(text) => { + seen.hello.lock().unwrap().push(text.to_string()) + } + ws::Message::Binary(bytes) => { + seen.frames.lock().unwrap().push(bytes.to_vec()) + } + ws::Message::Close(_) => return, + _ => {} + } + } + }) + }, + ), + ) + .with_state(seen); + + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let port = listener.local_addr().unwrap().port(); + tokio::spawn(async move { axum::serve(listener, app).await.unwrap() }); + format!("ws://127.0.0.1:{port}/frames") + } + + trait Saturating { + /// True if there was a hang-up left to spend. + fn fetch_saturating_sub(&self) -> bool; + } + + impl Saturating for std::sync::atomic::AtomicU32 { + fn fetch_saturating_sub(&self) -> bool { + self.fetch_update(Ordering::SeqCst, Ordering::SeqCst, |n| { + Some(n.saturating_sub(1)) + }) + .is_ok_and(|previous| previous > 0) + } + } + + async fn until(what: &str, mut ready: impl FnMut() -> bool) { + for _ in 0..200 { + if ready() { + return; + } + tokio::time::sleep(Duration::from_millis(25)).await; + } + panic!("timed out waiting for {what}"); + } + + fn config(url: &str) -> Config { + Config { + url: url.to_owned(), + fps: 5.0, + longest: 640, + quality: 70, + encoding: Encoding::H264, + } + } + + /// The hello, printed, so the other end's parser can be checked against it. + /// + /// `spaces/vision-demo/receiver.py` reads `robot.name` out of this line and files the stream + /// under it. A field renamed on one side of that is a stream that arrives labelled "a duck" + /// forever, which nothing fails on — so the shape is asserted here and the receiver's own test + /// reads this test's output. + #[test] + fn the_hello_names_the_robot_where_the_receiver_looks() { + let producer = crate::producer::Producer { + name: Some("olducky".to_owned()), + serial: Some("3fa1c51b".to_owned()), + release: "0.10.0".to_owned(), + api_version: 23, + }; + let line = hello(&config("wss://x/frames"), &producer, 90); + let parsed: serde_json::Value = serde_json::from_str(&line).unwrap(); + + assert_eq!(parsed["type"], "hello"); + assert_eq!( + parsed["robot"]["name"], "olducky", + "where the receiver looks for the name" + ); + assert_eq!(parsed["robot"]["kind"], "microduck"); + assert_eq!( + parsed["frames"]["encoding"], "h264", + "the default, and what the VPU makes" + ); + assert_eq!( + parsed["frames"]["annexb"], true, + "one message is one access unit" + ); + // Upright already, and the mount angle reported separately so a receiver cannot apply a + // turn that has been applied. + assert_eq!(parsed["frames"]["rotate"], 0); + assert_eq!(parsed["frames"]["mount_rotate"], 90); + println!("{line}"); + + // And the other one, because a receiver has to be able to tell them apart on this field + // alone: JPEG needs no keyframe and carries no stream state, which is why it stays + // reachable at all. + let mut jpeg = config("wss://x/frames"); + jpeg.encoding = Encoding::Jpeg; + let line = hello(&jpeg, &producer, 90); + let parsed: serde_json::Value = serde_json::from_str(&line).unwrap(); + assert_eq!(parsed["frames"]["encoding"], "jpeg"); + assert_eq!(parsed["frames"]["annexb"], false); + println!("{line}"); + } + + /// **The hello, the bearer and the frames**, which is the whole of the wire a Space reads. + #[tokio::test] + async fn frames_reach_a_receiver_the_robot_dialled() { + let seen = Arc::new(Seen::default()); + let url = receiver(Arc::clone(&seen)).await; + + let (to_socket, from_encoder) = tokio::sync::mpsc::channel::>(2); + let counters = Arc::new(Counters::default()); + let running = Arc::new(AtomicBool::new(true)); + let task = tokio::spawn(pump( + url.clone(), + Some("a-robot-token".to_owned()), + r#"{"type":"hello"}"#.to_owned(), + from_encoder, + Arc::clone(&counters), + Arc::clone(&running), + Timings::default(), + )); + + to_socket.send(vec![0xff, 0xd8, 1, 2]).await.unwrap(); + to_socket.send(vec![0xff, 0xd8, 3, 4]).await.unwrap(); + until("two frames", || seen.frames.lock().unwrap().len() == 2).await; + + assert_eq!( + seen.bearers.lock().unwrap()[0].as_deref(), + Some("Bearer a-robot-token"), + "the receiver has to be able to say whose camera this is" + ); + assert_eq!( + seen.hello.lock().unwrap()[0], + r#"{"type":"hello"}"#, + "and what it is about to be sent, before the first frame" + ); + assert_eq!(seen.frames.lock().unwrap()[1], vec![0xff, 0xd8, 3, 4]); + assert_eq!(counters.sent.load(Ordering::Relaxed), 2); + assert!(counters.connected.load(Ordering::Relaxed)); + + // Stopping is dropping the encoder's end: the pump drains and returns. + running.store(false, Ordering::Relaxed); + drop(to_socket); + tokio::time::timeout(Duration::from_secs(5), task) + .await + .expect("the pump stops when its frames do") + .unwrap(); + assert!(!counters.connected.load(Ordering::Relaxed)); + } + + /// **A gap in a predicted stream is abandoned to the next keyframe, not patched over.** + /// + /// The drop policy is the one thing H.264 changed about this module, and it is the one thing + /// that fails invisibly: keeping the newest unit is right for JPEG and wrong here, because a + /// P-frame whose reference was dropped decodes to garbage that looks like a bad camera rather + /// than a bad transport. So once anything is dropped, everything is dropped until a keyframe. + /// + /// Driven through the real [`Streamer`], with an encoder that hands out a scripted stream and + /// a receiver that never reads — which is what makes the channel fill. + #[tokio::test] + async fn a_dropped_h264_unit_is_followed_by_a_wait_for_a_keyframe() { + let dir = tempfile::tempdir().unwrap(); + // A never-answering receiver: the pump keeps redialling, so nothing is ever drained and + // the encoder's channel fills after two units. Port 1 is reliably nobody. + let handed = Arc::new(std::sync::Mutex::new(Vec::new())); + let script = Arc::new(std::sync::Mutex::new( + // key, then four predicted, then a key. The first two get into the channel; the rest + // arrive while it is full. + vec![true, false, false, false, false, true, false] + .into_iter() + .collect::>(), + )); + + let seen = Arc::clone(&handed); + let remaining = Arc::clone(&script); + let encode: Encode = Arc::new(move |_config| { + let keyframe = remaining.lock().unwrap().pop_front()?; + seen.lock().unwrap().push(keyframe); + Some(Unit { + bytes: vec![if keyframe { 0x65 } else { 0x41 }], + keyframe, + }) + }); + + let streamer = Streamer::new( + Encoders { + jpeg: Arc::clone(&encode), + h264: Some(encode), + gate: None, + }, + crate::producer::Producer { + name: Some("olducky".to_owned()), + serial: None, + release: "0.10.0".to_owned(), + api_version: 23, + }, + 90, + dir.path().join("hf-token"), + ) + .with_timings(Timings { + backoff_start: Duration::from_millis(10), + backoff_max: Duration::from_millis(10), + jitter: 0.0, + }); + + let mut config = config("ws://127.0.0.1:1/frames"); + config.encoding = Encoding::H264; + streamer.start(config).expect("started"); + + // Everything the script offered is consumed, and the drops are counted. + until("the script to be exhausted", || { + handed.lock().unwrap().len() == 7 + }) + .await; + let status = streamer.status(); + assert!( + status["dropped"].as_u64().unwrap() >= 4, + "the units after the gap were dropped: {status}" + ); + assert_eq!( + status["sent"].as_u64().unwrap(), + 0, + "nothing ever connected: {status}" + ); + streamer.stop(); + } + + /// **A receiver going away is the ordinary case, not the end of the stream.** + /// + /// A Space restarts on every push and sleeps when nobody is looking at it, so the first dial + /// landing on a socket that hangs up immediately is what a robot should expect. What must + /// survive it is the stream: the next frame goes to the next connection, and the hello goes + /// with it, because the receiver on the other end of a redial is a new process that was told + /// nothing. + #[tokio::test] + async fn a_receiver_that_hangs_up_is_redialled() { + let seen = Arc::new(Seen::default()); + seen.hang_up.store(1, Ordering::SeqCst); + let url = receiver(Arc::clone(&seen)).await; + + let (to_socket, from_encoder) = tokio::sync::mpsc::channel::>(2); + let counters = Arc::new(Counters::default()); + let running = Arc::new(AtomicBool::new(true)); + tokio::spawn(pump( + url, + None, + r#"{"type":"hello"}"#.to_owned(), + from_encoder, + Arc::clone(&counters), + Arc::clone(&running), + // Milliseconds rather than seconds: this asserts a reconnect happens, not how patient + // the real one is. + Timings { + backoff_start: Duration::from_millis(20), + backoff_max: Duration::from_millis(40), + jitter: 0.1, + }, + )); + + until("the first dial to be hung up on", || { + counters.reconnects.load(Ordering::Relaxed) >= 1 + }) + .await; + + to_socket.send(vec![0xff, 0xd8, 9]).await.unwrap(); + until("a frame after the redial", || { + !seen.frames.lock().unwrap().is_empty() + }) + .await; + assert_eq!(seen.frames.lock().unwrap()[0], vec![0xff, 0xd8, 9]); + assert!( + !seen.hello.lock().unwrap().is_empty(), + "every connection opens with its own hello, since the receiver is new each time" + ); + running.store(false, Ordering::Relaxed); + } +} diff --git a/mediad/src/turn.rs b/mediad/src/turn.rs new file mode 100644 index 00000000..a5cf51e4 --- /dev/null +++ b/mediad/src/turn.rs @@ -0,0 +1,561 @@ +//! Relay candidates, so a consumer that cannot reach this robot directly still can. +//! +//! `webrtcsink` gathers **host** candidates (this robot's own addresses) and **srflx** ones (what +//! a STUN server says its public address is). Between two peers on one network the host +//! candidates pair immediately. Between a robot behind a home router and a consumer behind +//! whatever a cloud provider gives a container, srflx-to-srflx needs both NATs to allow a hole to +//! be punched — often they do, and often enough they do not, and the failure looks like a session +//! that negotiates perfectly and carries nothing. +//! +//! A **TURN** server is a relay in the middle: a peer reserves an address on it and hands that out +//! as a `relay` candidate, and the other side simply sends there. It always works, at the cost of +//! somebody's bandwidth, which is why it is the last resort ICE tries rather than the first. +//! +//! # Only the robot offers one, and that is not a simplification +//! +//! A connection needs **one** relay candidate, not two: if this robot offers one, a consumer that +//! can reach the internet at all can use it. So the credentials live here and a consumer needs +//! none — which matters more than it sounds, because `aiortc`'s STUN client works where its TURN +//! client does not, so a Python consumer *cannot* be the side that relays. +//! `reachy_mini`'s #1182 established this and it is the same arrangement here. +//! +//! # The credentials are short-lived, and fetching them must never be in the way +//! +//! Hugging Face hosts a proxy that mints Cloudflare TURN credentials for an account, which is why +//! this needs the same token the relay signs in with and no new secret anywhere. They expire, so +//! a task refreshes them at half their lifetime. +//! +//! **[`Relays::uris`] never blocks and never fails**, and that is the whole design of this module. +//! Its only caller runs inside GStreamer's `consumer-added` signal, where the SDP offer for that +//! consumer is not generated until the handler returns — so an HTTP request there would delay +//! every connection, including the LAN ones that will never use a relay, by however long the +//! proxy takes to answer. It reads what the refresher last stored, or nothing. + +use std::path::PathBuf; +use std::sync::{Arc, RwLock}; +use std::time::Duration; + +use serde::Deserialize; + +/// Hugging Face's TURN credentials proxy, addressed as the Space it is. +/// +/// **Not `turn.fastrtc.org`**, the vanity name `fastrtc`'s own code points at and `reachy_mini` +/// #1182 copied. That name is a dangling delegation: the `.org` registry still names four Route53 +/// nameservers for the zone, the hosted zone behind them is gone, and all four answer `REFUSED` +/// for the zone they are authoritative for. The service never stopped answering — only the name +/// in front of it did — so this addresses `fastrtc/turn-service` directly. +/// +/// Which also takes a name that can be *taken over* out of the path: a signed-in robot sends its +/// account token down this URL every five minutes, and whoever wins a race to have AWS assign +/// them one of those four delegated nameservers would serve records for the name, pass DNS +/// validation for a certificate on it, and be handed the token. `reachy_mini` #1408 made the same +/// move and measured a relay pair carrying video through it. +pub const DEFAULT_TURN_ENDPOINT: &str = "https://fastrtc-turn-service.hf.space/credentials"; + +/// A `--turn-url` worth handing the account token to, or the reason it is not one. +/// +/// The token goes out as a bearer header on every refresh, so the destination is checked before +/// it can: **`https`**, unless the host is loopback and the endpoint is therefore a test fake or +/// a stand-in on the board itself. Userinfo is refused because one credential per request is +/// enough, and a query or fragment because [`fetch`] appends `?ttl=` to whatever it is given — +/// silently landing the TTL in a fragment, or as a second value of an existing parameter. +/// +/// A `clap` `value_parser`, so a wrong value stops the daemon while somebody is still looking at +/// the terminal. An endpoint that is wrong rather than refused becomes a warning every thirty +/// seconds for the life of the daemon, which is how a log stops being read. +pub fn parse_endpoint(value: &str) -> Result { + let url = url::Url::parse(value).map_err(|why| format!("not a URL: {why}"))?; + + if !url.username().is_empty() || url.password().is_some() { + return Err("carries userinfo, and one credential per request is enough".to_owned()); + } + if url.query().is_some() || url.fragment().is_some() { + return Err("carries a query or a fragment, and the ttl is appended to it".to_owned()); + } + + let loopback = match url.host() { + Some(url::Host::Domain(name)) => name == "localhost", + Some(url::Host::Ipv4(address)) => address.is_loopback(), + Some(url::Host::Ipv6(address)) => address.is_loopback(), + None => return Err("names no host".to_owned()), + }; + match url.scheme() { + "https" => {} + "http" if loopback => {} + "http" => { + return Err("is plain http, which puts the account token on the wire".to_owned()); + } + other => return Err(format!("{other} is not a scheme these can be fetched over")), + } + + Ok(url.into()) +} + +/// How long to ask for the credentials to be valid. +const TTL: Duration = Duration::from_secs(600); + +/// Refresh at half of [`TTL`], so a credential is replaced well before it expires. +const REFRESH_RATIO: f64 = 0.5; + +/// How long to wait after a *transient* failure. +/// +/// Only a transient one earns the short retry. A robot nobody has signed in has nothing to retry +/// for, and this task runs for the daemon's whole life — retrying that fast would put a line in +/// the journal every half minute forever. +const RETRY_AFTER_FAILURE: Duration = Duration::from_secs(30); + +const HTTP_TIMEOUT: Duration = Duration::from_secs(10); + +/// What the proxy answers with, of which two fields are read. +/// +/// `iceServers` is camelCase on the wire — it is the browser's `RTCConfiguration` shape, which is +/// where these dicts are destined — while the keys *inside* a `meta` on the rendezvous are +/// snake_case. Two conventions in one system, and the cost of assuming either is a field that +/// silently deserialises to empty. +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +struct Credentials { + #[serde(default)] + ice_servers: Vec, +} + +#[derive(Debug, Deserialize)] +struct IceServer { + /// One URL or several; the proxy sends both shapes. + urls: Urls, + #[serde(default)] + username: Option, + #[serde(default)] + credential: Option, +} + +#[derive(Debug, Deserialize)] +#[serde(untagged)] +enum Urls { + One(String), + Many(Vec), +} + +impl Urls { + fn iter(&self) -> impl Iterator { + match self { + Urls::One(url) => std::slice::from_ref(url).iter(), + Urls::Many(urls) => urls.iter(), + } + } +} + +/// The relay servers this robot currently holds, if any. +#[derive(Debug, Default)] +pub struct Relays { + /// Replaced whole rather than mutated, so a reader sees the previous set or the new one and + /// never half of either. + uris: RwLock>, +} + +impl Relays { + /// A holder with nothing in it, which is also what a robot has until the first refresh. + pub fn empty() -> Arc { + Arc::new(Self::default()) + } + + /// The `turn://` URIs to offer, or empty. + /// + /// **Never blocks and never panics.** `try_read` rather than `read`: the writer holds the lock + /// for one assignment, so contention is close to impossible — and if it happens, offering + /// host and srflx candidates for one consumer is a far better outcome than stalling a + /// GStreamer signal handler, which stalls that consumer's offer. + pub fn uris(&self) -> Arc<[String]> { + match self.uris.try_read() { + Ok(held) => Arc::clone(&held), + Err(_) => Arc::from([] as [String; 0]), + } + } + + fn store(&self, uris: Vec) { + if let Ok(mut held) = self.uris.write() { + *held = Arc::from(uris); + } + } +} + +/// Keep [`Relays`] fresh for as long as this process runs. +/// +/// Spawned once at startup, whatever the account state: a robot signed in later starts offering +/// relay candidates without a restart, which is the same property the relay itself has. +pub async fn maintain(relays: Arc, token_path: PathBuf, endpoint: String) { + let period = TTL.mul_f64(REFRESH_RATIO); + let mut said_there_is_no_token = false; + + loop { + let wait = match hf_robot_account::read_access_token(&token_path) { + None => { + // The steady state of a robot nobody has signed in. Said once, because it is not + // news every five minutes for the life of the daemon. + if !said_there_is_no_token { + said_there_is_no_token = true; + tracing::info!( + "no account token, so this robot offers no relay candidates; a login is \ + what lets it be reached from a network that cannot punch a hole to it" + ); + } + period + } + Some(token) => { + said_there_is_no_token = false; + match fetch(&endpoint, &token).await { + Ok(uris) if uris.is_empty() => { + // The proxy answered and offered no relay. Not worth hammering. + tracing::info!("the TURN proxy offered no relay servers"); + period + } + Ok(uris) => { + tracing::info!( + servers = uris.len(), + hosts = ?uris.iter().map(|uri| hostname(uri)).collect::>(), + "refreshed the relay credentials" + ); + relays.store(uris); + period + } + // Best effort, always: a robot with no relay candidates is reachable from + // most places, and one that refused to stream because a proxy was down would + // be reachable from none. + Err(why) => { + tracing::warn!(%why, "could not fetch relay credentials"); + RETRY_AFTER_FAILURE + } + } + } + }; + tokio::time::sleep(wait).await; + } +} + +/// One request to the proxy, turned into the URIs `webrtcbin` takes. +async fn fetch(endpoint: &str, token: &str) -> Result, String> { + let client = reqwest::Client::builder() + .timeout(HTTP_TIMEOUT) + .build() + .map_err(|e| format!("no HTTP client: {e}"))?; + + // The TTL in the URL rather than through `query`, which wants a `reqwest` feature this + // workspace does not enable — and one integer needs no encoder. + let separator = if endpoint.contains('?') { '&' } else { '?' }; + let url = format!("{endpoint}{separator}ttl={}", TTL.as_secs()); + let response = client + .get(&url) + .bearer_auth(token) + .send() + .await + .map_err(|e| format!("GET {endpoint}: {}", because(&e)))?; + if !response.status().is_success() { + return Err(format!("GET {endpoint}: HTTP {}", response.status())); + } + let credentials: Credentials = response + .json() + .await + .map_err(|e| format!("GET {endpoint}: {}", because(&e)))?; + Ok(turn_uris(&credentials.ice_servers)) +} + +/// `{"urls", "username", "credential"}` to `turn://user:pass@host:port`. +/// +/// `stun:` entries and entries with no credentials are skipped: `webrtcbin` takes a STUN server +/// through its own property, and `add-turn-server` rejects a URI with no userinfo. +fn turn_uris(servers: &[IceServer]) -> Vec { + let mut uris = Vec::new(); + for server in servers { + let (Some(user), Some(secret)) = (&server.username, &server.credential) else { + continue; + }; + for url in server.urls.iter() { + let (scheme, rest) = match url.split_once(':') { + Some(parts) => parts, + None => continue, + }; + if !matches!(scheme.to_ascii_lowercase().as_str(), "turn" | "turns") || rest.is_empty() + { + continue; + } + // Percent-encoded, so a password containing `:`, `@` or `/` cannot corrupt the URI — + // and these are generated secrets, so it will eventually contain one of them. + uris.push(format!( + "{scheme}://{}:{}@{rest}", + encode(user), + encode(secret) + )); + } + } + uris +} + +/// The userinfo half of a URI, with everything that has meaning there escaped. +fn encode(value: &str) -> String { + let mut out = String::with_capacity(value.len()); + for byte in value.bytes() { + match byte { + b'A'..=b'Z' | b'a'..=b'z' | b'0'..=b'9' | b'-' | b'.' | b'_' | b'~' => { + out.push(byte as char); + } + other => out.push_str(&format!("%{other:02X}")), + } + } + out +} + +/// An error and everything under it, on one line. +/// +/// **`reqwest`'s own message stops at "error sending request"**, and the half that matters is +/// underneath: a name that does not resolve, a refused connection and a certificate that does not +/// verify all print identically otherwise. This cost an afternoon of wondering whether a robot had +/// no network, when the answer was that the endpoint's whole domain had no DNS records — +/// unresolvable from the board and from three public resolvers alike. +fn because(error: &(dyn std::error::Error + 'static)) -> String { + let mut out = error.to_string(); + let mut source = error.source(); + while let Some(cause) = source { + // Repeated text is worse than none: `reqwest` wraps `hyper` wraps `io`, and each layer + // often restates the one below it. + let text = cause.to_string(); + if !out.contains(&text) { + out.push_str(": "); + out.push_str(&text); + } + source = cause.source(); + } + out +} + +/// A URI with its credentials removed, which is the only form that may be logged. +fn hostname(uri: &str) -> String { + uri.rsplit_once('@') + .map(|(_, host)| host.to_owned()) + .unwrap_or_else(|| uri.to_owned()) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn servers(json: &str) -> Vec { + serde_json::from_str::(json) + .unwrap() + .ice_servers + } + + /// What the proxy sends, in both shapes it sends it. + #[test] + fn credentials_become_uris_webrtcbin_accepts() { + let uris = turn_uris(&servers( + r#"{"iceServers":[ + {"urls":"stun:stun.cloudflare.com:3478"}, + {"urls":["turn:turn.cloudflare.com:3478?transport=udp", + "turns:turn.cloudflare.com:5349?transport=tcp"], + "username":"user-1","credential":"secret-1"} + ]}"#, + )); + + assert_eq!( + uris, + vec![ + "turn://user-1:secret-1@turn.cloudflare.com:3478?transport=udp", + "turns://user-1:secret-1@turn.cloudflare.com:5349?transport=tcp", + ], + "one URI per URL, and the STUN entry is not one of them" + ); + } + + /// A generated secret contains `:` and `/` sooner or later, and an unescaped one silently + /// produces a URI naming the wrong host — with credentials in it, so it cannot be logged to + /// find out. + #[test] + fn credentials_are_escaped_rather_than_interpolated() { + let uris = turn_uris(&servers( + r#"{"iceServers":[{"urls":"turn:relay:3478", + "username":"a:b@c","credential":"p/q?r=s"}]}"#, + )); + assert_eq!(uris, vec!["turn://a%3Ab%40c:p%2Fq%3Fr%3Ds@relay:3478"]); + } + + /// An entry with no credentials is not a relay this robot can offer. + #[test] + fn entries_without_credentials_are_skipped() { + assert!(turn_uris(&servers(r#"{"iceServers":[{"urls":"turn:relay:3478"}]}"#)).is_empty()); + assert!(turn_uris(&servers(r#"{"iceServers":[]}"#)).is_empty()); + } + + /// **The default is one the guard accepts.** A default that fails its own check is a daemon + /// that will not start at all, on every robot at once. + #[test] + fn the_default_endpoint_is_one_the_token_may_be_sent_to() { + assert_eq!( + parse_endpoint(DEFAULT_TURN_ENDPOINT).as_deref(), + Ok(DEFAULT_TURN_ENDPOINT), + "and it survives the round trip unchanged, so `fetch` appends `?ttl=` to what was set" + ); + } + + /// What the guard is for: the destinations the account token must not go to. + #[test] + fn an_endpoint_that_would_leak_the_token_is_refused_at_argument_parsing() { + for (endpoint, because) in [ + ("http://turn.example/credentials", "plain http"), + ("https://user:pass@turn.example/credentials", "userinfo"), + ("https://turn.example/credentials?ttl=1", "query"), + ("https://turn.example/credentials#f", "fragment"), + ("ftp://turn.example/credentials", "scheme"), + ("/credentials", "relative"), + ] { + assert!( + parse_endpoint(endpoint).is_err(), + "{endpoint} was accepted, and it should have been refused for its {because}" + ); + } + } + + /// And loopback http is not one of them — every test below dials one. + #[test] + fn a_loopback_fake_needs_no_certificate() { + for endpoint in [ + "http://127.0.0.1:8080/credentials", + "http://localhost:8080/credentials", + "http://[::1]:8080/credentials", + ] { + assert!(parse_endpoint(endpoint).is_ok(), "{endpoint}"); + } + } + + /// An unreachable endpoint says *why* it was unreachable. + #[tokio::test] + async fn a_failure_names_its_cause_and_not_just_itself() { + // A domain that cannot resolve, which is exactly what the real endpoint did. + let error = fetch("https://turn.invalid./credentials", "hf_abc") + .await + .expect_err("`.invalid` does not resolve, by RFC 2606"); + assert!( + error.to_lowercase().contains("dns") || error.to_lowercase().contains("resolve"), + "the cause has to survive to the log line: {error}" + ); + } + + /// **Credentials must never reach a log line.** + #[test] + fn only_the_host_is_loggable() { + assert_eq!( + hostname("turns://user-1:secret-1@turn.cloudflare.com:5349"), + "turn.cloudflare.com:5349" + ); + assert!(!hostname("turns://user-1:secret-1@relay:5349").contains("secret-1")); + // A URI with no userinfo cannot leak one, and is logged whole. + assert_eq!(hostname("turn:relay:3478"), "turn:relay:3478"); + } + + /// An empty holder answers, rather than making a caller handle "not yet". + #[test] + fn a_robot_that_has_fetched_nothing_offers_nothing() { + let relays = Relays::empty(); + assert!(relays.uris().is_empty()); + + relays.store(vec!["turn://u:p@relay:3478".to_owned()]); + assert_eq!(relays.uris().len(), 1); + + // Replaced whole, so a reader never sees a mixture of two sets. + relays.store(vec![ + "turn://u2:p2@relay:3478".to_owned(), + "turns://u2:p2@relay:5349".to_owned(), + ]); + assert_eq!(relays.uris().len(), 2); + assert!(relays.uris().iter().all(|uri| uri.contains("u2"))); + } + + /// The proxy refusing is not this robot's problem to solve, only to report. + #[tokio::test] + async fn a_proxy_that_refuses_leaves_the_robot_without_relays() { + let app = axum::Router::new().route( + "/credentials", + axum::routing::get(|| async { axum::http::StatusCode::UNAUTHORIZED }), + ); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let endpoint = format!("http://{}/credentials", listener.local_addr().unwrap()); + tokio::spawn(async move { + let _ = axum::serve(listener, app).await; + }); + + let error = fetch(&endpoint, "hf_abc").await.expect_err("a 401"); + assert!(error.contains("401"), "{error}"); + } + + /// And the ordinary path, end to end against a proxy that answers what HF's answers. + #[tokio::test] + async fn a_refresh_stores_what_the_proxy_offers() { + use std::sync::atomic::{AtomicUsize, Ordering}; + + let asked = std::sync::Arc::new(AtomicUsize::new(0)); + let seen = std::sync::Arc::clone(&asked); + let app = axum::Router::new().route( + "/credentials", + axum::routing::get(move |headers: axum::http::HeaderMap| { + let seen = std::sync::Arc::clone(&seen); + async move { + seen.fetch_add(1, Ordering::SeqCst); + assert_eq!( + headers.get("authorization").unwrap(), + "Bearer hf_abc", + "the robot's own token is what mints these" + ); + axum::Json(serde_json::json!({ + "iceServers": [{ + "urls": ["turn:turn.cloudflare.com:3478?transport=udp"], + "username": "u", "credential": "p", + }], + })) + } + }), + ); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let endpoint = format!("http://{}/credentials", listener.local_addr().unwrap()); + tokio::spawn(async move { + let _ = axum::serve(listener, app).await; + }); + + let dir = tempfile::tempdir().unwrap(); + let token_path = dir.path().join("hf-token"); + std::fs::write(&token_path, r#"{"access_token":"hf_abc"}"#).unwrap(); + + let relays = Relays::empty(); + let task = tokio::spawn(maintain( + std::sync::Arc::clone(&relays), + token_path, + endpoint, + )); + for _ in 0..100 { + if !relays.uris().is_empty() { + break; + } + tokio::time::sleep(Duration::from_millis(20)).await; + } + task.abort(); + + assert_eq!( + relays.uris().as_ref(), + ["turn://u:p@turn.cloudflare.com:3478?transport=udp".to_owned()] + ); + assert_eq!(asked.load(Ordering::SeqCst), 1, "once, not on a spin"); + } + + /// A robot with no account asks nobody for credentials. + #[tokio::test] + async fn no_token_means_no_request() { + let dir = tempfile::tempdir().unwrap(); + let relays = Relays::empty(); + let task = tokio::spawn(maintain( + std::sync::Arc::clone(&relays), + dir.path().join("hf-token"), + "http://127.0.0.1:1/credentials".to_owned(), + )); + tokio::time::sleep(Duration::from_millis(200)).await; + task.abort(); + assert!(relays.uris().is_empty()); + } +} diff --git a/mediad/src/upstream.rs b/mediad/src/upstream.rs index b7eaf4e8..583fb116 100644 --- a/mediad/src/upstream.rs +++ b/mediad/src/upstream.rs @@ -103,6 +103,15 @@ impl Pool { } /// Send one line to `service` on `lane`'s connection, connecting first if needed. + /// + /// A daemon that restarted between two requests is ordinary here: `robotd` is the one an + /// update restarts, and `updaterd` restarts itself from a release's postinstall hook. The + /// connection from before the restart is still in the pool, its reader has already seen the + /// socket close, and the first write into it fails. Those bytes never left, so they are written + /// again on a fresh connection rather than reported. Reporting them told the console that a + /// daemon listening on a fresh socket was not answering, once per lane, after every restart. + /// Once and not in a loop: a daemon that is genuinely gone fails the reconnect, and that is + /// the error worth reporting. pub async fn send( &mut self, service: proto::Service, @@ -110,25 +119,35 @@ impl Pool { line: &str, ) -> io::Result<()> { let key = (service, lane); + let mut bytes = line.as_bytes().to_vec(); + bytes.push(b'\n'); + if !self.conns.contains_key(&key) { let conn = self.open(service, lane).await?; self.conns.insert(key, conn); } - let conn = self.conns.get_mut(&key).expect("just inserted"); - - let mut bytes = line.as_bytes().to_vec(); - bytes.push(b'\n'); + match self.write(key, &bytes).await { + Err(e) if peer_is_gone(&e) => { + let conn = self.open(service, lane).await?; + self.conns.insert(key, conn); + self.write(key, &bytes).await + } + done => done, + } + } + /// One bounded write on the connection for `key`. Any failure drops that connection, so + /// nothing keeps writing into a dead socket. Only this lane's: the others may be perfectly + /// alive. + async fn write(&mut self, key: (proto::Service, proto::Lane), bytes: &[u8]) -> io::Result<()> { + let conn = self.conns.get_mut(&key).expect("connection present"); let write = async { - conn.write.write_all(&bytes).await?; + conn.write.write_all(bytes).await?; conn.write.flush().await }; match tokio::time::timeout(WRITE_TIMEOUT, write).await { Ok(Ok(())) => Ok(()), Ok(Err(e)) => { - // A broken pipe here is ordinary — the daemon restarted. Drop this lane's - // connection so the next call reconnects rather than writing into a dead socket - // forever. Only this lane's: the others may be perfectly alive. self.conns.remove(&key); Err(e) } @@ -183,3 +202,100 @@ impl Pool { Ok(Conn { write }) } } + +/// Whether a write failed because the peer went away, rather than being slow or refusing. Only +/// these are worth one more try, because only these mean the bytes went to a daemon that is no +/// longer there. A timeout is a daemon that is there and stuck, and that is not retried. +fn peer_is_gone(e: &io::Error) -> bool { + matches!( + e.kind(), + io::ErrorKind::BrokenPipe + | io::ErrorKind::ConnectionReset + | io::ErrorKind::ConnectionAborted + | io::ErrorKind::NotConnected + ) +} + +#[cfg(test)] +mod tests { + use super::*; + use tokio::io::{AsyncBufReadExt, AsyncWriteExt, BufReader}; + + /// A fake daemon that answers one request with `response` and hangs up. + fn serve_once(path: &Path, response: proto::Response) -> tokio::task::JoinHandle { + let listener = tokio::net::UnixListener::bind(path).unwrap(); + tokio::spawn(async move { + let (stream, _) = listener.accept().await.unwrap(); + let (read, mut write) = stream.into_split(); + let request = BufReader::new(read) + .lines() + .next_line() + .await + .unwrap() + .unwrap(); + let mut line = serde_json::to_vec(&response).unwrap(); + line.push(b'\n'); + write.write_all(&line).await.unwrap(); + write.flush().await.unwrap(); + request + }) + } + + /// The daemon restarted between two requests. + /// + /// The same pool `btd` has, with the same hole it had: the first write after a restart went + /// into the socket the old daemon closed, failed, and the console was told the service was + /// not answering while it was listening on a fresh socket the whole time. + #[tokio::test] + async fn a_restarted_daemon_gets_the_next_request_not_the_one_after() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("robotd.sock"); + let sockets = Sockets { + robot: path.clone(), + updater: dir.path().join("updaterd.sock"), + config: dir.path().join("configd.sock"), + pad: dir.path().join("padd.sock"), + tof: dir.path().join("tofd.sock"), + }; + let (replies, mut forwarded) = mpsc::channel(8); + let mut pool = Pool::new(sockets, replies); + + let before = serve_once( + &path, + proto::Response::ok(Some(proto::Id::Number(1)), &serde_json::json!({})), + ); + pool.send( + proto::Service::Robot, + proto::Lane::Prompt, + r#"{"jsonrpc":"2.0","id":1,"method":"robot.health"}"#, + ) + .await + .unwrap(); + assert!( + forwarded.recv().await.is_some(), + "the first answer is forwarded" + ); + // The daemon has answered and hung up: it is restarting. + before.await.unwrap(); + + // And it is back, listening on a fresh socket at the same path. + std::fs::remove_file(&path).unwrap(); + let after = serve_once( + &path, + proto::Response::ok(Some(proto::Id::Number(2)), &serde_json::json!({})), + ); + pool.send( + proto::Service::Robot, + proto::Lane::Prompt, + r#"{"jsonrpc":"2.0","id":2,"method":"robot.health"}"#, + ) + .await + .expect("a daemon that is back must get the request, not a broken pipe"); + let request = after.await.unwrap(); + assert!(request.contains(r#""id":2"#), "{request}"); + assert!( + forwarded.recv().await.is_some(), + "and its answer is forwarded" + ); + } +} diff --git a/mediad/src/web.rs b/mediad/src/web.rs index 8ab2be1e..2bd649ae 100644 --- a/mediad/src/web.rs +++ b/mediad/src/web.rs @@ -1,6 +1,6 @@ //! The page, served by the daemon it drives. //! -//! One route, one file, no build step. `http://:8080/` and there is nothing else to run — +//! An embedded console and an on-demand PNG snapshot, with no frontend build step. `http://:8080/` and there is nothing else to run — //! which is the whole of it, and `webrtc-console.md` §1 is why it is worth a dependency and a //! second port. //! @@ -38,6 +38,8 @@ //! and say nothing about why. use std::net::SocketAddr; +use std::path::PathBuf; +use std::sync::Arc; use anyhow::{Context, Result}; use axum::Router; @@ -72,7 +74,7 @@ pub fn page(signalling_port: u32) -> String { /// Returns only on failure — a bind that was refused, or a listener that died. The caller decides /// what that costs; in `mediad` it costs the page and not the video, because a robot that streams /// and answers control calls with no console is a great deal better than one that does neither. -pub async fn serve(host: &str, port: u16, page: String) -> Result<()> { +pub async fn serve(host: &str, port: u16, page: String, frame_socket: PathBuf) -> Result<()> { let address: SocketAddr = format!("{host}:{port}") .parse() .with_context(|| format!("{host}:{port} is not an address to listen on"))?; @@ -81,20 +83,151 @@ pub async fn serve(host: &str, port: u16, page: String) -> Result<()> { .with_context(|| format!("could not listen on {address}"))?; tracing::info!(%address, "serving the console"); - axum::serve(listener, router(page)) + axum::serve(listener, router(page, frame_socket)) .await .context("the console's listener stopped") } -/// One route, returning `page`. -fn router(page: String) -> Router { - Router::new().route("/", get(move || std::future::ready(Html(page)))) +/// The console and its bounded, uncached snapshot endpoint. +fn router(page: String, frame_socket: PathBuf) -> Router { + let slots = Arc::new(tokio::sync::Semaphore::new(4)); + Router::new() + .route("/", get(move || std::future::ready(Html(page)))) + .route( + "/frame", + get(move || snapshot(frame_socket.clone(), slots.clone())), + ) +} + +async fn snapshot(socket: PathBuf, slots: Arc) -> axum::response::Response { + use axum::http::{StatusCode, header}; + use axum::response::IntoResponse; + let result: Result> = async { + let permit = slots + .try_acquire_owned() + .context("snapshot capacity reached")?; + let (metadata, pixels) = crate::snapshot::fetch(&socket).await?; + tokio::task::spawn_blocking(move || { + // The permit belongs to the encoder, even if the HTTP client disconnects. + let _permit = permit; + crate::snapshot::png(metadata, pixels) + }) + .await? + } + .await; + let mut response = match result { + Ok(png) => ([(header::CONTENT_TYPE, "image/png")], png).into_response(), + Err(error) => { + tracing::debug!(%error, "snapshot unavailable"); + ( + StatusCode::SERVICE_UNAVAILABLE, + "Camera snapshot unavailable; retry when capture is running.\n", + ) + .into_response() + } + }; + response + .headers_mut() + .insert(header::CACHE_CONTROL, "no-store".parse().unwrap()); + response } #[cfg(test)] mod tests { use super::*; + /// The PNG comes out upright, because a picture is the one reply with nowhere to carry the + /// mount angle. A quarter turn swaps the axes and moves the bright pixel; an upright mount + /// leaves both alone, and the same route has to do each. + #[tokio::test] + async fn frame_route_handshakes_and_returns_a_decodable_uncached_upright_png() { + for (rotate, size, bright) in [(0, (2, 1), (1, 0)), (90, (1, 2), (0, 1))] { + frame_route_returns(rotate, size, bright).await; + } + } + + async fn frame_route_returns(rotate: u32, size: (u32, u32), bright: (u32, u32)) { + use tokio::io::{AsyncBufReadExt, AsyncWriteExt, BufReader}; + let dir = tempfile::tempdir().unwrap(); + let socket = dir.path().join("media.sock"); + let camera = tokio::net::UnixListener::bind(&socket).unwrap(); + let producer = tokio::spawn(async move { + let (stream, _) = camera.accept().await.unwrap(); + let (read, mut write) = stream.into_split(); + let mut reader = BufReader::new(read); + let mut line = String::new(); + reader.read_line(&mut line).await.unwrap(); + let hello: proto::Request = serde_json::from_str(&line).unwrap(); + assert_eq!(hello.method, proto::method::HELLO); + assert_eq!(hello.params.unwrap()["api_version"], proto::API_VERSION); + let response = proto::Response::ok( + hello.id, + &proto::HelloResult { + api_version: proto::API_VERSION, + daemon_version: None, + revision: None, + }, + ); + write + .write_all(format!("{}\n", serde_json::to_string(&response).unwrap()).as_bytes()) + .await + .unwrap(); + line.clear(); + reader.read_line(&mut line).await.unwrap(); + let request: proto::Request = serde_json::from_str(&line).unwrap(); + assert_eq!(request.method, proto::method::MEDIA_FRAME); + let header = proto::MediaFrameHeader { + width: 2, + height: 1, + format: "UYVY".into(), + bytes: 4, + captured_at_unix_us: 1, + rotate, + }; + let mut reply = serde_json::to_vec(&proto::Response::ok(request.id, &header)).unwrap(); + reply.push(b'\n'); + reply.extend([128, 16, 128, 235]); + write.write_all(&reply).await.unwrap(); + }); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap(); + let server = tokio::spawn(async move { + axum::serve(listener, router(page(8443), socket)) + .await + .unwrap(); + }); + let response = reqwest::get(format!("http://{address}/frame")) + .await + .unwrap(); + assert_eq!(response.status(), 200); + assert_eq!(response.headers()["content-type"], "image/png"); + assert_eq!(response.headers()["cache-control"], "no-store"); + let decoded = image::load_from_memory(&response.bytes().await.unwrap()) + .unwrap() + .to_rgb8(); + assert_eq!(decoded.dimensions(), size); + assert!(decoded.get_pixel(bright.0, bright.1)[0] > 250); + // The dark pixel is the frame's first either way round: a clockwise turn takes source + // (0, 0) to upright (0, 0), so only the bright one has to move for the turn to be real. + assert!(decoded.get_pixel(0, 0)[0] < 5); + producer.await.unwrap(); + server.abort(); + } + + #[tokio::test] + async fn unavailable_and_busy_camera_fail_without_caching() { + let dir = tempfile::tempdir().unwrap(); + for slots in [0, 1] { + let response = snapshot( + dir.path().join("missing.sock"), + Arc::new(tokio::sync::Semaphore::new(slots)), + ) + .await; + assert_eq!(response.status(), 503); + assert_eq!(response.headers()["cache-control"], "no-store"); + } + } + /// The whole of what this module does to the page. #[test] fn both_tokens_are_filled_in() { @@ -121,6 +254,27 @@ mod tests { ); } + /// **The page the robot serves must be in LAN mode, and the page in the repository must not + /// be.** + /// + /// One file serves two hosts: substituting the port is what tells it a robot served it, and + /// the copy pushed to the Space keeps the token so the page reaches for the rendezvous + /// instead — a browser on an https page cannot open a `ws://` at all, so getting this + /// backwards produces a console that connects to nothing and says nothing. + /// `scripts/publish-console.sh` asserts the other half of it at deploy time. + #[test] + fn the_unsubstituted_page_is_the_remote_one() { + assert!( + PAGE.contains(PORT_TOKEN), + "the page in the repository must carry the port token: it is what the Space copy \ + reads as `no robot served me`" + ); + assert!( + !page(8443).contains(PORT_TOKEN), + "and the served page must not, or the robot would serve a page that ignores it" + ); + } + /// A non-default `--port` reaches the page, which is the reason the substitution exists at all: /// a page carrying a constant would still be dialling 8443. #[test] @@ -140,7 +294,11 @@ mod tests { .expect("a loopback port"); let address = listener.local_addr().expect("the port it took"); tokio::spawn(async move { - let _ = axum::serve(listener, router(page(8443))).await; + let _ = axum::serve( + listener, + router(page(8443), PathBuf::from(proto::socket::MEDIA)), + ) + .await; }); let mut stream = tokio::net::TcpStream::connect(address) diff --git a/mediad/webclient/index.html b/mediad/webclient/index.html index adb6e6e7..b9d43e21 100644 --- a/mediad/webclient/index.html +++ b/mediad/webclient/index.html @@ -1,5 +1,16 @@ + + + + duck console