feat: network lab (stream.sh --netem)

.github/workflows/ci.yml

@@ -362,6 +362,70 @@ jobs:
   # tap sits in the pipe, so check the broadcast still comes through, then that
   # the server serves the built UI, its config, and an event stream carrying
   # both container logs and ingest-clock samples.
+  # stream.sh --netem: the relay's traffic to viewers is shaped, the dashboard's
+  # HTTP is not, and the dashboard can switch profiles at runtime.
+  smoke-test-podman-netem:
+    needs: smoke-test-podman-build
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/download-artifact@v8
+        with:
+          name: moq-lab-image
+          path: /tmp
+
+      - uses: actions/download-artifact@v8
+        with:
+          name: moq-lab-assets
+          path: assets
+
+      - name: Load the sandbox image
+        run: gunzip -c /tmp/moq-lab-image.tar.gz | podman load
+
+      # A rootless container can't load kernel modules itself.
+      - name: Load the tc modules
+        run: sudo modprobe -a sch_netem sch_prio cls_u32
+
+      - name: Smoke test -- network lab
+        run: |
+          podman run -d --name ci-netem --init --cap-add NET_ADMIN \
+            -e MOQ_LAB_NETEM=satellite -e MOQ_LAB_DASHBOARD_PORT=18080 -p 18080:18080/tcp \
+            -v "$PWD/assets/ci-test.mp4:/media/input.mp4:ro" \
+            -p 15449:15449/udp -p 15449:15449/tcp \
+            moq-lab /media/input.mp4 ci-test.hang false 15449 false 30 false 6
+
+          ok=false
+          for _ in $(seq 1 30); do
+            if curl -sf http://localhost:15449/announced 2>/dev/null | grep -q "ci-test.hang"; then
+              ok=true; break
+            fi
+            sleep 1
+          done
+          if [ "$ok" != true ]; then
+            echo "broadcast never announced with --netem"; podman logs ci-netem; exit 1
+          fi
+
+          fail=0
+          podman logs ci-netem 2>&1 | grep -q "Network profile 'satellite': qdisc netem" \
+            || { echo "satellite profile not applied at startup"; fail=1; }
+
+          # satellite adds 300 ms to every packet the relay sends out, and nothing to the
+          # dashboard's HTTP (its clock-offset measurement needs a symmetric path).
+          slow() { curl -s -o /dev/null -w '%{time_total}' "$1" | LC_NUMERIC=C awk '{ exit !($1 >= 0.3) }'; }
+          slow http://localhost:15449/announced || { echo "relay traffic is not delayed"; fail=1; }
+          slow http://localhost:18080/api/time && { echo "dashboard traffic is delayed"; fail=1; }
+
+          post() {
+            curl -s -o /dev/null -w '%{http_code}' -X POST -H 'content-type: application/json' \
+              -d "{\"profile\":\"$1\"}" http://localhost:18080/api/netem
+          }
+          [ "$(post off)" = 200 ] || { echo "switching to off failed"; fail=1; }
+          slow http://localhost:15449/announced && { echo "relay traffic still delayed after off"; fail=1; }
+          [ "$(post 'x; reboot')" = 400 ] || { echo "an unknown profile was not refused"; fail=1; }
+
+          [ "$fail" -ne 0 ] && podman logs ci-netem
+          podman rm -f ci-netem
+          exit "$fail"
+
   smoke-test-podman-dashboard:
     needs: smoke-test-podman-build
     runs-on: ubuntu-latest

CHANGELOG.md

@@ -12,6 +12,18 @@ any breaking changes worth flagging for the next person to bump.
 ## Unreleased
 
 ### Added
+- Network lab: `stream.sh --netem PROFILE` (`mobile`, `lossy`,
+  `congested`, `satellite`, `off`) shapes what the relay sends to its
+  viewers with Linux netem. The profiles are in the new `lib/netem.mjs`. With
+  `--dashboard`, a **Network lab** card switches profiles at runtime
+  (`GET`/`POST /api/netem`, which only accepts a profile name and needs
+  `Content-Type: application/json`). Only relay -> viewer traffic is shaped:
+  not the ingest, and not the dashboard's HTTP API. A prio qdisc and a u32
+  filter on the relay port do this on both `eth0` and `lo`, because with
+  rootless Podman the viewer traffic reaches the relay over `lo`. The
+  container gets `NET_ADMIN` only with `--netem`. The image now includes
+  `iproute2`. New CI job `smoke-test-podman-netem`. See README, "Network
+  lab".
 - Dashboard **Player** card: plays the stream with `@moq/watch` 0.6.6 in
   every mode. With `--sgai-mode` it is the SGAI player that was missing: it
   reads the Event Timeline (`dashboard/src/lib/sgai.js`) and switches from

CONTRIBUTING.md

@@ -91,7 +91,7 @@ Podman, so no real content broadcast — the Media Timeline needs one, see
 below). It also checks that no event is lost between the publisher and the
 subscriber (`sgai/event-delivery.ci-check.mjs`). `smoke-test-podman-build` builds the actual sandbox image and the
 synthetic test clip once, sharing both via artifacts with one job per
-pipeline mode (`smoke-test-podman-base`, `-auth`, `-ssai`, `-ssai-abr`,
+pipeline mode (`smoke-test-podman-base`, `-auth`, `-netem`, `-ssai`, `-ssai-abr`,
 `-csai`, `-sgai`, `-dashboard`) — split out so each mode gets its own pass/fail check instead of
 one job with five sequential steps, and a failure in one doesn't block the
 others from reporting. Together they check the relay's HTTP API (broadcast

Containerfile

@@ -37,7 +37,7 @@ RUN pnpm build
 FROM debian:bookworm-slim
 
 RUN apt-get update && apt-get install -y --no-install-recommends \
-    ffmpeg ca-certificates curl nodejs && \
+    ffmpeg ca-certificates curl nodejs iproute2 && \
     rm -rf /var/lib/apt/lists/*
 
 COPY --from=builder /usr/local/bin/moq /usr/local/bin/moq

README.md

@@ -365,6 +365,7 @@ podman run --rm -i --entrypoint moq moq-lab auth sign --key - --subscribe '**' \
 | `--auth-key FILE` | off | Like `--auth`, but sign with the key in `FILE` |
 | `--dashboard` | off | Serve the real-time monitoring web UI on `http://localhost:8080` |
 | `--dashboard-port N` | `8080` | Like `--dashboard`, on port `N` (must differ from `--port`) |
+| `--netem PROFILE` | off | Shape the network between the relay and its viewers: `mobile`, `lossy`, `congested`, `satellite`, or `off` (see [Network lab](#network-lab---netem)) |
 
 ### npm/pnpm shortcut
 
@@ -1031,6 +1032,56 @@ pnpm dev                               # terminal 2: http://localhost:5173
 Changes to the UI reach `./stream.sh --dashboard` the next time the image
 builds, which `stream.sh` does on every run.
 
+### Network lab (`--netem`)
+
+`--netem PROFILE` makes the network between the relay and its viewers worse,
+with Linux [netem](https://man7.org/linux/man-pages/man8/tc-netem.8.html).
+Use it to see how MoQ behaves on a bad network: the charts above show what
+happens to RTT, bitrate, jitter, frame rate, and latency.
+
+```bash
+./stream.sh bbb --dashboard --netem mobile
+```
+
+| Profile | Delay | Loss | Rate |
+|---|---|---|---|
+| `mobile` | 60 ms ± 20 ms | 0.5% | 8 Mbit/s |
+| `lossy` | — | 3% | — |
+| `congested` | 40 ms ± 10 ms | 1% | 3 Mbit/s |
+| `satellite` | 300 ms ± 20 ms | — | 10 Mbit/s |
+| `off` | — | — | — |
+
+With `--dashboard`, the **Network lab** card switches between the profiles
+while the stream runs, so you can compare them on the same stream. The
+profiles are defined in `lib/netem.mjs`. The dashboard can only pick one of
+them by name.
+
+What is shaped:
+
+- Only what the relay sends to its viewers: players, the dashboard page's
+  MoQ sessions, and the host-side SGAI scripts. So the delay is one way: the
+  RTT goes up by the delay, not by twice the delay.
+- Not the ingest pipeline: `moq import` talks to the relay over
+  `localhost` inside the container.
+- Not the dashboard's own HTTP API: it measures the clock offset between
+  your machine and the container, which needs the same delay both ways.
+
+Good to know:
+
+- `stream.sh` gives the container the `NET_ADMIN` capability, but only
+  with `--netem`. It is limited to the container's own network.
+- With rootless Podman, the viewer traffic reaches the relay on the
+  container's loopback interface, so the shaping is set up on `lo` as well
+  as on `eth0`.
+- With a high delay (for example `satellite`), the browser may connect over
+  the WebSocket fallback (TCP) instead of WebTransport: the two race, and
+  the slower QUIC handshake can lose. The header shows which one is in use.
+  Over TCP, frames then arrive in bursts, so the frame rate and the latency
+  jump around, and some 1-second ticks show "—".
+- On Linux, the host kernel needs the `sch_netem`, `sch_prio`, and `cls_u32`
+  modules. Most distributions load them on demand; if `tc` fails, run
+  `sudo modprobe -a sch_netem sch_prio cls_u32`.
+
 ### Limitations
 
 - With `--sgai-mode`, the Ad Decisioning Publisher runs on your host, not

dashboard/server/server.mjs

@@ -8,6 +8,8 @@
 //   GET /api/time     this container's clock, so the browser can correct for host/VM clock skew
 //   GET /api/events   Server-Sent Events: `log` (container stdout/stderr lines) and
 //                     `ingest` (lib/ingest-clock.mjs samples, for end-to-end latency)
+//   GET /api/netem    the active network profile (stream.sh --netem)
+//   POST /api/netem   switch to another profile: {"profile": "<name>"}
 //
 // Usage: node server.mjs --port 8080 --relay-port 4443 --broadcast bbb.hang --mode base
 //                        --abr-ladder false --log-file /tmp/moq-lab.log --ingest-file /tmp/moq-lab-ingest.json
@@ -17,6 +19,7 @@ import { extname, join, normalize, sep } from "node:path";
 import { fileURLToPath } from "node:url";
 import { parseArgs } from "../../lib/cli.mjs";
 import { createLogger } from "../../lib/log.mjs";
+import { applyProfile, currentShaping, PROFILES } from "../../lib/netem.mjs";
 
 const log = createLogger("dashboard");
 
@@ -38,8 +41,17 @@ const CONFIG = {
     // can reach the dashboard is fine for this sandbox: they could already reach the
     // terminal it was printed in.
     jwt: process.env.MOQ_LAB_VIEWER_JWT || null,
+    // Network profiles can only be switched when stream.sh --netem gave the
+    // container NET_ADMIN, so the dashboard only offers them then.
+    netem: process.env.MOQ_LAB_NETEM
+        ? Object.fromEntries(Object.entries(PROFILES).map(([name, p]) => [name, p.label]))
+        : null,
 };
 
+let netemProfile = process.env.MOQ_LAB_NETEM || null;
+// A profile name is a few bytes; refuse anything bigger before parsing it.
+const MAX_BODY_BYTES = 1024;
+
 // How much log history a newly opened tab gets.
 const LOG_BACKLOG = 1000;
 
@@ -131,6 +143,55 @@ function sendJson(res, body) {
     res.end(JSON.stringify(body));
 }
 
+async function sendNetem(res) {
+    sendJson(res, { profile: netemProfile, shaping: await currentShaping() });
+}
+
+function readJsonBody(req) {
+    return new Promise((resolve, reject) => {
+        let body = "";
+        req.setEncoding("utf8");
+        req.on("data", (chunk) => {
+            body += chunk;
+            if (body.length > MAX_BODY_BYTES) {
+                reject(new Error("body too large"));
+                req.destroy();
+            }
+        });
+        req.on("end", () => {
+            try {
+                resolve(JSON.parse(body));
+            } catch {
+                reject(new Error("body is not JSON"));
+            }
+        });
+        req.on("error", reject);
+    });
+}
+
+async function handleNetem(req, res) {
+    if (!CONFIG.netem) {
+        res.writeHead(404).end();
+        return;
+    }
+    if (req.method !== "GET" && req.method !== "POST") {
+        res.writeHead(405).end();
+        return;
+    }
+    if (req.method === "POST") {
+        // A cross-site page can't send this content type without a CORS preflight,
+        // which this server never answers, so only the dashboard itself can switch.
+        if (req.headers["content-type"] !== "application/json") throw new Error("expected application/json");
+        const { profile } = await readJsonBody(req);
+        // applyProfile() only accepts a name from PROFILES; it never builds a command
+        // from the request.
+        await applyProfile(profile);
+        netemProfile = profile;
+        log(`network profile: ${profile}`);
+    }
+    await sendNetem(res);
+}
+
 function serveStatic(req, res, pathname) {
     let decoded;
     try {
@@ -164,6 +225,14 @@ function serveStatic(req, res, pathname) {
 const server = createServer((req, res) => {
     const { pathname } = new URL(req.url, "http://localhost");
 
+    if (pathname === "/api/netem") {
+        handleNetem(req, res).catch((err) => {
+            res.writeHead(400, { "content-type": "text/plain" });
+            res.end(`${err.message}\n`);
+        });
+        return;
+    }
+
     if (req.method !== "GET" && req.method !== "HEAD") {
         res.writeHead(405).end();
         return;

dashboard/src/App.jsx

@@ -2,6 +2,7 @@ import { useCallback, useEffect, useRef, useState } from "react";
 import { Card, Stat } from "./components/Card.jsx";
 import { LineChart } from "./components/LineChart.jsx";
 import { LogStream } from "./components/LogStream.jsx";
+import { Network } from "./components/Network.jsx";
 import { Player } from "./components/Player.jsx";
 import { StatusBadge } from "./components/Status.jsx";
 import { fetchConfig, measureClockOffset, relayUrlFor, subscribeEvents } from "./lib/api.js";
@@ -163,6 +164,8 @@ export default function App() {
 
             {config && <Player config={config} relayUrl={relayUrlFor(config)} />}
 
+            {config?.netem && <Network profiles={config.netem} />}
+
             <Card title="Transport & QUIC" subtitle="Measured on this browser's own MoQ session to the relay">
                 <div className="grid grid-cols-2 gap-3 md:grid-cols-3 xl:grid-cols-5">
                     <Stat

dashboard/src/components/Network.jsx

@@ -0,0 +1,64 @@
+import { useEffect, useState } from "react";
+import { fetchNetem, setNetem } from "../lib/api.js";
+import { Card } from "./Card.jsx";
+import { StatusBadge } from "./Status.jsx";
+
+// Switches stream.sh --netem's network profile while the stream runs. The charts
+// below show what the change does to RTT, loss, jitter, bitrate, and latency.
+export function Network({ profiles }) {
+    const [state, setState] = useState(null);
+    const [busy, setBusy] = useState(false);
+    const [error, setError] = useState(null);
+
+    useEffect(() => {
+        fetchNetem().then(setState, (err) => setError(err.message));
+    }, []);
+
+    const onChange = async (e) => {
+        setBusy(true);
+        setError(null);
+        try {
+            setState(await setNetem(e.target.value));
+        } catch (err) {
+            setError(err.message);
+        } finally {
+            setBusy(false);
+        }
+    };
+
+    const shaped = state?.profile && state.profile !== "off";
+
+    return (
+        <Card
+            title="Network lab"
+            subtitle="Shapes the traffic from the container to its viewers (Linux netem)"
+            aside={
+                <StatusBadge status={shaped ? "warning" : "good"}>{shaped ? "Shaped" : "Not shaped"}</StatusBadge>
+            }
+        >
+            <div className="flex flex-col gap-3">
+                <label className="flex flex-wrap items-center gap-2 text-xs text-ink-3">
+                    Profile
+                    <select
+                        value={state?.profile ?? ""}
+                        onChange={onChange}
+                        disabled={busy || !state}
+                        className="rounded-md border border-line bg-surface-2 px-2 py-1 font-mono text-xs text-ink focus:border-series focus:outline-none disabled:opacity-60"
+                    >
+                        {Object.entries(profiles).map(([name, label]) => (
+                            <option key={name} value={name}>
+                                {label}
+                            </option>
+                        ))}
+                    </select>
+                </label>
+                <p className="font-mono text-xs break-all text-ink-3">{state?.shaping || "tc: no root qdisc"}</p>
+                {error && <p className="font-mono text-xs text-critical">{error}</p>}
+                <p className="text-[11px] text-ink-3">
+                    Only outgoing traffic is shaped, so the delay is one way: the RTT goes up by the delay, not
+                    twice. The ingest pipeline inside the container is not affected.
+                </p>
+            </div>
+        </Card>
+    );
+}

dashboard/src/lib/api.js

@@ -6,6 +6,23 @@ export async function fetchConfig() {
     return res.json();
 }
 
+// The active network profile (stream.sh --netem), and a switch to another one.
+export async function fetchNetem() {
+    const res = await fetch("/api/netem", { cache: "no-store" });
+    if (!res.ok) throw new Error(`/api/netem: HTTP ${res.status}`);
+    return res.json();
+}
+
+export async function setNetem(profile) {
+    const res = await fetch("/api/netem", {
+        method: "POST",
+        headers: { "content-type": "application/json" },
+        body: JSON.stringify({ profile }),
+    });
+    if (!res.ok) throw new Error((await res.text()).trim() || `/api/netem: HTTP ${res.status}`);
+    return res.json();
+}
+
 // The relay is published on the same host the dashboard was opened from.
 // http:// (not https://) is deliberate: @moq/net then fetches the relay's
 // self-signed certificate fingerprint from /certificate.sha256 and pins it for