From 27a4565422fdf02e631cc103a7c5fa2cc947504e Mon Sep 17 00:00:00 2001 From: R0ck Date: Sun, 25 Oct 2026 00:01:37 +0100 Subject: [PATCH 1/4] node.debug: attach a debugger to a running node A native node being debugged is held, not marked down, when a breakpoint stops it answering ticks: the rest of the run goes on and it catches up when let go. An emulated board is restarted with the emulator's GDB stub (QEMU -gdb on loopback, Renode StartGdbServer), optionally held at reset. The answer gives the attach command and a VS Code configuration; the row menu gains Debug this node. For #191. --- CHANGELOG.md | 7 + docs/README.md | 1 + docs/debugging-firmware.md | 71 ++++++++ docs/scripting-api.md | 2 +- docs/scripting-verbs.md | 9 +- docs/verb-reference.md | 29 +++- docs/verbs.json | 49 ++++++ internal/app/session/nodedebug.go | 169 ++++++++++++++++++++ internal/app/session/nodedebug.verbs.json | 51 ++++++ internal/app/session/nodedebug_live_test.go | 77 +++++++++ internal/app/session/verbs.go | 1 + internal/firmware/emulated/emulated_node.go | 8 + internal/firmware/emulated/gdbstub.go | 47 ++++++ internal/firmware/emulated/gdbstub_test.go | 47 ++++++ internal/firmware/emulated/renode.go | 5 +- internal/firmware/lockstep.go | 16 ++ internal/sim/engine/debug.go | 110 +++++++++++++ internal/sim/engine/debug_test.go | 80 +++++++++ internal/sim/engine/engine.go | 6 +- internal/sim/engine/firmware.go | 3 + internal/sim/engine/firmwarerun.go | 11 +- internal/ui/workbench/nodeview/nodeview.go | 3 +- tools/verbdoc/facade.json | 1 + 23 files changed, 790 insertions(+), 13 deletions(-) create mode 100644 docs/debugging-firmware.md create mode 100644 internal/app/session/nodedebug.go create mode 100644 internal/app/session/nodedebug.verbs.json create mode 100644 internal/app/session/nodedebug_live_test.go create mode 100644 internal/firmware/emulated/gdbstub.go create mode 100644 internal/firmware/emulated/gdbstub_test.go create mode 100644 internal/sim/engine/debug.go create mode 100644 internal/sim/engine/debug_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index a38e02b4..bd34c961 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -23,6 +23,13 @@ had changed in them - which is the gap this file exists to close. ### Added +- **Debug a node's firmware while the run goes on.** `node.debug`, and + "Debug this node" on a node's row menu, say how to attach gdb or VS Code to + a running node. A native node gets its process, binary and whether it has + symbols, and a breakpoint now holds it rather than getting it marked down: + the rest of the mesh carries on, and it rejoins when let go. An emulated + board is restarted with QEMU's or Renode's GDB stub, optionally held at + reset so boot can be stepped through. See `docs/debugging-firmware.md`. - **Script input at an instant of simulated time.** `console.type`, `console.cli` and the `companion.*` verbs that write to a node take `at_ms`: the call is held and runs when the run reaches that instant, so a script diff --git a/docs/README.md b/docs/README.md index 2a14cd3f..a8385c86 100644 --- a/docs/README.md +++ b/docs/README.md @@ -50,6 +50,7 @@ lives here and **is** maintained - unlike the working notes below. | `install.md` | per platform, including the macOS and Windows signing caveats | | `firmware-build-settings.md` | what each setting in a build's own window does, and what it costs | | `native-and-emulated.md` | where the real firmware ends and the simulation begins, and the two ways a node can run | +| `debugging-firmware.md` | attaching gdb or an IDE to a running node, native or emulated | ## Machine-specific, and deliberately so diff --git a/docs/debugging-firmware.md b/docs/debugging-firmware.md new file mode 100644 index 00000000..b2ad2447 --- /dev/null +++ b/docs/debugging-firmware.md @@ -0,0 +1,71 @@ +# Debugging a node's firmware + +`node.debug` makes a running node debuggable and says how to attach. The +firmware is real, so the debugger is the ordinary one: gdb, lldb, or VS Code +with the C/C++ extension. + +## Native nodes + +A native node is an ordinary process. Ask for it: + +``` +node.debug {"node": "West Lomond"} +``` + +The answer has the process ID, the binary, whether the binary has symbols, a +gdb command line and a VS Code `attach` configuration: + +``` +gdb -p 2104750 ~/.cache/meshbench/firmware/native/.../meshcore-simple_repeater-linux-amd64 +``` + +A native node normally has to answer every tick within 30 seconds or it is +marked down. Once `node.debug` has been called it is held instead. A +breakpoint stops it, the rest of the mesh carries on, and when the debugger +lets it go it catches up to the run's clock and rejoins. Its clock jumps by +however long it was stopped, so `sim.state` reports the run as not +reproducible from then on. + +- `symbols` is `debug` when the binary carries DWARF, `symbols` when it has a + symbol table only, and `stripped` when it has neither. For source-level + debugging, build MeshCore from a checkout with `meshbench dev`. +- On Linux, attaching to a process that is not the debugger's own child needs + ptrace permission: `kernel.yama.ptrace_scope` set to 0, or the debugger run + as root. +- A native node runs `setup()` as soon as it starts, so a fault during boot + cannot be caught by attaching afterwards. + +`node.debug {"node": ..., "off": true}` puts the deadline back. + +## Emulated boards + +An emulated board's firmware runs inside QEMU or Renode, and the way in is the +emulator's GDB stub. `node.debug` restarts the board with one: + +``` +node.debug {"node": "Deck", "wait": true} +``` + +- `gdb_port` is where the stub listens. Pass `port` to choose it. +- `wait` holds the CPU at reset until a debugger continues, so boot can be + stepped through. +- `debugger` is the gdb that speaks the board's architecture: + `xtensa-esp32-elf-gdb` or `xtensa-esp32s3-elf-gdb` for the ESP32 boards + under QEMU, `arm-none-eabi-gdb` (or `gdb-multiarch`) for the nRF52 boards + under Renode. + +The image a board runs is a flashable `.bin` or `.uf2` with no symbols. Point +the debugger at the `firmware.elf` the same build produced, for a PlatformIO +build `.pio/build//firmware.elf`: + +``` +arm-none-eabi-gdb -ex 'target remote 127.0.0.1:3333' .pio/build/RAK_4631_repeater/firmware.elf +``` + +The answer includes a VS Code configuration for this, with +`miDebuggerServerAddress` set to the stub. + +QEMU's stub listens on loopback only. Renode's listens on every interface, so +only ask for one on a machine you trust the network around. + +`node.debug {"node": ..., "off": true}` restarts the board without the stub. diff --git a/docs/scripting-api.md b/docs/scripting-api.md index df0e4f90..67a5dd77 100644 --- a/docs/scripting-api.md +++ b/docs/scripting-api.md @@ -39,7 +39,7 @@ and a verb added tomorrow is usable today. ### Why two layers rather than one -**260** addressable methods hand-written across two +**261** addressable methods hand-written across two languages is twice that many things to keep in step. Keeping them in step is what the verb manifest (`docs/verbs.json`) is for: a hand-written surface that names a verb the tree no longer has is exactly the drift it catches, where once a tool could call a deleted verb and nothing diff --git a/docs/scripting-verbs.md b/docs/scripting-verbs.md index 29e6554d..54767131 100644 --- a/docs/scripting-verbs.md +++ b/docs/scripting-verbs.md @@ -1,8 +1,8 @@ # Every verb, and the call that covers it The wire underneath [scripting-api.md](scripting-api.md). The store registers -**222** public verbs a script can call and -**38** internal callbacks it cannot, **260** +**223** public verbs a script can call and +**38** internal callbacks it cannot, **261** in total, plus the two the socket answers itself, with what each reads, what each returns, and which façade call reaches it. @@ -77,8 +77,8 @@ Two verbs are **not** in this table: ## What this table shows about the surface -- **222** public verbs, plus **38** - the workbench keeps to itself, **260** registered in +- **223** public verbs, plus **38** + the workbench keeps to itself, **261** registered in total. The façade covers the public verbs, over roughly 60 calls once objects and properties absorb them. - **The naming is not regular.** `node.*` and `nodes.*` are both node verbs and @@ -131,6 +131,7 @@ Two verbs are **not** in this table: | `node.antenna` | *a bare string*, `node` string | `node`, `pattern`, `gain_dbi_peak`, `beamwidth_deg`, `front_to_back_db`, `bearing_deg`, `downtilt_deg`, `polarisation`, `feedline_db`, `peak_dbi` | `node.antenna` | | `node.boardview` 🪟 | *a bare string*, `node` string, `tab` string | `node`, `board`, `tab` | `wb.board_view(node)` | | `node.card` | *a bare string*, `node` string, `fitted` bool, `file` string, `wipe` bool | `node`, `slot`, `fitted`, `file`, `own_file`, `bytes`, `required_by_firmware`, `board_has_slot`, `wiped` | `node.card(fitted=\|file=\|wipe=)` | +| `node.debug` | *a bare string*, `node` string, `wait` bool, `port` number, `off` bool | `node`, `debugging`, `backend`, `pid`, `binary`, `symbols`, `gdb_port`, `wait`, `debugger`, `image`, `attach`, `vscode`, `note` | `node.debug(wait=, port=, off=)` | | `node.energy` | *a bare string*, `node` string | `node` | `node.energy()` | | `node.output` | *a bare string*, `node` string, `source` string, `lines` number | `node`, `source`, `lines`, `total`, `path`, `tail`, `note`, `tracing` | `node.output(source)` | | `node.output_window` 🪟 | *a bare string*, `node` string, `source` string | `node`, `source` | `node.output_window(source)` | diff --git a/docs/verb-reference.md b/docs/verb-reference.md index 36e84e35..e06e1d25 100644 --- a/docs/verb-reference.md +++ b/docs/verb-reference.md @@ -3,9 +3,9 @@ Generated. Run `tools/verbdoc/verbdoc.py` to rewrite it and `tools/verbdoc/verbdoc.py --check` to fail when it is stale. -The store registers 260 verbs: 222 a script may call and +The store registers 261 verbs: 223 a script may call and 38 the workbench calls on itself, which the socket refuses. Of those, -260 say what they are for and 0 do not yet; the ones that +261 say what they are for and 0 do not yet; the ones that do not are marked, and what is printed for them is read out of the handler rather than said by it. @@ -490,6 +490,31 @@ Report or change what is in one node's card slot: whether a card is fitted, whic **Client** `node.card(fitted=|file=|wipe=)` +### `node.debug` + +Make a running node debuggable and say how to attach: a native node is held rather than marked down while a breakpoint stops it, and an emulated board is restarted with its emulator's GDB stub. + +**Takes** + +| parameter | type | | what | +|---|---|---|---| +| `node` | string | required, primary | the node to debug; refused when absent, unknown, or not running firmware | +| `wait` | bool | optional | for an emulated board, hold its CPU at reset until a debugger continues, so boot can be stepped through; refused for a native node, which has already booted by the time anything can attach | +| `port` | number | optional | for an emulated board, the port its GDB stub listens on; absent picks a free one | +| `off` | bool | optional | stop debugging: a native node is given its tick deadline back, an emulated board is restarted without the stub | + +**Answers** `node`, `debugging`, `backend`, `pid`, `binary`, `symbols`, `gdb_port`, `wait`, `debugger`, `image`, `attach`, `vscode`, `note`. `attach` is a debugger command line and `vscode` a launch configuration to paste. A native node has `pid`, `binary` and `symbols`: `debug` with DWARF, `symbols` with a symbol table only, `stripped` with neither. An emulated board has `gdb_port`, `wait`, the `debugger` for its architecture and the `image` it runs, which has no symbols: the debugger wants the firmware.elf the same build produced. From the first call, `sim.state` reports the run as not reproducible, because a debugger stopping a node moves its clock. + +**Example** - attach gdb to a native repeater + +```json +{"id":1,"method":"node.debug","params":{"node":"West Lomond"}} +``` + +Not made by the test suite: this call needs more than the two-node headless session the runnable examples go to. + +**Client** `node.debug(wait=, port=, off=)` + ### `node.energy` Run a year of sun and battery at one named node, at the duty cycle the run measured for it rather than one typed into a form, for the node window that has a name and no selection to work from. diff --git a/docs/verbs.json b/docs/verbs.json index bc16e7ab..4248d2e0 100644 --- a/docs/verbs.json +++ b/docs/verbs.json @@ -2953,6 +2953,55 @@ "runnable": true } }, + "node.debug": { + "what": "make a running node debuggable and say how to attach: a native node is held rather than marked down while a breakpoint stops it, and an emulated board is restarted with its emulator's GDB stub", + "params": [ + { + "name": "node", + "type": "string", + "required": true, + "primary": true, + "what": "the node to debug; refused when absent, unknown, or not running firmware" + }, + { + "name": "wait", + "type": "bool", + "what": "for an emulated board, hold its CPU at reset until a debugger continues, so boot can be stepped through; refused for a native node, which has already booted by the time anything can attach" + }, + { + "name": "port", + "type": "number", + "what": "for an emulated board, the port its GDB stub listens on; absent picks a free one" + }, + { + "name": "off", + "type": "bool", + "what": "stop debugging: a native node is given its tick deadline back, an emulated board is restarted without the stub" + } + ], + "returns": [ + "node", + "debugging", + "backend", + "pid", + "binary", + "symbols", + "gdb_port", + "wait", + "debugger", + "image", + "attach", + "vscode", + "note" + ], + "answers": "`attach` is a debugger command line and `vscode` a launch configuration to paste. A native node has `pid`, `binary` and `symbols`: `debug` with DWARF, `symbols` with a symbol table only, `stripped` with neither. An emulated board has `gdb_port`, `wait`, the `debugger` for its architecture and the `image` it runs, which has no symbols: the debugger wants the firmware.elf the same build produced. From the first call, `sim.state` reports the run as not reproducible, because a debugger stopping a node moves its clock.", + "example": { + "params": { + "node": "West Lomond" + }, + "what": "attach gdb to a native repeater" + } + }, "node.energy": { "what": "run a year of sun and battery at one named node, at the duty cycle the run measured for it rather than one typed into a form, for the node window that has a name and no selection to work from", "params": [ diff --git a/internal/app/session/nodedebug.go b/internal/app/session/nodedebug.go new file mode 100644 index 00000000..caa2d51d --- /dev/null +++ b/internal/app/session/nodedebug.go @@ -0,0 +1,169 @@ +// Attaching a debugger to a node's firmware (#191). +// +// A native node is an ordinary process, so gdb, lldb and an IDE already work +// on it; what was missing was which process, whether its binary has symbols, +// and a run that does not mark the node down the moment a breakpoint stops it. +// An emulated board's firmware is inside the emulator, and the way in is the +// emulator's GDB stub, which only exists if the board was started with one. +package session + +import ( + "context" + "debug/elf" + "fmt" + "net" + + "github.com/MeshBench/meshbench/internal/app/state" + "github.com/MeshBench/meshbench/internal/firmware/emulated" + "github.com/MeshBench/meshbench/internal/firmware/native" + "github.com/MeshBench/meshbench/internal/sim/engine" +) + +func registerNodeDebug(st *state.Store, s *Sim) { + st.Handle("node.debug", func(w *state.World, p any) (any, error) { + name := primaryString(p, "node") + if s.eng == nil { + return nil, ErrNoSimulation + } + n, ok := s.eng.NodeByName(name) + if !ok { + return nil, noSuchNode(name) + } + if n.Firmware == nil { + return nil, fmt.Errorf("%s is not running firmware, so there is nothing to debug", name) + } + off, _ := boolField(p, "off") + wait, _ := boolField(p, "wait") + port := 0 + if v, ok := namedNum(p, "port"); ok { + port = int(v) + } + switch b := n.Firmware.Backend.(type) { + case *native.Native: + if off { + s.eng.SetDebug(name, nil) + return map[string]any{"node": name, "debugging": false}, nil + } + if wait { + return nil, fmt.Errorf("%s is native: wait holds an emulated board at reset, and a "+ + "native node has already run setup() by the time it can be attached to", name) + } + s.eng.SetDebug(name, &engine.Debug{}) + s.taint(name + " is being debugged, and a debugger stopping it moves its clock") + out := nativeDebug(name, b) + w.Say(fmt.Sprintf("%s can be debugged, and a breakpoint holds it while the run goes on: %s", + name, out["attach"])) + return out, nil + case *emulated.EmulatedNode: + if off { + s.eng.SetDebug(name, nil) + if err := s.restartNode(name, w.Seed); err != nil { + return nil, err + } + return map[string]any{"node": name, "debugging": false}, nil + } + if port == 0 { + free, err := freePort() + if err != nil { + return nil, err + } + port = free + } + s.eng.SetDebug(name, &engine.Debug{GDBPort: port, Wait: wait}) + s.taint(name + " is being debugged, and a debugger stopping it moves its clock") + // The stub is part of how the emulator is started, so the board + // starts again with it. + if err := s.restartNode(name, w.Seed); err != nil { + s.eng.SetDebug(name, nil) + return nil, err + } + n, _ = s.eng.NodeByName(name) + em, _ := n.Firmware.Backend.(*emulated.EmulatedNode) + if em == nil { + return nil, fmt.Errorf("%s did not come back as an emulated board", name) + } + out := emulatedDebug(name, em) + w.Say(fmt.Sprintf("%s restarted with a GDB stub on port %d: %s", name, port, out["attach"])) + return out, nil + default: + return nil, fmt.Errorf("%s runs on a backend nothing can attach a debugger to", name) + } + }) +} + +func (s *Sim) restartNode(name string, seed uint64) error { + if err := s.stopNode(name); err != nil { + return err + } + return s.startNode(context.Background(), name, seed) +} + +// nativeDebug is how to attach to a native node: its process, its binary and +// whether that binary can be read at source level. +func nativeDebug(name string, b *native.Native) map[string]any { + pid := b.PID() + syms := symbols(b.Path) + out := map[string]any{ + "node": name, "debugging": true, "backend": "native", + "pid": pid, "binary": b.Path, "symbols": syms, + "attach": fmt.Sprintf("gdb -p %d %s", pid, b.Path), + "vscode": map[string]any{ + "name": "MeshBench " + name, "type": "cppdbg", "request": "attach", + "program": b.Path, "processId": fmt.Sprint(pid), "MIMode": "gdb", + }, + "note": "a breakpoint holds this node and the rest of the run goes on; when it is let go it " + + "catches up to the run's clock. On Linux, attaching needs ptrace permission: " + + "kernel.yama.ptrace_scope 0, or the debugger run as root", + } + if syms == "stripped" { + out["note"] = out["note"].(string) + ". This binary has no symbols: build the firmware " + + "with meshbench dev from a checkout to debug it at source level" + } + return out +} + +// emulatedDebug is how to reach an emulated board's GDB stub. +func emulatedDebug(name string, em *emulated.EmulatedNode) map[string]any { + port, gdb := em.GDB() + target := fmt.Sprintf("127.0.0.1:%d", port) + return map[string]any{ + "node": name, "debugging": true, "backend": "emulated", + "gdb_port": port, "wait": em.GDBWait, "debugger": gdb, "image": em.Image, + "attach": fmt.Sprintf("%s -ex 'target remote %s' firmware.elf", gdb, target), + "vscode": map[string]any{ + "name": "MeshBench " + name, "type": "cppdbg", "request": "launch", + "program": "${workspaceFolder}/.pio/build//firmware.elf", + "MIMode": "gdb", "miDebuggerPath": gdb, "miDebuggerServerAddress": target, + "cwd": "${workspaceFolder}", + }, + "note": "the image the board runs has no symbols: point the debugger at the firmware.elf " + + "the same build produced. With wait, the CPU is held at reset until the debugger continues", + } +} + +// symbols says whether a binary can be debugged at source level: "debug" with +// DWARF, "symbols" with a symbol table only, "stripped" with neither, or +// "unknown" when it cannot be read as ELF (a Mach-O or PE build). +func symbols(path string) string { + f, err := elf.Open(path) + if err != nil { + return "unknown" + } + defer func() { _ = f.Close() }() + if f.Section(".debug_info") != nil { + return "debug" + } + if f.Section(".symtab") != nil { + return "symbols" + } + return "stripped" +} + +func freePort() (int, error) { + l, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + return 0, err + } + defer func() { _ = l.Close() }() + return l.Addr().(*net.TCPAddr).Port, nil +} diff --git a/internal/app/session/nodedebug.verbs.json b/internal/app/session/nodedebug.verbs.json new file mode 100644 index 00000000..44a2dddd --- /dev/null +++ b/internal/app/session/nodedebug.verbs.json @@ -0,0 +1,51 @@ +{ + "node.debug": { + "what": "make a running node debuggable and say how to attach: a native node is held rather than marked down while a breakpoint stops it, and an emulated board is restarted with its emulator's GDB stub", + "params": [ + { + "name": "node", + "type": "string", + "required": true, + "primary": true, + "what": "the node to debug; refused when absent, unknown, or not running firmware" + }, + { + "name": "wait", + "type": "bool", + "what": "for an emulated board, hold its CPU at reset until a debugger continues, so boot can be stepped through; refused for a native node, which has already booted by the time anything can attach" + }, + { + "name": "port", + "type": "number", + "what": "for an emulated board, the port its GDB stub listens on; absent picks a free one" + }, + { + "name": "off", + "type": "bool", + "what": "stop debugging: a native node is given its tick deadline back, an emulated board is restarted without the stub" + } + ], + "returns": [ + "node", + "debugging", + "backend", + "pid", + "binary", + "symbols", + "gdb_port", + "wait", + "debugger", + "image", + "attach", + "vscode", + "note" + ], + "answers": "`attach` is a debugger command line and `vscode` a launch configuration to paste. A native node has `pid`, `binary` and `symbols`: `debug` with DWARF, `symbols` with a symbol table only, `stripped` with neither. An emulated board has `gdb_port`, `wait`, the `debugger` for its architecture and the `image` it runs, which has no symbols: the debugger wants the firmware.elf the same build produced. From the first call, `sim.state` reports the run as not reproducible, because a debugger stopping a node moves its clock.", + "example": { + "params": { + "node": "West Lomond" + }, + "what": "attach gdb to a native repeater" + } + } +} diff --git a/internal/app/session/nodedebug_live_test.go b/internal/app/session/nodedebug_live_test.go new file mode 100644 index 00000000..9c2ac43b --- /dev/null +++ b/internal/app/session/nodedebug_live_test.go @@ -0,0 +1,77 @@ +package session + +import ( + "context" + "os" + "syscall" + "testing" + "time" + + "github.com/MeshBench/meshbench/internal/app/state" + "github.com/MeshBench/meshbench/internal/world/scenario" +) + +// A native node being debugged can be stopped, as a breakpoint stops it, and +// the run goes on without it; let go, it rejoins rather than being marked +// down (#191). +func TestLiveADebuggedNativeNodeSurvivesABreakpoint(t *testing.T) { + if os.Getenv("MESHBENCH_LIVE") == "" { + t.Skip("set MESHBENCH_LIVE=1") + } + ctx, cancel := context.WithTimeout(context.Background(), 120*time.Second) + defer cancel() + st := state.New(10) + s := &Sim{gpuAsked: true} + Register(st, s) + go st.Run(ctx) + a, b := repeaterNode("A"), repeaterNode("B") + b.Position.Lon += 0.001 + s.build([]scenario.Node{a, b}, 869.618) + if err := s.eng.AttachNative(ctx, 4417); err != nil { + t.Fatal(err) + } + defer s.Close() + + got, err := st.Do(ctx, "node.debug", map[string]any{"node": "A"}) + if err != nil { + t.Fatal(err) + } + m := got.(map[string]any) + pid, _ := m["pid"].(int) + if pid <= 0 || m["binary"] == "" { + t.Fatalf("node.debug answered %v", m) + } + t.Logf("attach: %v (symbols: %v)", m["attach"], m["symbols"]) + + // What a breakpoint does to the process. + if err := syscall.Kill(pid, syscall.SIGSTOP); err != nil { + t.Fatal(err) + } + before := s.eng.NowMs() + if err := s.eng.Run(ctx, before+5000); err != nil { + t.Fatal(err) + } + if _, held := s.eng.Held("A"); !held { + t.Fatal("a stopped node being debugged was not held") + } + if s.eng.NowMs() < before+5000 { + t.Fatalf("the run stopped with the node: %d", s.eng.NowMs()) + } + + if err := syscall.Kill(pid, syscall.SIGCONT); err != nil { + t.Fatal(err) + } + time.Sleep(200 * time.Millisecond) + if err := s.eng.Run(ctx, s.eng.NowMs()+2000); err != nil { + t.Fatal(err) + } + if _, held := s.eng.Held("A"); held { + t.Error("let go, the node is still held") + } + if down := s.eng.FirmwareFailures(); len(down) != 0 { + t.Errorf("marked down: %+v", down) + } + if s.notReproducible() == "" { + t.Error("a run with a node being debugged still claims it can be repeated") + } +} diff --git a/internal/app/session/verbs.go b/internal/app/session/verbs.go index 34c262b5..72c66086 100644 --- a/internal/app/session/verbs.go +++ b/internal/app/session/verbs.go @@ -47,6 +47,7 @@ func Register(st *state.Store, s *Sim) { registerPresetsAndPlace(st, s) registerCompanion(st, s) registerMeshCLI(st, s) + registerNodeDebug(st, s) registerRFMode(st, s) registerRFRealism(st, s) registerRFEnvironment(st, s) diff --git a/internal/firmware/emulated/emulated_node.go b/internal/firmware/emulated/emulated_node.go index 6df13f0e..c099e8b0 100644 --- a/internal/firmware/emulated/emulated_node.go +++ b/internal/firmware/emulated/emulated_node.go @@ -175,6 +175,13 @@ type EmulatedNode struct { // Emulator selects QEMU or Renode. Emulator Emulator + // GDBPort, when set, starts the emulator's GDB stub on that loopback + // port: source-level debugging of the real firmware binary, the same as a + // probe on a board. GDBWait holds the CPU at reset until a debugger lets + // it go, so a fault during boot can be caught. + GDBPort int + GDBWait bool + // Renode wiring. Platform is its base description; the rest is where the // radio hangs off and which pins carry chip select and DIO1. Chip select // matters more than it looks: Renode's SPI model never ends a transmission, @@ -397,6 +404,7 @@ func (e *EmulatedNode) Start(ctx context.Context, bridge string) (err error) { args = append(args, "-m", fmt.Sprintf("%dM", e.PSRAMMB)) } args = append(args, qemuDebugArgs(e.Dir)...) + args = append(args, e.qemuGDBArgs()...) e.qemu = exec.CommandContext(ctx, qemuBin, args...) e.qemu.Env = e.radioEnv e.qemu.Stdout, e.qemu.Stderr = emuLog, emuLog diff --git a/internal/firmware/emulated/gdbstub.go b/internal/firmware/emulated/gdbstub.go new file mode 100644 index 00000000..bb3bac2a --- /dev/null +++ b/internal/firmware/emulated/gdbstub.go @@ -0,0 +1,47 @@ +package emulated + +import "fmt" + +// qemuGDBArgs starts QEMU's GDB stub, on loopback only: a stub is the whole +// machine, memory and registers, to anything that reaches the port. +func (e *EmulatedNode) qemuGDBArgs() []string { + if e.GDBPort <= 0 { + return nil + } + args := []string{"-gdb", fmt.Sprintf("tcp:127.0.0.1:%d", e.GDBPort)} + if e.GDBWait { + // The CPU does not start until a debugger says continue. + args = append(args, "-S") + } + return args +} + +// renodeStart is the end of the node's script: the GDB stub when one is +// asked for, and starting the machine. Renode's stub listens on every +// interface, which is why only a run on this machine asks for it. Held at +// reset, the machine is not started here at all: the stub's autostart starts +// it when a debugger connects and continues. +func (e *EmulatedNode) renodeStart() string { + if e.GDBPort <= 0 { + return "start\n" + } + if e.GDBWait { + return fmt.Sprintf("machine StartGdbServer %d true\n", e.GDBPort) + } + return fmt.Sprintf("machine StartGdbServer %d\nstart\n", e.GDBPort) +} + +// GDB is how a debugger reaches this node: the port, and the debugger that +// speaks the board's architecture. Zero when no stub was asked for. +func (e *EmulatedNode) GDB() (port int, debugger string) { + if e.GDBPort <= 0 { + return 0, "" + } + if e.Emulator == Renode { + return e.GDBPort, "arm-none-eabi-gdb" + } + if e.Machine == "esp32s3" { + return e.GDBPort, "xtensa-esp32s3-elf-gdb" + } + return e.GDBPort, "xtensa-esp32-elf-gdb" +} diff --git a/internal/firmware/emulated/gdbstub_test.go b/internal/firmware/emulated/gdbstub_test.go new file mode 100644 index 00000000..e12a4223 --- /dev/null +++ b/internal/firmware/emulated/gdbstub_test.go @@ -0,0 +1,47 @@ +package emulated + +import ( + "reflect" + "strings" + "testing" +) + +// The GDB stub is asked for only when a port is given, on loopback for QEMU, +// and holding the CPU at reset only when asked to (#191). +func TestTheGDBStubIsAskedForOnlyWhenWanted(t *testing.T) { + if got := (&EmulatedNode{}).qemuGDBArgs(); got != nil { + t.Errorf("no port, yet %v", got) + } + if got := (&EmulatedNode{}).renodeStart(); got != "start\n" { + t.Errorf("no port, yet %q", got) + } + run := &EmulatedNode{GDBPort: 3333} + if got := run.qemuGDBArgs(); !reflect.DeepEqual(got, []string{"-gdb", "tcp:127.0.0.1:3333"}) { + t.Errorf("qemu: %v", got) + } + if got := run.renodeStart(); got != "machine StartGdbServer 3333\nstart\n" { + t.Errorf("renode: %q", got) + } + held := &EmulatedNode{GDBPort: 3333, GDBWait: true} + if got := held.qemuGDBArgs(); got[len(got)-1] != "-S" { + t.Errorf("qemu held at reset: %v", got) + } + if got := held.renodeStart(); strings.Contains(got, "start\n") && !strings.Contains(got, "true") { + t.Errorf("renode held at reset starts anyway: %q", got) + } +} + +func TestTheDebuggerFollowsTheArchitecture(t *testing.T) { + for _, c := range []struct { + n *EmulatedNode + want string + }{ + {&EmulatedNode{GDBPort: 1, Emulator: Renode}, "arm-none-eabi-gdb"}, + {&EmulatedNode{GDBPort: 1, Machine: "esp32s3"}, "xtensa-esp32s3-elf-gdb"}, + {&EmulatedNode{GDBPort: 1, Machine: "esp32"}, "xtensa-esp32-elf-gdb"}, + } { + if _, got := c.n.GDB(); got != c.want { + t.Errorf("%s/%s: %s, want %s", c.n.Machine, c.want, got, c.want) + } + } +} diff --git a/internal/firmware/emulated/renode.go b/internal/firmware/emulated/renode.go index 87017859..00c24a72 100644 --- a/internal/firmware/emulated/renode.go +++ b/internal/firmware/emulated/renode.go @@ -108,10 +108,9 @@ machine LoadPlatformDescription @%[1]s/usbd.repl %[5]s %[9]sradiospi.lora Connect -%[8]s%[7]sstart -`, SupportDir(), firmware.SafeNodeName(e.NodeName), e.Platform, repl, flash, +%[8]s%[7]s%[10]s`, SupportDir(), firmware.SafeNodeName(e.NodeName), e.Platform, repl, flash, renode.UnregisterStockSPI(), renode.RenodeTrace(), - renode.ConsoleTerminal(conPort, e.ConsoleOnUSB), idle) + renode.ConsoleTerminal(conPort, e.ConsoleOnUSB), idle, e.renodeStart()) if err := os.WriteFile(script, []byte(body), 0o644); err != nil { return "", err } diff --git a/internal/firmware/lockstep.go b/internal/firmware/lockstep.go index 244b2457..76d9de70 100644 --- a/internal/firmware/lockstep.go +++ b/internal/firmware/lockstep.go @@ -119,3 +119,19 @@ func (b *Bridge) SetChannelBusy(busy bool) error { } return nil } + +// CaughtUp reports, without waiting, whether the node has acknowledged the tick +// to atMs: for a node a debugger is holding, which the engine asks each tick +// rather than waits on. +func (b *Bridge) CaughtUp(atMs uint32) bool { + for { + select { + case got := <-b.acked: + if got >= atMs { + return true + } + default: + return false + } + } +} diff --git a/internal/sim/engine/debug.go b/internal/sim/engine/debug.go new file mode 100644 index 00000000..5e56dcc4 --- /dev/null +++ b/internal/sim/engine/debug.go @@ -0,0 +1,110 @@ +package engine + +import ( + "context" + "errors" + "time" +) + +// Debugging a node's firmware while the run goes on. +// +// A native node is a process in lockstep with the engine: every tick waits for +// its acknowledgement. Stopped at a breakpoint it sends none, and the tick's +// deadline would mark it down - so a debugger could never be used on a node +// for longer than the deadline. A node being debugged is held instead: the +// tick it did not answer is remembered, the rest of the mesh carries on +// without it, and when the debugger lets it go it acknowledges, catches up to +// the current instant on its next tick and rejoins. Its clock jumps forward by +// however long it was held, which is what a real board stopped by a JTAG probe +// does too, and the run is no longer one a seed can repeat. +// +// An emulated board's CPU also gets the emulator's own GDB stub, which is the +// only way into it: the emulator is the process, and the firmware is inside. + +// Debug is how a node is to be run for a debugger. +type Debug struct { + // GDBPort is where an emulated board's GDB stub listens, on loopback. + // Zero for a native node, which a debugger attaches to by its PID. + GDBPort int + // Wait holds an emulated board's CPU at reset until a debugger lets it + // go, so a fault during boot can be caught. + Wait bool +} + +// debugGrace is how long a node being debugged gets to answer a tick before +// it is taken to be stopped in a debugger and held. Long enough that a node +// merely slow on a loaded machine is not held for nothing, which would move +// its clock; short enough that the window stays usable while it is stopped. +var debugGrace = 2 * time.Second + +// SetDebug marks a node as being debugged, or with nil as not. It takes +// effect for an emulated board when its firmware next starts, and for a +// native node at once. +func (e *Engine) SetDebug(name string, d *Debug) { + e.mu.Lock() + defer e.mu.Unlock() + if d == nil { + delete(e.debug, name) + delete(e.held, name) + return + } + if e.debug == nil { + e.debug = map[string]Debug{} + } + e.debug[name] = *d +} + +// Debugging reports whether a node is being debugged, and how. +func (e *Engine) Debugging(name string) (Debug, bool) { + e.mu.Lock() + defer e.mu.Unlock() + d, ok := e.debug[name] + return d, ok +} + +// Held reports whether a debugger is holding a node, and since which tick. +func (e *Engine) Held(name string) (uint32, bool) { + e.mu.Lock() + defer e.mu.Unlock() + at, ok := e.held[name] + return at, ok +} + +// stillHeld is true when a node being debugged has not yet answered the tick +// it was held at, and so is skipped this tick. Released, it rejoins. +func (e *Engine) stillHeld(t ticked) bool { + e.mu.Lock() + at, held := e.held[t.name] + e.mu.Unlock() + if !held { + return false + } + if !t.fw.Bridge.CaughtUp(at) { + return true + } + e.mu.Lock() + delete(e.held, t.name) + e.mu.Unlock() + return false +} + +// waitDebugged waits for a node being debugged, and holds it rather than +// failing it when it does not answer in time. Anything but a timeout - the +// process has gone - is still the failure it always was. +func (e *Engine) waitDebugged(ctx context.Context, t ticked) error { + wctx, cancel := context.WithTimeout(ctx, debugGrace) + defer cancel() + err := t.fw.Bridge.WaitAdvance(wctx, t.atMs) + if err == nil || !errors.Is(err, context.DeadlineExceeded) || ctx.Err() != nil { + return err + } + e.mu.Lock() + if e.held == nil { + e.held = map[string]uint32{} + } + e.held[t.name] = t.atMs + e.mu.Unlock() + return errHeld +} + +var errHeld = errors.New("held by a debugger") diff --git a/internal/sim/engine/debug_test.go b/internal/sim/engine/debug_test.go new file mode 100644 index 00000000..c89f8704 --- /dev/null +++ b/internal/sim/engine/debug_test.go @@ -0,0 +1,80 @@ +package engine + +import ( + "context" + "testing" + "time" +) + +// A node stopped in a debugger is held, not failed, and the rest of the mesh +// goes on; when the debugger lets it go it answers, and the next tick takes it +// to the run's current instant (#191). +func TestANodeInADebuggerIsHeldAndRejoins(t *testing.T) { + old := debugGrace + debugGrace = 200 * time.Millisecond + t.Cleanup(func() { debugGrace = old }) + + e := New(flatGround{}, Config{StepMs: 10, FirmwareTickTimeout: 30 * time.Second}) + defer func() { _ = e.Close() }() + stopped := attachedNode(t, e, "stopped") + talker := attachedNode(t, e, "talker") + e.SetDebug("stopped", &Debug{}) + + step := func() { + t.Helper() + done := make(chan error, 1) + go func() { done <- e.Step(context.Background()) }() + talker.ack(t, talker.nextTick(t)) + select { + case err := <-done: + if err != nil { + t.Fatalf("Step: %v", err) + } + case <-time.After(5 * time.Second): + t.Fatal("a node in a debugger stopped the run") + } + } + + // The first tick reaches it and goes unanswered: held, not down. + step() + heldAt := stopped.nextTick(t) + if down := e.FirmwareFailures(); len(down) != 0 { + t.Fatalf("a held node was marked down: %+v", down) + } + if at, ok := e.Held("stopped"); !ok || at != heldAt { + t.Fatalf("Held = %d, %v; want %d, true", at, ok, heldAt) + } + + // While held it is not sent another tick. + step() + select { + case at := <-stopped.ticks: + t.Fatalf("a held node was ticked again, to %d", at) + case <-time.After(100 * time.Millisecond): + } + + // Let go: it answers the tick it was held at, and rejoins at the run's + // current instant. + stopped.ack(t, heldAt) + time.Sleep(50 * time.Millisecond) + done := make(chan error, 1) + go func() { done <- e.Step(context.Background()) }() + at := stopped.nextTick(t) + talker.ack(t, talker.nextTick(t)) + stopped.ack(t, at) + if err := <-done; err != nil { + t.Fatal(err) + } + if at <= heldAt { + t.Errorf("rejoined at %d, not past the %d it was held at", at, heldAt) + } + if _, ok := e.Held("stopped"); ok { + t.Error("still held after answering") + } + + // Not debugged, the same silence is a failure, as it always was. + e.SetDebug("stopped", nil) + if _, ok := e.Debugging("stopped"); ok { + t.Error("SetDebug(nil) left it debugged") + } +} diff --git a/internal/sim/engine/engine.go b/internal/sim/engine/engine.go index 1d7db559..39d5343d 100644 --- a/internal/sim/engine/engine.go +++ b/internal/sim/engine/engine.go @@ -192,7 +192,11 @@ type Engine struct { // crashed process, noticed once and then skipped rather than retried // every tick. firmwareNewlyDown is the subset a caller has not yet been // told about; FirmwareFailures drains it. - firmwareDown map[string]bool + firmwareDown map[string]bool + // debug is which nodes are being debugged, and held which of them a + // debugger is holding, at the tick they did not answer. See debug.go. + debug map[string]Debug + held map[string]uint32 firmwareNewlyDown []FirmwareFailure // classCounts is events by class, kept as they are recorded so the cards // that show them never walk the log. diff --git a/internal/sim/engine/firmware.go b/internal/sim/engine/firmware.go index a57206db..d0710c11 100644 --- a/internal/sim/engine/firmware.go +++ b/internal/sim/engine/firmware.go @@ -180,6 +180,9 @@ func (e *Engine) AttachNativeProgress(ctx context.Context, seed uint64, progress fail(fmt.Errorf("%s: %w", n.specRef().Name, err)) return } + if d, ok := e.Debugging(n.specRef().Name); ok { + em.GDBPort, em.GDBWait = d.GDBPort, d.Wait + } backend = em } else { key := string(role) + "@" + n.specRef().Firmware.Version diff --git a/internal/sim/engine/firmwarerun.go b/internal/sim/engine/firmwarerun.go index e4ae3d2c..fd5d291c 100644 --- a/internal/sim/engine/firmwarerun.go +++ b/internal/sim/engine/firmwarerun.go @@ -7,6 +7,7 @@ package engine import ( "context" + "errors" "time" "github.com/MeshBench/meshbench/internal/firmware" @@ -65,6 +66,9 @@ func (e *Engine) runFirmware(ctx context.Context, now uint32) error { busy := e.channelBusy(now) sent := make([]ticked, 0, len(targets)) for _, t := range targets { + if e.stillHeld(t) { + continue + } // What the channel sounds like here, before the node decides whether to // talk. MeshCore asks its radio this in Dispatcher::checkSend, and the // answer has to arrive before the tick it applies to - a node told after @@ -81,7 +85,9 @@ func (e *Engine) runFirmware(ctx context.Context, now uint32) error { } for _, t := range sent { if err := e.waitTicked(ctx, t); err != nil { - e.markFirmwareDown(t.name, t.fw, err.Error()) + if !errors.Is(err, errHeld) { + e.markFirmwareDown(t.name, t.fw, err.Error()) + } continue } // The radio reports how the firmware has configured it in the same @@ -100,6 +106,9 @@ func (e *Engine) runFirmware(ctx context.Context, now uint32) error { // the end of the tick: deferring inside the loop would hold three hundred // timers open until every node had answered. func (e *Engine) waitTicked(ctx context.Context, t ticked) error { + if _, ok := e.Debugging(t.name); ok { + return e.waitDebugged(ctx, t) + } wctx, cancel := context.WithTimeout(ctx, e.firmwareTickTimeout()) defer cancel() return t.fw.Bridge.WaitAdvance(wctx, t.atMs) diff --git a/internal/ui/workbench/nodeview/nodeview.go b/internal/ui/workbench/nodeview/nodeview.go index 2b73ea48..51e50d6d 100644 --- a/internal/ui/workbench/nodeview/nodeview.go +++ b/internal/ui/workbench/nodeview/nodeview.go @@ -300,7 +300,8 @@ func nodeMenuFor(name string, s *state.Snapshot) []comp.MenuItem { } items := []comp.MenuItem{{Label: "Select on the map", Action: "nodes.select_many"}} if running { - items = append(items, comp.MenuItem{Label: "Stop this node", Action: "node.stop"}) + items = append(items, comp.MenuItem{Label: "Stop this node", Action: "node.stop"}, + comp.MenuItem{Label: "Debug this node", Action: "node.debug"}) } else { items = append(items, comp.MenuItem{Label: "Start this node", Action: "node.start"}) } diff --git a/tools/verbdoc/facade.json b/tools/verbdoc/facade.json index 9cfcc057..0c4008a6 100644 --- a/tools/verbdoc/facade.json +++ b/tools/verbdoc/facade.json @@ -51,6 +51,7 @@ "node.output_window": "node.output_window(source)", "node.start": "node.start()", "node.stop": "node.stop()", + "node.debug": "node.debug(wait=, port=, off=)", "node.set_firmware": "node.firmware = build", "node.set_firmware_only": "node.set_firmware(build, apply=False)", "node.truerf": "node.true_rf = bool", From 2a1f9c1d3fe16ff045562363015cf95c36a2df70 Mon Sep 17 00:00:00 2001 From: R0ck Date: Sun, 25 Oct 2026 00:18:22 +0100 Subject: [PATCH 2/4] nodedebug: lint --- internal/app/session/nodedebug.go | 26 +++++++++++++++----------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/internal/app/session/nodedebug.go b/internal/app/session/nodedebug.go index caa2d51d..1852432f 100644 --- a/internal/app/session/nodedebug.go +++ b/internal/app/session/nodedebug.go @@ -103,7 +103,14 @@ func (s *Sim) restartNode(name string, seed uint64) error { func nativeDebug(name string, b *native.Native) map[string]any { pid := b.PID() syms := symbols(b.Path) - out := map[string]any{ + note := "a breakpoint holds this node and the rest of the run goes on; when it is let go it " + + "catches up to the run's clock. On Linux, attaching needs ptrace permission: " + + "kernel.yama.ptrace_scope 0, or the debugger run as root" + if syms == "stripped" { + note += ". This binary has no symbols: build the firmware with meshbench dev from a " + + "checkout to debug it at source level" + } + return map[string]any{ "node": name, "debugging": true, "backend": "native", "pid": pid, "binary": b.Path, "symbols": syms, "attach": fmt.Sprintf("gdb -p %d %s", pid, b.Path), @@ -111,15 +118,8 @@ func nativeDebug(name string, b *native.Native) map[string]any { "name": "MeshBench " + name, "type": "cppdbg", "request": "attach", "program": b.Path, "processId": fmt.Sprint(pid), "MIMode": "gdb", }, - "note": "a breakpoint holds this node and the rest of the run goes on; when it is let go it " + - "catches up to the run's clock. On Linux, attaching needs ptrace permission: " + - "kernel.yama.ptrace_scope 0, or the debugger run as root", + "note": note, } - if syms == "stripped" { - out["note"] = out["note"].(string) + ". This binary has no symbols: build the firmware " + - "with meshbench dev from a checkout to debug it at source level" - } - return out } // emulatedDebug is how to reach an emulated board's GDB stub. @@ -160,10 +160,14 @@ func symbols(path string) string { } func freePort() (int, error) { - l, err := net.Listen("tcp", "127.0.0.1:0") + var lc net.ListenConfig + l, err := lc.Listen(context.Background(), "tcp", "127.0.0.1:0") if err != nil { return 0, err } defer func() { _ = l.Close() }() - return l.Addr().(*net.TCPAddr).Port, nil + if a, ok := l.Addr().(*net.TCPAddr); ok { + return a.Port, nil + } + return 0, fmt.Errorf("no TCP port to hand out") } From 9e879ab2ffa8d0577e6df8ca742d1939ef65ff2b Mon Sep 17 00:00:00 2001 From: R0ck Date: Fri, 30 Oct 2026 16:58:47 +0000 Subject: [PATCH 3/4] node.debug: a held node that exits or restarts is no longer held for ever, and a bad port is refused --- internal/app/session/nodedebug.go | 5 +- internal/app/session/nodedebug_live_test.go | 2 +- internal/firmware/emulated/gdbstub_test.go | 2 +- internal/firmware/lockstep.go | 5 +- internal/sim/engine/debug.go | 23 ++++-- internal/sim/engine/debug_test.go | 80 ++++++++++++++++++++- internal/sim/engine/engine.go | 2 +- 7 files changed, 107 insertions(+), 12 deletions(-) diff --git a/internal/app/session/nodedebug.go b/internal/app/session/nodedebug.go index 1852432f..f475a33a 100644 --- a/internal/app/session/nodedebug.go +++ b/internal/app/session/nodedebug.go @@ -1,4 +1,4 @@ -// Attaching a debugger to a node's firmware (#191). +// Attaching a debugger to a node's firmware. // // A native node is an ordinary process, so gdb, lldb and an IDE already work // on it; what was missing was which process, whether its binary has symbols, @@ -36,6 +36,9 @@ func registerNodeDebug(st *state.Store, s *Sim) { wait, _ := boolField(p, "wait") port := 0 if v, ok := namedNum(p, "port"); ok { + if v < 1 || v > 65535 || v != float64(int(v)) { + return nil, fmt.Errorf("port %v is not a TCP port: it is a whole number from 1 to 65535", v) + } port = int(v) } switch b := n.Firmware.Backend.(type) { diff --git a/internal/app/session/nodedebug_live_test.go b/internal/app/session/nodedebug_live_test.go index 9c2ac43b..a7652f0e 100644 --- a/internal/app/session/nodedebug_live_test.go +++ b/internal/app/session/nodedebug_live_test.go @@ -13,7 +13,7 @@ import ( // A native node being debugged can be stopped, as a breakpoint stops it, and // the run goes on without it; let go, it rejoins rather than being marked -// down (#191). +// down. func TestLiveADebuggedNativeNodeSurvivesABreakpoint(t *testing.T) { if os.Getenv("MESHBENCH_LIVE") == "" { t.Skip("set MESHBENCH_LIVE=1") diff --git a/internal/firmware/emulated/gdbstub_test.go b/internal/firmware/emulated/gdbstub_test.go index e12a4223..be71a6c3 100644 --- a/internal/firmware/emulated/gdbstub_test.go +++ b/internal/firmware/emulated/gdbstub_test.go @@ -7,7 +7,7 @@ import ( ) // The GDB stub is asked for only when a port is given, on loopback for QEMU, -// and holding the CPU at reset only when asked to (#191). +// and holding the CPU at reset only when asked to. func TestTheGDBStubIsAskedForOnlyWhenWanted(t *testing.T) { if got := (&EmulatedNode{}).qemuGDBArgs(); got != nil { t.Errorf("no port, yet %v", got) diff --git a/internal/firmware/lockstep.go b/internal/firmware/lockstep.go index 76d9de70..0860303d 100644 --- a/internal/firmware/lockstep.go +++ b/internal/firmware/lockstep.go @@ -122,10 +122,13 @@ func (b *Bridge) SetChannelBusy(busy bool) error { // CaughtUp reports, without waiting, whether the node has acknowledged the tick // to atMs: for a node a debugger is holding, which the engine asks each tick -// rather than waits on. +// rather than waits on. A node that has gone counts as caught up: held, it +// would never be ticked again and so never be noticed as down. func (b *Bridge) CaughtUp(atMs uint32) bool { for { select { + case <-b.done: + return true case got := <-b.acked: if got >= atMs { return true diff --git a/internal/sim/engine/debug.go b/internal/sim/engine/debug.go index 5e56dcc4..dc15a02b 100644 --- a/internal/sim/engine/debug.go +++ b/internal/sim/engine/debug.go @@ -4,6 +4,8 @@ import ( "context" "errors" "time" + + "github.com/MeshBench/meshbench/internal/firmware" ) // Debugging a node's firmware while the run goes on. @@ -37,6 +39,15 @@ type Debug struct { // its clock; short enough that the window stays usable while it is stopped. var debugGrace = 2 * time.Second +// heldTick is the tick a held node did not answer, and the firmware that did +// not answer it. The firmware travels with the tick because a node restarted +// while held is a new process that is never sent that tick, so it would never +// answer it and would stay held for the rest of the run. +type heldTick struct { + atMs uint32 + fw *firmware.Node +} + // SetDebug marks a node as being debugged, or with nil as not. It takes // effect for an emulated board when its firmware next starts, and for a // native node at once. @@ -66,20 +77,20 @@ func (e *Engine) Debugging(name string) (Debug, bool) { func (e *Engine) Held(name string) (uint32, bool) { e.mu.Lock() defer e.mu.Unlock() - at, ok := e.held[name] - return at, ok + h, ok := e.held[name] + return h.atMs, ok } // stillHeld is true when a node being debugged has not yet answered the tick // it was held at, and so is skipped this tick. Released, it rejoins. func (e *Engine) stillHeld(t ticked) bool { e.mu.Lock() - at, held := e.held[t.name] + h, held := e.held[t.name] e.mu.Unlock() if !held { return false } - if !t.fw.Bridge.CaughtUp(at) { + if h.fw == t.fw && !t.fw.Bridge.CaughtUp(h.atMs) { return true } e.mu.Lock() @@ -100,9 +111,9 @@ func (e *Engine) waitDebugged(ctx context.Context, t ticked) error { } e.mu.Lock() if e.held == nil { - e.held = map[string]uint32{} + e.held = map[string]heldTick{} } - e.held[t.name] = t.atMs + e.held[t.name] = heldTick{atMs: t.atMs, fw: t.fw} e.mu.Unlock() return errHeld } diff --git a/internal/sim/engine/debug_test.go b/internal/sim/engine/debug_test.go index c89f8704..e7683960 100644 --- a/internal/sim/engine/debug_test.go +++ b/internal/sim/engine/debug_test.go @@ -4,11 +4,13 @@ import ( "context" "testing" "time" + + "github.com/MeshBench/meshbench/internal/firmware" ) // A node stopped in a debugger is held, not failed, and the rest of the mesh // goes on; when the debugger lets it go it answers, and the next tick takes it -// to the run's current instant (#191). +// to the run's current instant. func TestANodeInADebuggerIsHeldAndRejoins(t *testing.T) { old := debugGrace debugGrace = 200 * time.Millisecond @@ -78,3 +80,79 @@ func TestANodeInADebuggerIsHeldAndRejoins(t *testing.T) { t.Error("SetDebug(nil) left it debugged") } } + +// heldNode is a node being debugged that has been held at the first tick it +// left unanswered, beside one that answers every tick. +func heldNode(t *testing.T) (e *Engine, stopped, talker *peer, step func()) { + t.Helper() + old := debugGrace + debugGrace = 200 * time.Millisecond + t.Cleanup(func() { debugGrace = old }) + e = New(flatGround{}, Config{StepMs: 10, FirmwareTickTimeout: 30 * time.Second}) + t.Cleanup(func() { _ = e.Close() }) + stopped = attachedNode(t, e, "stopped") + talker = attachedNode(t, e, "talker") + e.SetDebug("stopped", &Debug{}) + step = func() { + t.Helper() + done := make(chan error, 1) + go func() { done <- e.Step(context.Background()) }() + talker.ack(t, talker.nextTick(t)) + select { + case err := <-done: + if err != nil { + t.Fatalf("Step: %v", err) + } + case <-time.After(5 * time.Second): + t.Fatal("the run stopped") + } + } + step() + stopped.nextTick(t) + if _, ok := e.Held("stopped"); !ok { + t.Fatal("not held") + } + return e, stopped, talker, step +} + +// A held node whose process goes - killed from the debugger, say - is the +// failure it always was, rather than held for the rest of the run. +func TestAHeldNodeThatGoesIsMarkedDown(t *testing.T) { + e, stopped, _, step := heldNode(t) + _ = stopped.conn.Close() + time.Sleep(50 * time.Millisecond) + step() + if down := e.FirmwareFailures(); len(down) != 1 || down[0].Name != "stopped" { + t.Fatalf("FirmwareFailures = %+v, want stopped", down) + } +} + +// A held node started again is a new process that was never sent the tick +// the old one was held at, so it is ticked rather than held waiting for it. +func TestAHeldNodeStartedAgainIsTicked(t *testing.T) { + e, _, talker, _ := heldNode(t) + br, err := firmware.Listen("127.0.0.1:0", "stopped") + if err != nil { + t.Fatal(err) + } + n, _ := e.NodeByName("stopped") + n.Firmware = &firmware.Node{Bridge: br, Backend: quietBackend{}} + fresh := dialPeer(t, br.Addr()) + waitUntilAttached(t, br) + + done := make(chan error, 1) + go func() { done <- e.Step(context.Background()) }() + select { + case at := <-fresh.ticks: + fresh.ack(t, at) + case <-time.After(5 * time.Second): + t.Fatal("the restarted node was never ticked") + } + talker.ack(t, talker.nextTick(t)) + if err := <-done; err != nil { + t.Fatal(err) + } + if _, ok := e.Held("stopped"); ok { + t.Error("the restarted node is still held") + } +} diff --git a/internal/sim/engine/engine.go b/internal/sim/engine/engine.go index 39d5343d..8f44f893 100644 --- a/internal/sim/engine/engine.go +++ b/internal/sim/engine/engine.go @@ -196,7 +196,7 @@ type Engine struct { // debug is which nodes are being debugged, and held which of them a // debugger is holding, at the tick they did not answer. See debug.go. debug map[string]Debug - held map[string]uint32 + held map[string]heldTick firmwareNewlyDown []FirmwareFailure // classCounts is events by class, kept as they are recorded so the cards // that show them never walk the log. From 1e3e50dace04c536e1a150ece4547401374c3191 Mon Sep 17 00:00:00 2001 From: R0ck Date: Fri, 30 Oct 2026 17:36:11 +0000 Subject: [PATCH 4/4] debug: say Renode's GDB stub listens on every interface, not loopback --- internal/firmware/emulated/emulated_node.go | 11 +++++++---- internal/sim/engine/debug.go | 3 ++- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/internal/firmware/emulated/emulated_node.go b/internal/firmware/emulated/emulated_node.go index 53e2812a..9ffc5709 100644 --- a/internal/firmware/emulated/emulated_node.go +++ b/internal/firmware/emulated/emulated_node.go @@ -178,10 +178,13 @@ type EmulatedNode struct { // Emulator selects QEMU or Renode. Emulator Emulator - // GDBPort, when set, starts the emulator's GDB stub on that loopback - // port: source-level debugging of the real firmware binary, the same as a - // probe on a board. GDBWait holds the CPU at reset until a debugger lets - // it go, so a fault during boot can be caught. + // GDBPort, when set, starts the emulator's GDB stub on that port. QEMU's + // listens on loopback; Renode's StartGdbServer takes no address and + // listens on every interface, so anyone who can reach the port can read + // and write the board while it is open. Source-level debugging of the + // real firmware binary, the same as a probe on a board. GDBWait holds the + // CPU at reset until a debugger lets it go, so a fault during boot can be + // caught. GDBPort int GDBWait bool diff --git a/internal/sim/engine/debug.go b/internal/sim/engine/debug.go index dc15a02b..a406f25c 100644 --- a/internal/sim/engine/debug.go +++ b/internal/sim/engine/debug.go @@ -25,7 +25,8 @@ import ( // Debug is how a node is to be run for a debugger. type Debug struct { - // GDBPort is where an emulated board's GDB stub listens, on loopback. + // GDBPort is where an emulated board's GDB stub listens: on loopback + // under QEMU, on every interface under Renode. // Zero for a native node, which a debugger attaches to by its PID. GDBPort int // Wait holds an emulated board's CPU at reset until a debugger lets it