From dd1a9a345107d70c7400e1508e7ee92272d76627 Mon Sep 17 00:00:00 2001 From: Daviduche03 Date: Fri, 6 Nov 2026 08:53:45 +0100 Subject: [PATCH] agent: expose the streamer's wait, record and screenshot MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The console can already say when the screen settles, save an MP4 from the encode it is already running, and hand out its last frame. None of that was reachable from the agent API — which is where it is worth the most, because a computer-use loop could only sleep and hope the UI had caught up. - GET /api/desktops/{id}/wait — returns when the screen stops changing (or at the first change with ?change=1). Damage-driven inside the guest, so waiting costs nothing while the screen is still: no polling, no screenshots. - POST /api/desktops/{id}/record?seconds= — an MP4 written from the live H.264, so no second encoder and no extra CPU. Clamped to 300s. - GET /api/desktops/{id}/screenshot — the streamer's frame when there is one (a copy, not a process spawn), the agent's ImageMagick capture otherwise. An image built before voxl keeps working: /screenshot falls back to the agent and /wait answers 501 with a reason. Audio now switches on by itself for an image that ships a PulseAudio socket, instead of being hardcoded off. --- README.md | 2 + deploy/guest/init | 19 ++++++++-- docs/agent-api.md | 26 +++++++++++++ internal/api/lifecycle.go | 74 ++++++++++++++++++++++++++++++++++++- internal/api/record_test.go | 28 ++++++++++++++ internal/api/server.go | 47 +++++++++++++++++++++++ 6 files changed, 191 insertions(+), 5 deletions(-) create mode 100644 internal/api/record_test.go diff --git a/README.md b/README.md index 6d3ae66..d900c09 100644 --- a/README.md +++ b/README.md @@ -138,6 +138,8 @@ HTTP with no SSH keys and no screen. curl -X POST localhost:7070/api/desktops/$ID/exec -d '{"cmd":"uname -a"}' curl "localhost:7070/api/desktops/$ID/files?path=/home" curl "localhost:7070/api/desktops/$ID/file?path=/etc/hostname" +curl "localhost:7070/api/desktops/$ID/wait?idle=300" # block until the screen settles +curl -X POST "localhost:7070/api/desktops/$ID/record?seconds=10" -o clip.mp4 ``` | Client | Install | diff --git a/deploy/guest/init b/deploy/guest/init index 9cab2cf..edfd031 100644 --- a/deploy/guest/init +++ b/deploy/guest/init @@ -332,12 +332,23 @@ else # It is the same per-VM secret noVNC is given, so neither console widens # what a sibling VM on the NAT could already reach. if command -v voxl >/dev/null 2>&1; then + # Audio only when the image has a sound stack to stream: voxl plays the + # output monitor, and pointing it at a server that is not there only + # delays the first frame. Our images ship no audio yet, so this is off + # by default and switches on by itself for an image that adds one. + AUDIO="--audio-enabled=false" + if [ -S /run/pulse/native ] || [ -S /var/run/pulse/native ] || \ + [ -S "$XDG_RUNTIME_DIR/pulse/native" ]; then + AUDIO="--audio-enabled=true" + fi if [ "$VNC_AUTH" = "1" ] && [ -n "$VNCPASS" ]; then - VOXL_PASSWORD="$VNCPASS" voxl --addr 0.0.0.0 --port 6901 \ - --audio-enabled=false >/var/log/voxl.log 2>&1 & + # shellcheck disable=SC2086 + VOXL_PASSWORD="$VNCPASS" voxl --addr 0.0.0.0 --port 6901 $AUDIO \ + >/var/log/voxl.log 2>&1 & else - voxl --addr 0.0.0.0 --port 6901 --enable-basic-auth=false \ - --audio-enabled=false >/var/log/voxl.log 2>&1 & + # shellcheck disable=SC2086 + voxl --addr 0.0.0.0 --port 6901 --enable-basic-auth=false $AUDIO \ + >/var/log/voxl.log 2>&1 & fi i=0 while [ $i -lt 50 ]; do diff --git a/docs/agent-api.md b/docs/agent-api.md index c569346..8f0e48c 100644 --- a/docs/agent-api.md +++ b/docs/agent-api.md @@ -67,6 +67,8 @@ POST /api/desktops/{id}/file/move {from,to} GET /api/desktops/{id}/screenshot → PNG GET /api/desktops/{id}/screen → {width, height} POST /api/desktops/{id}/input {type:"text"|"key"|"move"|"click", …} +GET /api/desktops/{id}/wait?idle=300&timeout=10000 → {changed, timed_out, seq, waited_ms} +POST /api/desktops/{id}/record?seconds=10 → MP4 ``` - implemented in the guest with `xwd` (+ an X image tool) and `xdotool`. - `text` is `{"type":"text","text":"hello"}`; `key` is `{"type":"key","key":"Return"}` @@ -75,6 +77,30 @@ POST /api/desktops/{id}/input {type:"text"|"key"|"move"|"click", …} - The guest injects the input with XTEST, so it lands on the focused window whatever is streaming the screen. +**Screenshots come from the streamer when there is one.** voxl already holds +the last frame, so `/screenshot` is a copy rather than a process spawn; an +image without voxl falls back to the agent's own capture. + +**`/wait` is the reason a computer-use agent can be reliable.** It returns when +the screen stops changing, so a loop reads as *act → wait → look* instead of +*act → sleep → hope*. It is damage-driven inside the guest (XDamage), so no +polling and no screenshots happen while the screen is still. + +| Query | Meaning | +|---|---| +| `idle` | ms the screen must stay still before returning (default 300) | +| `change` | return at the first change instead of at stillness | +| `region=x,y,w,h` | only watch that rectangle | +| `since=` | return at once if anything changed after that sequence | +| `timeout` | give up after this many ms (default 10000) | + +A blinking text caret redraws about twice a second, so keep `idle` under 500 ms +or leave the caret out of `region`. + +**`/record` reuses the encode that is already running**, so a clip costs no +extra CPU and no second pass. It answers with an MP4 (max 300 s), has no cursor +in the frame, and ends early on a resize. + **Why the guest does the input, and not the VNC server:** the guest's VNC server (TigerVNC) accepts key events on the wire and then drops them before they reach the X server. This was checked properly, not assumed: the RFB diff --git a/internal/api/lifecycle.go b/internal/api/lifecycle.go index 73110b0..6e136c6 100644 --- a/internal/api/lifecycle.go +++ b/internal/api/lifecycle.go @@ -4,6 +4,8 @@ import ( "context" "encoding/json" "net/http" + "net/url" + "strconv" "time" ) @@ -120,8 +122,19 @@ func (s *Server) agentProcesses(w http.ResponseWriter, r *http.Request) { } // agentScreenshot returns a PNG of the guest's screen. +// +// The streamer already holds the last frame, so asking it costs a copy instead +// of spawning ImageMagick inside the guest. An image without voxl falls back to +// the agent, which captures the root window itself. func (s *Server) agentScreenshot(w http.ResponseWriter, r *http.Request) { - s.agentProxy(w, r, r.PathValue("id"), "/screenshot") + vm, ok := s.visibleVM(w, r, r.PathValue("id")) + if !ok { + return + } + if s.voxlRequest(w, r, vm, "/screenshot", nil, 10*time.Second) { + return + } + s.agentProxy(w, r, vm.ID, "/screenshot") } // agentScreen returns the guest's screen size. @@ -129,6 +142,65 @@ func (s *Server) agentScreen(w http.ResponseWriter, r *http.Request) { s.agentProxy(w, r, r.PathValue("id"), "/screen") } +// agentWait blocks until the desktop's screen settles (or, with ?change=1, +// until it changes at all) and answers with voxl's own JSON: changed, timed_out, +// seq and waited_ms. +// +// This is the difference between an agent that sleeps after every click and one +// that reacts to what actually happened. It is damage-driven inside the guest — +// no polling, no screenshots — so waiting costs nothing while the screen is +// still. +func (s *Server) agentWait(w http.ResponseWriter, r *http.Request) { + vm, ok := s.visibleVM(w, r, r.PathValue("id")) + if !ok { + return + } + q := url.Values{} + for _, k := range []string{"idle", "change", "region", "since", "timeout"} { + if v := r.URL.Query().Get(k); v != "" { + q.Set(k, v) + } + } + // voxl's own default timeout is 10s; give the request room above it. + if !s.voxlRequest(w, r, vm, "/wait", q, 60*time.Second) { + writeErr(w, r, http.StatusNotImplemented, "stream_unavailable", + "this desktop has no voxl streamer, so it cannot report when the screen settles", nil) + } +} + +// agentRecord records the screen for a few seconds and answers with an MP4. +// +// voxl saves the H.264 it is already encoding, so there is no second encoder +// and no extra CPU. It does not include the cursor (viewers draw it), and a +// resize ends it. +func (s *Server) agentRecord(w http.ResponseWriter, r *http.Request) { + vm, ok := s.visibleVM(w, r, r.PathValue("id")) + if !ok { + return + } + seconds := recordSeconds(r.URL.Query().Get("seconds")) + w.Header().Set("Content-Disposition", `attachment; filename="`+vm.ID+`.mp4"`) + if !s.voxlRequest(w, r, vm, "/record", url.Values{"seconds": {strconv.Itoa(seconds)}}, + time.Duration(seconds+30)*time.Second) { + w.Header().Del("Content-Disposition") + writeErr(w, r, http.StatusNotImplemented, "stream_unavailable", + "this desktop has no voxl streamer, so it cannot record", nil) + } +} + +// recordSeconds bounds a recording request. 10s is long enough to show a short +// interaction; 300s is voxl's own default ceiling. +func recordSeconds(v string) int { + n, err := strconv.Atoi(v) + if err != nil || n <= 0 { + return 10 + } + if n > 300 { + return 300 + } + return n +} + // agentInput injects a keystroke, a click or a pointer move into the guest's // desktop. The guest does it with XTEST, so it works whether or not anyone is // watching the screen. diff --git a/internal/api/record_test.go b/internal/api/record_test.go new file mode 100644 index 0000000..8735fa9 --- /dev/null +++ b/internal/api/record_test.go @@ -0,0 +1,28 @@ +package api + +import "testing" + +// A recording request is user input, and it decides both how long the guest +// encodes for and how long the daemon waits. An out-of-range value must not +// become "record until the browser goes away" or a zero-length clip. +func TestRecordSecondsIsBounded(t *testing.T) { + cases := []struct { + in string + want int + }{ + {"", 10}, + {"nonsense", 10}, + {"0", 10}, + {"-5", 10}, + {"1", 1}, + {"10", 10}, + {"300", 300}, + {"301", 300}, + {"999999", 300}, + } + for _, c := range cases { + if got := recordSeconds(c.in); got != c.want { + t.Errorf("recordSeconds(%q) = %d, want %d", c.in, got, c.want) + } + } +} diff --git a/internal/api/server.go b/internal/api/server.go index 349effb..6774ff8 100644 --- a/internal/api/server.go +++ b/internal/api/server.go @@ -157,6 +157,8 @@ func (s *Server) Handler() http.Handler { mux.HandleFunc("GET /api/desktops/{id}/screenshot", s.agentScreenshot) mux.HandleFunc("GET /api/desktops/{id}/screen", s.agentScreen) mux.HandleFunc("POST /api/desktops/{id}/input", s.agentInput) + mux.HandleFunc("GET /api/desktops/{id}/wait", s.agentWait) + mux.HandleFunc("POST /api/desktops/{id}/record", s.agentRecord) // Memory checkpoints: live RAM+device snapshots on disk, restorable // without booting (qemu backend; see docs/snapshots.md). @@ -1607,6 +1609,51 @@ func (s *Server) voxlStream(w http.ResponseWriter, r *http.Request) { proxy.ServeHTTP(w, r) } +// voxlRequest performs one request against a desktop's streamer, authenticated +// with the same per-VM secret the console uses, and relays the answer. +// +// It returns false without writing anything when the guest has no streamer (an +// image built before voxl), so a caller can fall back to the agent's own +// implementation. Once the guest answers, its status and body are relayed +// verbatim: voxl says why it refused better than a rephrasing would. +func (s *Server) voxlRequest(w http.ResponseWriter, r *http.Request, vm *desktop.VM, path string, query url.Values, timeout time.Duration) bool { + if s.cfg == nil || s.cfg.GuestVoxlPort <= 0 { + return false + } + ip := vm.IP() + if ip == "" { + return false + } + target := net.JoinHostPort(ip, strconv.Itoa(s.cfg.GuestVoxlPort)) + q := url.Values{} + for k, v := range query { + q[k] = v + } + if p := vm.VNCPassword(); p != "" { + q.Set("token", p) + } + ctx, cancel := context.WithTimeout(r.Context(), timeout) + defer cancel() + req, err := http.NewRequestWithContext(ctx, http.MethodGet, "http://"+target+path+"?"+q.Encode(), nil) + if err != nil { + return false + } + resp, err := guestTransport.RoundTrip(req) + if err != nil { + return false + } + defer resp.Body.Close() + for _, h := range []string{"Content-Type", "Content-Length", "X-Voxl-Seq"} { + if v := resp.Header.Get(h); v != "" { + w.Header().Set(h, v) + } + } + w.WriteHeader(resp.StatusCode) + // A recording is streamed as it is produced, so this copy is the work. + _, _ = io.Copy(w, resp.Body) + return true +} + func (s *Server) novnc(w http.ResponseWriter, r *http.Request) { id := r.PathValue("id") if _, ok := s.visibleVM(w, r, id); !ok {