diff --git a/README.md b/README.md index 6d3ae66..d900c09 100644 --- a/README.md +++ b/README.md @@ -138,6 +138,8 @@ HTTP with no SSH keys and no screen. curl -X POST localhost:7070/api/desktops/$ID/exec -d '{"cmd":"uname -a"}' curl "localhost:7070/api/desktops/$ID/files?path=/home" curl "localhost:7070/api/desktops/$ID/file?path=/etc/hostname" +curl "localhost:7070/api/desktops/$ID/wait?idle=300" # block until the screen settles +curl -X POST "localhost:7070/api/desktops/$ID/record?seconds=10" -o clip.mp4 ``` | Client | Install | diff --git a/deploy/guest/init b/deploy/guest/init index 9cab2cf..edfd031 100644 --- a/deploy/guest/init +++ b/deploy/guest/init @@ -332,12 +332,23 @@ else # It is the same per-VM secret noVNC is given, so neither console widens # what a sibling VM on the NAT could already reach. if command -v voxl >/dev/null 2>&1; then + # Audio only when the image has a sound stack to stream: voxl plays the + # output monitor, and pointing it at a server that is not there only + # delays the first frame. Our images ship no audio yet, so this is off + # by default and switches on by itself for an image that adds one. + AUDIO="--audio-enabled=false" + if [ -S /run/pulse/native ] || [ -S /var/run/pulse/native ] || \ + [ -S "$XDG_RUNTIME_DIR/pulse/native" ]; then + AUDIO="--audio-enabled=true" + fi if [ "$VNC_AUTH" = "1" ] && [ -n "$VNCPASS" ]; then - VOXL_PASSWORD="$VNCPASS" voxl --addr 0.0.0.0 --port 6901 \ - --audio-enabled=false >/var/log/voxl.log 2>&1 & + # shellcheck disable=SC2086 + VOXL_PASSWORD="$VNCPASS" voxl --addr 0.0.0.0 --port 6901 $AUDIO \ + >/var/log/voxl.log 2>&1 & else - voxl --addr 0.0.0.0 --port 6901 --enable-basic-auth=false \ - --audio-enabled=false >/var/log/voxl.log 2>&1 & + # shellcheck disable=SC2086 + voxl --addr 0.0.0.0 --port 6901 --enable-basic-auth=false $AUDIO \ + >/var/log/voxl.log 2>&1 & fi i=0 while [ $i -lt 50 ]; do diff --git a/docs/agent-api.md b/docs/agent-api.md index c569346..8f0e48c 100644 --- a/docs/agent-api.md +++ b/docs/agent-api.md @@ -67,6 +67,8 @@ POST /api/desktops/{id}/file/move {from,to} GET /api/desktops/{id}/screenshot → PNG GET /api/desktops/{id}/screen → {width, height} POST /api/desktops/{id}/input {type:"text"|"key"|"move"|"click", …} +GET /api/desktops/{id}/wait?idle=300&timeout=10000 → {changed, timed_out, seq, waited_ms} +POST /api/desktops/{id}/record?seconds=10 → MP4 ``` - implemented in the guest with `xwd` (+ an X image tool) and `xdotool`. - `text` is `{"type":"text","text":"hello"}`; `key` is `{"type":"key","key":"Return"}` @@ -75,6 +77,30 @@ POST /api/desktops/{id}/input {type:"text"|"key"|"move"|"click", …} - The guest injects the input with XTEST, so it lands on the focused window whatever is streaming the screen. +**Screenshots come from the streamer when there is one.** voxl already holds +the last frame, so `/screenshot` is a copy rather than a process spawn; an +image without voxl falls back to the agent's own capture. + +**`/wait` is the reason a computer-use agent can be reliable.** It returns when +the screen stops changing, so a loop reads as *act → wait → look* instead of +*act → sleep → hope*. It is damage-driven inside the guest (XDamage), so no +polling and no screenshots happen while the screen is still. + +| Query | Meaning | +|---|---| +| `idle` | ms the screen must stay still before returning (default 300) | +| `change` | return at the first change instead of at stillness | +| `region=x,y,w,h` | only watch that rectangle | +| `since=` | return at once if anything changed after that sequence | +| `timeout` | give up after this many ms (default 10000) | + +A blinking text caret redraws about twice a second, so keep `idle` under 500 ms +or leave the caret out of `region`. + +**`/record` reuses the encode that is already running**, so a clip costs no +extra CPU and no second pass. It answers with an MP4 (max 300 s), has no cursor +in the frame, and ends early on a resize. + **Why the guest does the input, and not the VNC server:** the guest's VNC server (TigerVNC) accepts key events on the wire and then drops them before they reach the X server. This was checked properly, not assumed: the RFB diff --git a/internal/api/lifecycle.go b/internal/api/lifecycle.go index 73110b0..6e136c6 100644 --- a/internal/api/lifecycle.go +++ b/internal/api/lifecycle.go @@ -4,6 +4,8 @@ import ( "context" "encoding/json" "net/http" + "net/url" + "strconv" "time" ) @@ -120,8 +122,19 @@ func (s *Server) agentProcesses(w http.ResponseWriter, r *http.Request) { } // agentScreenshot returns a PNG of the guest's screen. +// +// The streamer already holds the last frame, so asking it costs a copy instead +// of spawning ImageMagick inside the guest. An image without voxl falls back to +// the agent, which captures the root window itself. func (s *Server) agentScreenshot(w http.ResponseWriter, r *http.Request) { - s.agentProxy(w, r, r.PathValue("id"), "/screenshot") + vm, ok := s.visibleVM(w, r, r.PathValue("id")) + if !ok { + return + } + if s.voxlRequest(w, r, vm, "/screenshot", nil, 10*time.Second) { + return + } + s.agentProxy(w, r, vm.ID, "/screenshot") } // agentScreen returns the guest's screen size. @@ -129,6 +142,65 @@ func (s *Server) agentScreen(w http.ResponseWriter, r *http.Request) { s.agentProxy(w, r, r.PathValue("id"), "/screen") } +// agentWait blocks until the desktop's screen settles (or, with ?change=1, +// until it changes at all) and answers with voxl's own JSON: changed, timed_out, +// seq and waited_ms. +// +// This is the difference between an agent that sleeps after every click and one +// that reacts to what actually happened. It is damage-driven inside the guest — +// no polling, no screenshots — so waiting costs nothing while the screen is +// still. +func (s *Server) agentWait(w http.ResponseWriter, r *http.Request) { + vm, ok := s.visibleVM(w, r, r.PathValue("id")) + if !ok { + return + } + q := url.Values{} + for _, k := range []string{"idle", "change", "region", "since", "timeout"} { + if v := r.URL.Query().Get(k); v != "" { + q.Set(k, v) + } + } + // voxl's own default timeout is 10s; give the request room above it. + if !s.voxlRequest(w, r, vm, "/wait", q, 60*time.Second) { + writeErr(w, r, http.StatusNotImplemented, "stream_unavailable", + "this desktop has no voxl streamer, so it cannot report when the screen settles", nil) + } +} + +// agentRecord records the screen for a few seconds and answers with an MP4. +// +// voxl saves the H.264 it is already encoding, so there is no second encoder +// and no extra CPU. It does not include the cursor (viewers draw it), and a +// resize ends it. +func (s *Server) agentRecord(w http.ResponseWriter, r *http.Request) { + vm, ok := s.visibleVM(w, r, r.PathValue("id")) + if !ok { + return + } + seconds := recordSeconds(r.URL.Query().Get("seconds")) + w.Header().Set("Content-Disposition", `attachment; filename="`+vm.ID+`.mp4"`) + if !s.voxlRequest(w, r, vm, "/record", url.Values{"seconds": {strconv.Itoa(seconds)}}, + time.Duration(seconds+30)*time.Second) { + w.Header().Del("Content-Disposition") + writeErr(w, r, http.StatusNotImplemented, "stream_unavailable", + "this desktop has no voxl streamer, so it cannot record", nil) + } +} + +// recordSeconds bounds a recording request. 10s is long enough to show a short +// interaction; 300s is voxl's own default ceiling. +func recordSeconds(v string) int { + n, err := strconv.Atoi(v) + if err != nil || n <= 0 { + return 10 + } + if n > 300 { + return 300 + } + return n +} + // agentInput injects a keystroke, a click or a pointer move into the guest's // desktop. The guest does it with XTEST, so it works whether or not anyone is // watching the screen. diff --git a/internal/api/record_test.go b/internal/api/record_test.go new file mode 100644 index 0000000..8735fa9 --- /dev/null +++ b/internal/api/record_test.go @@ -0,0 +1,28 @@ +package api + +import "testing" + +// A recording request is user input, and it decides both how long the guest +// encodes for and how long the daemon waits. An out-of-range value must not +// become "record until the browser goes away" or a zero-length clip. +func TestRecordSecondsIsBounded(t *testing.T) { + cases := []struct { + in string + want int + }{ + {"", 10}, + {"nonsense", 10}, + {"0", 10}, + {"-5", 10}, + {"1", 1}, + {"10", 10}, + {"300", 300}, + {"301", 300}, + {"999999", 300}, + } + for _, c := range cases { + if got := recordSeconds(c.in); got != c.want { + t.Errorf("recordSeconds(%q) = %d, want %d", c.in, got, c.want) + } + } +} diff --git a/internal/api/server.go b/internal/api/server.go index 349effb..6774ff8 100644 --- a/internal/api/server.go +++ b/internal/api/server.go @@ -157,6 +157,8 @@ func (s *Server) Handler() http.Handler { mux.HandleFunc("GET /api/desktops/{id}/screenshot", s.agentScreenshot) mux.HandleFunc("GET /api/desktops/{id}/screen", s.agentScreen) mux.HandleFunc("POST /api/desktops/{id}/input", s.agentInput) + mux.HandleFunc("GET /api/desktops/{id}/wait", s.agentWait) + mux.HandleFunc("POST /api/desktops/{id}/record", s.agentRecord) // Memory checkpoints: live RAM+device snapshots on disk, restorable // without booting (qemu backend; see docs/snapshots.md). @@ -1607,6 +1609,51 @@ func (s *Server) voxlStream(w http.ResponseWriter, r *http.Request) { proxy.ServeHTTP(w, r) } +// voxlRequest performs one request against a desktop's streamer, authenticated +// with the same per-VM secret the console uses, and relays the answer. +// +// It returns false without writing anything when the guest has no streamer (an +// image built before voxl), so a caller can fall back to the agent's own +// implementation. Once the guest answers, its status and body are relayed +// verbatim: voxl says why it refused better than a rephrasing would. +func (s *Server) voxlRequest(w http.ResponseWriter, r *http.Request, vm *desktop.VM, path string, query url.Values, timeout time.Duration) bool { + if s.cfg == nil || s.cfg.GuestVoxlPort <= 0 { + return false + } + ip := vm.IP() + if ip == "" { + return false + } + target := net.JoinHostPort(ip, strconv.Itoa(s.cfg.GuestVoxlPort)) + q := url.Values{} + for k, v := range query { + q[k] = v + } + if p := vm.VNCPassword(); p != "" { + q.Set("token", p) + } + ctx, cancel := context.WithTimeout(r.Context(), timeout) + defer cancel() + req, err := http.NewRequestWithContext(ctx, http.MethodGet, "http://"+target+path+"?"+q.Encode(), nil) + if err != nil { + return false + } + resp, err := guestTransport.RoundTrip(req) + if err != nil { + return false + } + defer resp.Body.Close() + for _, h := range []string{"Content-Type", "Content-Length", "X-Voxl-Seq"} { + if v := resp.Header.Get(h); v != "" { + w.Header().Set(h, v) + } + } + w.WriteHeader(resp.StatusCode) + // A recording is streamed as it is produced, so this copy is the work. + _, _ = io.Copy(w, resp.Body) + return true +} + func (s *Server) novnc(w http.ResponseWriter, r *http.Request) { id := r.PathValue("id") if _, ok := s.visibleVM(w, r, id); !ok {