Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -138,6 +138,8 @@ HTTP with no SSH keys and no screen.
curl -X POST localhost:7070/api/desktops/$ID/exec -d '{"cmd":"uname -a"}'
curl "localhost:7070/api/desktops/$ID/files?path=/home"
curl "localhost:7070/api/desktops/$ID/file?path=/etc/hostname"
curl "localhost:7070/api/desktops/$ID/wait?idle=300" # block until the screen settles
curl -X POST "localhost:7070/api/desktops/$ID/record?seconds=10" -o clip.mp4
```

| Client | Install |
Expand Down
19 changes: 15 additions & 4 deletions deploy/guest/init
Original file line number Diff line number Diff line change
Expand Up @@ -332,12 +332,23 @@ else
# It is the same per-VM secret noVNC is given, so neither console widens
# what a sibling VM on the NAT could already reach.
if command -v voxl >/dev/null 2>&1; then
# Audio only when the image has a sound stack to stream: voxl plays the
# output monitor, and pointing it at a server that is not there only
# delays the first frame. Our images ship no audio yet, so this is off
# by default and switches on by itself for an image that adds one.
AUDIO="--audio-enabled=false"
if [ -S /run/pulse/native ] || [ -S /var/run/pulse/native ] || \
[ -S "$XDG_RUNTIME_DIR/pulse/native" ]; then
AUDIO="--audio-enabled=true"
fi
if [ "$VNC_AUTH" = "1" ] && [ -n "$VNCPASS" ]; then
VOXL_PASSWORD="$VNCPASS" voxl --addr 0.0.0.0 --port 6901 \
--audio-enabled=false >/var/log/voxl.log 2>&1 &
# shellcheck disable=SC2086
VOXL_PASSWORD="$VNCPASS" voxl --addr 0.0.0.0 --port 6901 $AUDIO \
>/var/log/voxl.log 2>&1 &
else
voxl --addr 0.0.0.0 --port 6901 --enable-basic-auth=false \
--audio-enabled=false >/var/log/voxl.log 2>&1 &
# shellcheck disable=SC2086
voxl --addr 0.0.0.0 --port 6901 --enable-basic-auth=false $AUDIO \
>/var/log/voxl.log 2>&1 &
fi
i=0
while [ $i -lt 50 ]; do
Expand Down
26 changes: 26 additions & 0 deletions docs/agent-api.md
Original file line number Diff line number Diff line change
Expand Up @@ -67,6 +67,8 @@ POST /api/desktops/{id}/file/move {from,to}
GET /api/desktops/{id}/screenshot → PNG
GET /api/desktops/{id}/screen → {width, height}
POST /api/desktops/{id}/input {type:"text"|"key"|"move"|"click", …}
GET /api/desktops/{id}/wait?idle=300&timeout=10000 → {changed, timed_out, seq, waited_ms}
POST /api/desktops/{id}/record?seconds=10 → MP4
```
- implemented in the guest with `xwd` (+ an X image tool) and `xdotool`.
- `text` is `{"type":"text","text":"hello"}`; `key` is `{"type":"key","key":"Return"}`
Expand All @@ -75,6 +77,30 @@ POST /api/desktops/{id}/input {type:"text"|"key"|"move"|"click", …}
- The guest injects the input with XTEST, so it lands on the focused window
whatever is streaming the screen.

**Screenshots come from the streamer when there is one.** voxl already holds
the last frame, so `/screenshot` is a copy rather than a process spawn; an
image without voxl falls back to the agent's own capture.

**`/wait` is the reason a computer-use agent can be reliable.** It returns when
the screen stops changing, so a loop reads as *act → wait → look* instead of
*act → sleep → hope*. It is damage-driven inside the guest (XDamage), so no
polling and no screenshots happen while the screen is still.

| Query | Meaning |
|---|---|
| `idle` | ms the screen must stay still before returning (default 300) |
| `change` | return at the first change instead of at stillness |
| `region=x,y,w,h` | only watch that rectangle |
| `since=<seq>` | return at once if anything changed after that sequence |
| `timeout` | give up after this many ms (default 10000) |

A blinking text caret redraws about twice a second, so keep `idle` under 500 ms
or leave the caret out of `region`.

**`/record` reuses the encode that is already running**, so a clip costs no
extra CPU and no second pass. It answers with an MP4 (max 300 s), has no cursor
in the frame, and ends early on a resize.

**Why the guest does the input, and not the VNC server:** the guest's VNC
server (TigerVNC) accepts key events on the wire and then drops them before
they reach the X server. This was checked properly, not assumed: the RFB
Expand Down
74 changes: 73 additions & 1 deletion internal/api/lifecycle.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,8 @@ import (
"context"
"encoding/json"
"net/http"
"net/url"
"strconv"
"time"
)

Expand Down Expand Up @@ -120,15 +122,85 @@ func (s *Server) agentProcesses(w http.ResponseWriter, r *http.Request) {
}

// agentScreenshot returns a PNG of the guest's screen.
//
// The streamer already holds the last frame, so asking it costs a copy instead
// of spawning ImageMagick inside the guest. An image without voxl falls back to
// the agent, which captures the root window itself.
func (s *Server) agentScreenshot(w http.ResponseWriter, r *http.Request) {
s.agentProxy(w, r, r.PathValue("id"), "/screenshot")
vm, ok := s.visibleVM(w, r, r.PathValue("id"))
if !ok {
return
}
if s.voxlRequest(w, r, vm, "/screenshot", nil, 10*time.Second) {
return
}
s.agentProxy(w, r, vm.ID, "/screenshot")
}

// agentScreen returns the guest's screen size.
func (s *Server) agentScreen(w http.ResponseWriter, r *http.Request) {
s.agentProxy(w, r, r.PathValue("id"), "/screen")
}

// agentWait blocks until the desktop's screen settles (or, with ?change=1,
// until it changes at all) and answers with voxl's own JSON: changed, timed_out,
// seq and waited_ms.
//
// This is the difference between an agent that sleeps after every click and one
// that reacts to what actually happened. It is damage-driven inside the guest —
// no polling, no screenshots — so waiting costs nothing while the screen is
// still.
func (s *Server) agentWait(w http.ResponseWriter, r *http.Request) {
vm, ok := s.visibleVM(w, r, r.PathValue("id"))
if !ok {
return
}
q := url.Values{}
for _, k := range []string{"idle", "change", "region", "since", "timeout"} {
if v := r.URL.Query().Get(k); v != "" {
q.Set(k, v)
}
}
// voxl's own default timeout is 10s; give the request room above it.
if !s.voxlRequest(w, r, vm, "/wait", q, 60*time.Second) {
writeErr(w, r, http.StatusNotImplemented, "stream_unavailable",
"this desktop has no voxl streamer, so it cannot report when the screen settles", nil)
}
}

// agentRecord records the screen for a few seconds and answers with an MP4.
//
// voxl saves the H.264 it is already encoding, so there is no second encoder
// and no extra CPU. It does not include the cursor (viewers draw it), and a
// resize ends it.
func (s *Server) agentRecord(w http.ResponseWriter, r *http.Request) {
vm, ok := s.visibleVM(w, r, r.PathValue("id"))
if !ok {
return
}
seconds := recordSeconds(r.URL.Query().Get("seconds"))
w.Header().Set("Content-Disposition", `attachment; filename="`+vm.ID+`.mp4"`)
if !s.voxlRequest(w, r, vm, "/record", url.Values{"seconds": {strconv.Itoa(seconds)}},
time.Duration(seconds+30)*time.Second) {
w.Header().Del("Content-Disposition")
writeErr(w, r, http.StatusNotImplemented, "stream_unavailable",
"this desktop has no voxl streamer, so it cannot record", nil)
}
}

// recordSeconds bounds a recording request. 10s is long enough to show a short
// interaction; 300s is voxl's own default ceiling.
func recordSeconds(v string) int {
n, err := strconv.Atoi(v)
if err != nil || n <= 0 {
return 10
}
if n > 300 {
return 300
}
return n
}

// agentInput injects a keystroke, a click or a pointer move into the guest's
// desktop. The guest does it with XTEST, so it works whether or not anyone is
// watching the screen.
Expand Down
28 changes: 28 additions & 0 deletions internal/api/record_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,28 @@
package api

import "testing"

// A recording request is user input, and it decides both how long the guest
// encodes for and how long the daemon waits. An out-of-range value must not
// become "record until the browser goes away" or a zero-length clip.
func TestRecordSecondsIsBounded(t *testing.T) {
cases := []struct {
in string
want int
}{
{"", 10},
{"nonsense", 10},
{"0", 10},
{"-5", 10},
{"1", 1},
{"10", 10},
{"300", 300},
{"301", 300},
{"999999", 300},
}
for _, c := range cases {
if got := recordSeconds(c.in); got != c.want {
t.Errorf("recordSeconds(%q) = %d, want %d", c.in, got, c.want)
}
}
}
47 changes: 47 additions & 0 deletions internal/api/server.go
Original file line number Diff line number Diff line change
Expand Up @@ -157,6 +157,8 @@ func (s *Server) Handler() http.Handler {
mux.HandleFunc("GET /api/desktops/{id}/screenshot", s.agentScreenshot)
mux.HandleFunc("GET /api/desktops/{id}/screen", s.agentScreen)
mux.HandleFunc("POST /api/desktops/{id}/input", s.agentInput)
mux.HandleFunc("GET /api/desktops/{id}/wait", s.agentWait)
mux.HandleFunc("POST /api/desktops/{id}/record", s.agentRecord)

// Memory checkpoints: live RAM+device snapshots on disk, restorable
// without booting (qemu backend; see docs/snapshots.md).
Expand Down Expand Up @@ -1607,6 +1609,51 @@ func (s *Server) voxlStream(w http.ResponseWriter, r *http.Request) {
proxy.ServeHTTP(w, r)
}

// voxlRequest performs one request against a desktop's streamer, authenticated
// with the same per-VM secret the console uses, and relays the answer.
//
// It returns false without writing anything when the guest has no streamer (an
// image built before voxl), so a caller can fall back to the agent's own
// implementation. Once the guest answers, its status and body are relayed
// verbatim: voxl says why it refused better than a rephrasing would.
func (s *Server) voxlRequest(w http.ResponseWriter, r *http.Request, vm *desktop.VM, path string, query url.Values, timeout time.Duration) bool {
if s.cfg == nil || s.cfg.GuestVoxlPort <= 0 {
return false
}
ip := vm.IP()
if ip == "" {
return false
}
target := net.JoinHostPort(ip, strconv.Itoa(s.cfg.GuestVoxlPort))
q := url.Values{}
for k, v := range query {
q[k] = v
}
if p := vm.VNCPassword(); p != "" {
q.Set("token", p)
}
ctx, cancel := context.WithTimeout(r.Context(), timeout)
defer cancel()
req, err := http.NewRequestWithContext(ctx, http.MethodGet, "http://"+target+path+"?"+q.Encode(), nil)
if err != nil {
return false
}
resp, err := guestTransport.RoundTrip(req)
if err != nil {
return false
}
defer resp.Body.Close()
for _, h := range []string{"Content-Type", "Content-Length", "X-Voxl-Seq"} {
if v := resp.Header.Get(h); v != "" {
w.Header().Set(h, v)
}
}
w.WriteHeader(resp.StatusCode)
// A recording is streamed as it is produced, so this copy is the work.
_, _ = io.Copy(w, resp.Body)
return true
}

func (s *Server) novnc(w http.ResponseWriter, r *http.Request) {
id := r.PathValue("id")
if _, ok := s.visibleVM(w, r, id); !ok {
Expand Down
Loading