feat: Web-based remote desktop streaming via noVNC

.gitignore

@@ -172,6 +172,8 @@ tmp
 .uploads/
 test-results/
 app/backend/rk
+app/backend/rk-virtual-display
+tools/rk-virtual-display/rk-virtual-display
 
 # Go embed requires build/frontend/ to exist for compilation; track the .gitkeep
 app/backend/build/frontend/*

README.md

@@ -59,6 +59,51 @@ To access rk over HTTPS (e.g., from other machines on your tailnet), see:
 
 - [Tailscale guide](docs/wiki/tailscale.md) — zero-config with Tailscale Serve (recommended)
 
+## Desktop Streaming
+
+run-kit can stream graphical desktops to the browser alongside terminal windows.
+
+### macOS Setup
+
+```bash
+brew install --cask xquartz
+brew install x11vnc
+```
+
+**Log out and back in** (or reboot) after installing XQuartz.
+
+### Linux Setup
+
+```bash
+sudo apt install xvfb x11vnc
+```
+
+**Desktop environment** (optional, install one):
+
+```bash
+# KDE Plasma (recommended — full desktop)
+sudo apt install kde-plasma-desktop
+
+# GNOME
+sudo apt install gnome-session
+
+# Xfce (lightweight)
+sudo apt install xfce4
+
+# Openbox (minimal — just window management)
+sudo apt install openbox
+
+# Or skip — bare X11 with no window manager
+```
+
+### Usage
+
+Create a desktop from the command palette (`Cmd+K` → "New Desktop Window"), the window breadcrumb dropdown (`+ New Desktop`), or the dashboard.
+
+Each desktop is a tmux window named `desktop:{label}`. The `desktop:` prefix identifies it as a desktop — rename freely, just keep the prefix.
+
+See [docs/desktop-streaming.md](docs/desktop-streaming.md) for the full architecture, DE support details, isolation model, and troubleshooting.
+
 ## Self-Improvement Loop
 
 rk runs as a daemon in a dedicated tmux session. Lifecycle is managed via CLI flags on `rk serve`:

app/backend/api/relay.go

@@ -3,8 +3,10 @@ package api
 import (
 	"context"
 	"encoding/json"
+	"fmt"
 	"io"
 	"log/slog"
+	"net"
 	"net/http"
 	"os"
 	"os/exec"
@@ -65,24 +67,47 @@ func (s *Server) handleRelay(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	conn, err := upgrader.Upgrade(w, r, nil)
-	if err != nil {
-		slog.Error("websocket upgrade failed", "err", err)
-		return
-	}
-	defer conn.Close()
-
 	// Determine which tmux server this session lives on
 	server := serverFromRequest(r)
 
-	// Verify the session exists and select the target window
+	// Detect window type BEFORE WebSocket upgrade so desktop can use hijack
 	windows, err := s.tmux.ListWindows(r.Context(), session, server)
 	if err != nil || windows == nil {
-		slog.Warn("session not found", "session", session)
-		conn.WriteMessage(websocket.CloseMessage,
-			websocket.FormatCloseMessage(4004, "Session not found"))
+		http.Error(w, "Session not found", http.StatusNotFound)
+		return
+	}
+	var windowType string
+	windowFound := false
+	for _, win := range windows {
+		if win.Index == winIdx {
+			windowType = win.Type
+			windowFound = true
+			break
+		}
+	}
+	if !windowFound {
+		http.Error(w, "Window not found", http.StatusNotFound)
+		return
+	}
+
+	// Desktop: WebSocket-to-TCP proxy (browser WS ↔ x11vnc raw VNC)
+	if windowType == "desktop" {
+		if err := s.tmux.SelectWindow(session, winIdx, server); err != nil {
+			slog.Error("select-window failed", "err", err)
+		}
+		s.handleDesktopRelay(w, r, session, winIdx, server)
+		return
+	}
+
+	// Terminal: proceed with Gorilla WebSocket upgrade
+	conn, err := upgrader.Upgrade(w, r, nil)
+	if err != nil {
+		slog.Error("websocket upgrade failed", "err", err)
 		return
 	}
+	defer conn.Close()
+
+	// Terminal relay (existing behavior)
 	if err := s.tmux.SelectWindow(session, winIdx, server); err != nil {
 		slog.Error("select-window failed", "err", err, "session", session, "window", windowIndex)
 		conn.WriteMessage(websocket.CloseMessage,
@@ -190,3 +215,91 @@ func (s *Server) handleRelay(w http.ResponseWriter, r *http.Request) {
 		}
 	}
 }
+
+// desktopUpgrader negotiates the 'binary' subprotocol that noVNC/websockify use.
+var desktopUpgrader = websocket.Upgrader{
+	CheckOrigin:  func(r *http.Request) bool { return true },
+	Subprotocols: []string{"binary"},
+}
+
+// handleDesktopRelay proxies between a browser WebSocket and x11vnc's raw TCP VNC port.
+func (s *Server) handleDesktopRelay(w http.ResponseWriter, r *http.Request, session string, windowIndex int, server string) {
+	portStr, err := s.tmux.GetWindowOption(session, windowIndex, "@rk_vnc_port", server)
+	if err != nil {
+		slog.Warn("VNC port not found", "session", session, "window", windowIndex, "err", err)
+		http.Error(w, "VNC port not found", http.StatusBadGateway)
+		return
+	}
+	port, err := strconv.Atoi(portStr)
+	if err != nil {
+		http.Error(w, "Invalid VNC port", http.StatusBadGateway)
+		return
+	}
+
+	vncAddr := fmt.Sprintf("127.0.0.1:%d", port)
+	vncConn, err := net.DialTimeout("tcp", vncAddr, 10*time.Second)
+	if err != nil {
+		slog.Error("failed to connect to VNC server", "addr", vncAddr, "err", err)
+		http.Error(w, "VNC connection failed", http.StatusBadGateway)
+		return
+	}
+
+	conn, err := desktopUpgrader.Upgrade(w, r, nil)
+	if err != nil {
+		vncConn.Close()
+		slog.Error("desktop websocket upgrade failed", "err", err)
+		return
+	}
+
+	slog.Info("desktop relay connected", "session", session, "window", windowIndex, "vncAddr", vncAddr, "subprotocol", conn.Subprotocol())
+
+	conn.SetReadDeadline(time.Time{})
+	conn.SetPongHandler(func(string) error { return nil })
+
+	var once sync.Once
+	cleanup := func() {
+		once.Do(func() {
+			conn.Close()
+			vncConn.Close()
+		})
+	}
+	defer cleanup()
+
+	// Keepalive pings every 10s
+	go func() {
+		ticker := time.NewTicker(10 * time.Second)
+		defer ticker.Stop()
+		for range ticker.C {
+			if err := conn.WriteControl(websocket.PingMessage, nil, time.Now().Add(5*time.Second)); err != nil {
+				cleanup()
+				return
+			}
+		}
+	}()
+
+	// Browser WebSocket → VNC TCP
+	go func() {
+		defer cleanup()
+		for {
+			_, msg, err := conn.ReadMessage()
+			if err != nil {
+				return
+			}
+			if _, err := vncConn.Write(msg); err != nil {
+				return
+			}
+		}
+	}()
+
+	// VNC TCP → Browser WebSocket
+	buf := make([]byte, 32*1024)
+	for {
+		n, err := vncConn.Read(buf)
+		if err != nil {
+			return
+		}
+		if err := conn.WriteMessage(websocket.BinaryMessage, buf[:n]); err != nil {
+			return
+		}
+	}
+}

app/backend/api/router.go

@@ -39,6 +39,8 @@ type TmuxOps interface {
 	ListServers(ctx context.Context) ([]string, error)
 	KillServer(server string) error
 	ListKeys(server string) ([]string, error)
+	GetWindowOption(session string, windowIndex int, key, server string) (string, error)
+	SetWindowOption(session string, windowIndex int, key, value, server string) error
 }
 
 // Server holds handler dependencies.
@@ -126,6 +128,12 @@ func (p *prodTmuxOps) KillServer(server string) error {
 func (p *prodTmuxOps) ListKeys(server string) ([]string, error) {
 	return tmux.ListKeys(server)
 }
+func (p *prodTmuxOps) GetWindowOption(session string, windowIndex int, key, server string) (string, error) {
+	return tmux.GetWindowOption(session, windowIndex, key, server)
+}
+func (p *prodTmuxOps) SetWindowOption(session string, windowIndex int, key, value, server string) error {
+	return tmux.SetWindowOption(session, windowIndex, key, value, server)
+}
 
 // NewRouter creates the chi router with all middleware and routes.
 // Uses production dependencies (live tmux, real session fetcher).
@@ -178,6 +186,8 @@ func (s *Server) buildRouter() chi.Router {
 	r.Post("/api/sessions/{session}/windows/{index}/select", s.handleWindowSelect)
 	r.Post("/api/sessions/{session}/windows/{index}/split", s.handleWindowSplit)
 	r.Post("/api/sessions/{session}/windows/{index}/close-pane", s.handleClosePaneKill)
+	r.Post("/api/sessions/{session}/windows/{index}/resolution", s.handleWindowResolution)
+	r.Get("/api/sessions/{session}/windows/{index}/desktop-info", s.handleDesktopInfo)
 	r.Get("/api/directories", s.handleDirectories)
 	r.Post("/api/sessions/{session}/upload", s.handleUpload)
 	r.Get("/api/sessions/stream", s.handleSSE)

app/backend/api/sessions_test.go

@@ -3,6 +3,7 @@ package api
 import (
 	"context"
 	"encoding/json"
+	"fmt"
 	"log/slog"
 	"net/http"
 	"net/http/httptest"
@@ -65,6 +66,11 @@ type mockTmuxOps struct {
 	killActivePaneSession string
 	killActivePaneIndex   int
 
+	getWindowOptionResult string
+	setWindowOptionCalled bool
+	setWindowOptionKey    string
+	setWindowOptionValue  string
+
 	err error
 }
 
@@ -90,6 +96,20 @@ func (m *mockTmuxOps) CreateWindow(session, name, cwd, server string) error {
 	m.createWindowSession = session
 	m.createWindowName = name
 	m.createWindowCwd = cwd
+	// Append the created window to listWindowsResult so subsequent ListWindows
+	// calls can find it (needed for desktop window creation flow).
+	if m.err == nil {
+		nextIndex := len(m.listWindowsResult)
+		winType := "terminal"
+		if len(name) > 8 && name[:8] == "desktop:" {
+			winType = "desktop"
+		}
+		m.listWindowsResult = append(m.listWindowsResult, tmux.WindowInfo{
+			Index: nextIndex,
+			Name:  name,
+			Type:  winType,
+		})
+	}
 	return m.err
 }
 func (m *mockTmuxOps) KillWindow(session string, index int, server string) error {
@@ -143,6 +163,18 @@ func (m *mockTmuxOps) KillServer(server string) error {
 func (m *mockTmuxOps) ListKeys(server string) ([]string, error) {
 	return nil, nil
 }
+func (m *mockTmuxOps) GetWindowOption(session string, windowIndex int, key, server string) (string, error) {
+	if m.getWindowOptionResult != "" {
+		return m.getWindowOptionResult, nil
+	}
+	return "", fmt.Errorf("option not set")
+}
+func (m *mockTmuxOps) SetWindowOption(session string, windowIndex int, key, value, server string) error {
+	m.setWindowOptionCalled = true
+	m.setWindowOptionKey = key
+	m.setWindowOptionValue = value
+	return m.err
+}
 
 func newTestRouter(sf SessionFetcher, ops TmuxOps) http.Handler {
 	logger := slog.New(slog.NewTextHandler(os.Stderr, nil))