diff --git a/.gitignore b/.gitignore index 171db521..b16b6c4b 100644 --- a/.gitignore +++ b/.gitignore @@ -172,6 +172,8 @@ tmp .uploads/ test-results/ app/backend/rk +app/backend/rk-virtual-display +tools/rk-virtual-display/rk-virtual-display # Go embed requires build/frontend/ to exist for compilation; track the .gitkeep app/backend/build/frontend/* diff --git a/README.md b/README.md index 3d3c0720..6875bb7a 100644 --- a/README.md +++ b/README.md @@ -59,6 +59,51 @@ To access rk over HTTPS (e.g., from other machines on your tailnet), see: - [Tailscale guide](docs/wiki/tailscale.md) — zero-config with Tailscale Serve (recommended) +## Desktop Streaming + +run-kit can stream graphical desktops to the browser alongside terminal windows. + +### macOS Setup + +```bash +brew install --cask xquartz +brew install x11vnc +``` + +**Log out and back in** (or reboot) after installing XQuartz. + +### Linux Setup + +```bash +sudo apt install xvfb x11vnc +``` + +**Desktop environment** (optional, install one): + +```bash +# KDE Plasma (recommended — full desktop) +sudo apt install kde-plasma-desktop + +# GNOME +sudo apt install gnome-session + +# Xfce (lightweight) +sudo apt install xfce4 + +# Openbox (minimal — just window management) +sudo apt install openbox + +# Or skip — bare X11 with no window manager +``` + +### Usage + +Create a desktop from the command palette (`Cmd+K` → "New Desktop Window"), the window breadcrumb dropdown (`+ New Desktop`), or the dashboard. + +Each desktop is a tmux window named `desktop:{label}`. The `desktop:` prefix identifies it as a desktop — rename freely, just keep the prefix. + +See [docs/desktop-streaming.md](docs/desktop-streaming.md) for the full architecture, DE support details, isolation model, and troubleshooting. + ## Self-Improvement Loop rk runs as a daemon in a dedicated tmux session. Lifecycle is managed via CLI flags on `rk serve`: diff --git a/app/backend/api/relay.go b/app/backend/api/relay.go index 16a7c9c1..fd263dd7 100644 --- a/app/backend/api/relay.go +++ b/app/backend/api/relay.go @@ -3,8 +3,10 @@ package api import ( "context" "encoding/json" + "fmt" "io" "log/slog" + "net" "net/http" "os" "os/exec" @@ -65,24 +67,47 @@ func (s *Server) handleRelay(w http.ResponseWriter, r *http.Request) { return } - conn, err := upgrader.Upgrade(w, r, nil) - if err != nil { - slog.Error("websocket upgrade failed", "err", err) - return - } - defer conn.Close() - // Determine which tmux server this session lives on server := serverFromRequest(r) - // Verify the session exists and select the target window + // Detect window type BEFORE WebSocket upgrade so desktop can use hijack windows, err := s.tmux.ListWindows(r.Context(), session, server) if err != nil || windows == nil { - slog.Warn("session not found", "session", session) - conn.WriteMessage(websocket.CloseMessage, - websocket.FormatCloseMessage(4004, "Session not found")) + http.Error(w, "Session not found", http.StatusNotFound) + return + } + var windowType string + windowFound := false + for _, win := range windows { + if win.Index == winIdx { + windowType = win.Type + windowFound = true + break + } + } + if !windowFound { + http.Error(w, "Window not found", http.StatusNotFound) + return + } + + // Desktop: WebSocket-to-TCP proxy (browser WS ↔ x11vnc raw VNC) + if windowType == "desktop" { + if err := s.tmux.SelectWindow(session, winIdx, server); err != nil { + slog.Error("select-window failed", "err", err) + } + s.handleDesktopRelay(w, r, session, winIdx, server) + return + } + + // Terminal: proceed with Gorilla WebSocket upgrade + conn, err := upgrader.Upgrade(w, r, nil) + if err != nil { + slog.Error("websocket upgrade failed", "err", err) return } + defer conn.Close() + + // Terminal relay (existing behavior) if err := s.tmux.SelectWindow(session, winIdx, server); err != nil { slog.Error("select-window failed", "err", err, "session", session, "window", windowIndex) conn.WriteMessage(websocket.CloseMessage, @@ -190,3 +215,91 @@ func (s *Server) handleRelay(w http.ResponseWriter, r *http.Request) { } } } + +// desktopUpgrader negotiates the 'binary' subprotocol that noVNC/websockify use. +var desktopUpgrader = websocket.Upgrader{ + CheckOrigin: func(r *http.Request) bool { return true }, + Subprotocols: []string{"binary"}, +} + +// handleDesktopRelay proxies between a browser WebSocket and x11vnc's raw TCP VNC port. +func (s *Server) handleDesktopRelay(w http.ResponseWriter, r *http.Request, session string, windowIndex int, server string) { + portStr, err := s.tmux.GetWindowOption(session, windowIndex, "@rk_vnc_port", server) + if err != nil { + slog.Warn("VNC port not found", "session", session, "window", windowIndex, "err", err) + http.Error(w, "VNC port not found", http.StatusBadGateway) + return + } + port, err := strconv.Atoi(portStr) + if err != nil { + http.Error(w, "Invalid VNC port", http.StatusBadGateway) + return + } + + vncAddr := fmt.Sprintf("127.0.0.1:%d", port) + vncConn, err := net.DialTimeout("tcp", vncAddr, 10*time.Second) + if err != nil { + slog.Error("failed to connect to VNC server", "addr", vncAddr, "err", err) + http.Error(w, "VNC connection failed", http.StatusBadGateway) + return + } + + conn, err := desktopUpgrader.Upgrade(w, r, nil) + if err != nil { + vncConn.Close() + slog.Error("desktop websocket upgrade failed", "err", err) + return + } + + slog.Info("desktop relay connected", "session", session, "window", windowIndex, "vncAddr", vncAddr, "subprotocol", conn.Subprotocol()) + + conn.SetReadDeadline(time.Time{}) + conn.SetPongHandler(func(string) error { return nil }) + + var once sync.Once + cleanup := func() { + once.Do(func() { + conn.Close() + vncConn.Close() + }) + } + defer cleanup() + + // Keepalive pings every 10s + go func() { + ticker := time.NewTicker(10 * time.Second) + defer ticker.Stop() + for range ticker.C { + if err := conn.WriteControl(websocket.PingMessage, nil, time.Now().Add(5*time.Second)); err != nil { + cleanup() + return + } + } + }() + + // Browser WebSocket → VNC TCP + go func() { + defer cleanup() + for { + _, msg, err := conn.ReadMessage() + if err != nil { + return + } + if _, err := vncConn.Write(msg); err != nil { + return + } + } + }() + + // VNC TCP → Browser WebSocket + buf := make([]byte, 32*1024) + for { + n, err := vncConn.Read(buf) + if err != nil { + return + } + if err := conn.WriteMessage(websocket.BinaryMessage, buf[:n]); err != nil { + return + } + } +} diff --git a/app/backend/api/router.go b/app/backend/api/router.go index 2c149191..73c28b19 100644 --- a/app/backend/api/router.go +++ b/app/backend/api/router.go @@ -39,6 +39,8 @@ type TmuxOps interface { ListServers(ctx context.Context) ([]string, error) KillServer(server string) error ListKeys(server string) ([]string, error) + GetWindowOption(session string, windowIndex int, key, server string) (string, error) + SetWindowOption(session string, windowIndex int, key, value, server string) error } // Server holds handler dependencies. @@ -126,6 +128,12 @@ func (p *prodTmuxOps) KillServer(server string) error { func (p *prodTmuxOps) ListKeys(server string) ([]string, error) { return tmux.ListKeys(server) } +func (p *prodTmuxOps) GetWindowOption(session string, windowIndex int, key, server string) (string, error) { + return tmux.GetWindowOption(session, windowIndex, key, server) +} +func (p *prodTmuxOps) SetWindowOption(session string, windowIndex int, key, value, server string) error { + return tmux.SetWindowOption(session, windowIndex, key, value, server) +} // NewRouter creates the chi router with all middleware and routes. // Uses production dependencies (live tmux, real session fetcher). @@ -178,6 +186,8 @@ func (s *Server) buildRouter() chi.Router { r.Post("/api/sessions/{session}/windows/{index}/select", s.handleWindowSelect) r.Post("/api/sessions/{session}/windows/{index}/split", s.handleWindowSplit) r.Post("/api/sessions/{session}/windows/{index}/close-pane", s.handleClosePaneKill) + r.Post("/api/sessions/{session}/windows/{index}/resolution", s.handleWindowResolution) + r.Get("/api/sessions/{session}/windows/{index}/desktop-info", s.handleDesktopInfo) r.Get("/api/directories", s.handleDirectories) r.Post("/api/sessions/{session}/upload", s.handleUpload) r.Get("/api/sessions/stream", s.handleSSE) diff --git a/app/backend/api/sessions_test.go b/app/backend/api/sessions_test.go index 5cddfe72..6e1eb725 100644 --- a/app/backend/api/sessions_test.go +++ b/app/backend/api/sessions_test.go @@ -3,6 +3,7 @@ package api import ( "context" "encoding/json" + "fmt" "log/slog" "net/http" "net/http/httptest" @@ -65,6 +66,11 @@ type mockTmuxOps struct { killActivePaneSession string killActivePaneIndex int + getWindowOptionResult string + setWindowOptionCalled bool + setWindowOptionKey string + setWindowOptionValue string + err error } @@ -90,6 +96,20 @@ func (m *mockTmuxOps) CreateWindow(session, name, cwd, server string) error { m.createWindowSession = session m.createWindowName = name m.createWindowCwd = cwd + // Append the created window to listWindowsResult so subsequent ListWindows + // calls can find it (needed for desktop window creation flow). + if m.err == nil { + nextIndex := len(m.listWindowsResult) + winType := "terminal" + if len(name) > 8 && name[:8] == "desktop:" { + winType = "desktop" + } + m.listWindowsResult = append(m.listWindowsResult, tmux.WindowInfo{ + Index: nextIndex, + Name: name, + Type: winType, + }) + } return m.err } func (m *mockTmuxOps) KillWindow(session string, index int, server string) error { @@ -143,6 +163,18 @@ func (m *mockTmuxOps) KillServer(server string) error { func (m *mockTmuxOps) ListKeys(server string) ([]string, error) { return nil, nil } +func (m *mockTmuxOps) GetWindowOption(session string, windowIndex int, key, server string) (string, error) { + if m.getWindowOptionResult != "" { + return m.getWindowOptionResult, nil + } + return "", fmt.Errorf("option not set") +} +func (m *mockTmuxOps) SetWindowOption(session string, windowIndex int, key, value, server string) error { + m.setWindowOptionCalled = true + m.setWindowOptionKey = key + m.setWindowOptionValue = value + return m.err +} func newTestRouter(sf SessionFetcher, ops TmuxOps) http.Handler { logger := slog.New(slog.NewTextHandler(os.Stderr, nil)) diff --git a/app/backend/api/windows.go b/app/backend/api/windows.go index 117735fd..e448cb40 100644 --- a/app/backend/api/windows.go +++ b/app/backend/api/windows.go @@ -3,7 +3,13 @@ package api import ( "context" "encoding/json" + "fmt" + "log/slog" + "net" "net/http" + "os" + "path/filepath" + "runtime" "strconv" "strings" "time" @@ -21,8 +27,10 @@ func (s *Server) handleWindowCreate(w http.ResponseWriter, r *http.Request) { } var body struct { - Name string `json:"name"` - CWD string `json:"cwd"` + Name string `json:"name"` + CWD string `json:"cwd"` + Type string `json:"type"` + Resolution string `json:"resolution"` } if err := json.NewDecoder(r.Body).Decode(&body); err != nil { writeError(w, http.StatusBadRequest, "Invalid JSON body") @@ -36,6 +44,93 @@ func (s *Server) handleWindowCreate(w http.ResponseWriter, r *http.Request) { server := serverFromRequest(r) + // Desktop window creation + if body.Type == "desktop" { + resolution := body.Resolution + if resolution == "" { + resolution = "1920x1080" + } + if errMsg := validate.ValidateResolution(resolution); errMsg != "" { + writeError(w, http.StatusBadRequest, errMsg) + return + } + + // Allocate a free port for VNC + port, err := allocateFreePort() + if err != nil { + writeError(w, http.StatusInternalServerError, "Failed to allocate VNC port") + return + } + + // Derive display number from port + displayNum := port - 5900 + if displayNum < 0 { + displayNum = port % 1000 + } + + // Create tmux window with desktop: prefix. Auto-number if no name given. + desktopName := body.Name + if desktopName == "" || desktopName == "desktop" { + // Count existing desktop windows to generate next number + existingWindows, _ := s.tmux.ListWindows(r.Context(), session, server) + n := 1 + for _, w := range existingWindows { + if strings.HasPrefix(w.Name, "desktop:") { + n++ + } + } + desktopName = strconv.Itoa(n) + } + windowName := "desktop:" + desktopName + var resolvedCwd string + if windows, listErr := s.tmux.ListWindows(r.Context(), session, server); listErr == nil && len(windows) > 0 { + resolvedCwd = windows[0].WorktreePath + } + if err := s.tmux.CreateWindow(session, windowName, resolvedCwd, server); err != nil { + writeError(w, http.StatusInternalServerError, err.Error()) + return + } + + // Find the newly created window index + windows, err := s.tmux.ListWindows(r.Context(), session, server) + if err != nil { + writeError(w, http.StatusInternalServerError, err.Error()) + return + } + windowIndex := -1 + for _, win := range windows { + if win.Name == windowName { + windowIndex = win.Index + } + } + if windowIndex < 0 { + writeError(w, http.StatusInternalServerError, "Failed to find created desktop window") + return + } + + // Store VNC port as tmux window option + if err := s.tmux.SetWindowOption(session, windowIndex, "@rk_vnc_port", strconv.Itoa(port), server); err != nil { + slog.Error("failed to set VNC port window option", "err", err) + } + + // Write startup script to temp file (too large for send-keys buffer) + script := desktopStartupScript(displayNum, port, resolution) + scriptFile := fmt.Sprintf("/tmp/rk-desktop-%d.sh", port) + if err := os.WriteFile(scriptFile, []byte(script), 0700); err != nil { + writeError(w, http.StatusInternalServerError, "Failed to write startup script") + return + } + if err := s.tmux.SendKeys(session, windowIndex, scriptFile, server); err != nil { + slog.Error("failed to send desktop startup command", "err", err) + writeError(w, http.StatusInternalServerError, err.Error()) + return + } + + writeJSON(w, http.StatusCreated, map[string]bool{"ok": true}) + return + } + + // Terminal window creation (existing behavior) var resolvedCwd string if body.CWD != "" { if errMsg := validate.ValidatePath(body.CWD, "Working directory"); errMsg != "" { @@ -69,6 +164,136 @@ func (s *Server) handleWindowCreate(w http.ResponseWriter, r *http.Request) { writeJSON(w, http.StatusCreated, map[string]bool{"ok": true}) } +// allocateFreePort finds a free TCP port using the net.Listen trick. +func allocateFreePort() (int, error) { + l, err := net.Listen("tcp", ":0") + if err != nil { + return 0, err + } + port := l.Addr().(*net.TCPAddr).Port + l.Close() + return port, nil +} + +// desktopStartupScript generates a platform-appropriate bash script. +// Linux: Xvfb + x11vnc (virtual X display). +// macOS: rk-virtual-display (native virtual display via CGVirtualDisplay + ScreenCaptureKit + libvncserver). +func desktopStartupScript(displayNum, port int, resolution string) string { + if runtime.GOOS == "darwin" { + return desktopStartupScriptDarwin(port, resolution) + } + return desktopStartupScriptLinux(displayNum, port, resolution) +} + +func desktopStartupScriptDarwin(port int, resolution string) string { + // Resolve the path to rk-virtual-display by looking next to our own executable + selfPath, _ := os.Executable() + selfDir := "" + if selfPath != "" { + selfDir = filepath.Dir(selfPath) + } + + return fmt.Sprintf(`#!/bin/bash +RES="%s" +WIDTH="${RES%%x*}" +HEIGHT="${RES##*x}" + +# Find rk-virtual-display binary +RK_VD="" +for p in \ + "%s/rk-virtual-display" \ + /usr/local/bin/rk-virtual-display \ + "$HOME/.local/bin/rk-virtual-display"; do + [ -x "$p" ] && { RK_VD="$p"; break; } +done + +if [ -z "$RK_VD" ]; then + echo "ERROR: rk-virtual-display not found." + echo "Build it: cd tools/rk-virtual-display && make && make install" + exit 1 +fi + +exec "$RK_VD" --width "$WIDTH" --height "$HEIGHT" --port %d +`, resolution, selfDir, port) +} + +func desktopStartupScriptLinux(displayNum, port int, resolution string) string { + return fmt.Sprintf(`#!/bin/bash +export DISPLAY=:%d + +# Isolate per-desktop state so apps (browsers, etc.) don't collide across desktops. +DESKTOP_ID=desktop-%d +export XDG_RUNTIME_DIR="${XDG_RUNTIME_DIR:-/run/user/$(id -u)}/$DESKTOP_ID" +export XDG_CONFIG_HOME="$HOME/.config/$DESKTOP_ID" +export XDG_DATA_HOME="$HOME/.local/share/$DESKTOP_ID" +export XDG_CACHE_HOME="$HOME/.cache/$DESKTOP_ID" +export XDG_STATE_HOME="$HOME/.local/state/$DESKTOP_ID" +mkdir -p "$XDG_RUNTIME_DIR" "$XDG_CONFIG_HOME" "$XDG_DATA_HOME" "$XDG_CACHE_HOME" "$XDG_STATE_HOME" +chmod 0700 "$XDG_RUNTIME_DIR" + +# Disable KDE Wallet — it blocks browsers waiting for unlock in virtual sessions +cat > "$XDG_CONFIG_HOME/kwalletrc" << KWALLET +[Wallet] +Enabled=false +First Use=false +KWALLET + +# Chrome/Chromium ignore XDG — patch .desktop files and create wrappers +WRAPPER_DIR="$XDG_RUNTIME_DIR/bin" +DESKTOP_DIR="$XDG_DATA_HOME/applications" +mkdir -p "$WRAPPER_DIR" "$DESKTOP_DIR" + +for df in /usr/share/applications/google-chrome*.desktop /usr/share/applications/chromium*.desktop; do + [ -f "$df" ] || continue + REAL=$(grep -m1 "^Exec=" "$df" | sed 's/^Exec=//; s/ .*//') + BNAME=$(basename "$REAL") + DATA_DIR="$HOME/.config/$DESKTOP_ID/$BNAME" + cat > "$WRAPPER_DIR/$BNAME" << WRAPPER +#!/bin/bash +exec "$REAL" --user-data-dir="$DATA_DIR" --password-store=basic "\$@" +WRAPPER + chmod +x "$WRAPPER_DIR/$BNAME" + sed "s|Exec=$REAL|Exec=$WRAPPER_DIR/$BNAME|g" "$df" > "$DESKTOP_DIR/$(basename "$df")" +done +export PATH="$WRAPPER_DIR:$PATH" + +Xvfb :%d -screen 0 %sx24 & +sleep 1 + +# Detect window manager / desktop environment +WM="" +NEEDS_DBUS=false +RESOLVED="" +if command -v x-session-manager &>/dev/null; then + WM=x-session-manager + RESOLVED="$(readlink -f "$(command -v x-session-manager)" 2>/dev/null)" +elif command -v startplasma-x11 &>/dev/null; then + WM=startplasma-x11 + RESOLVED=startplasma-x11 +else + for wm in kwin_x11 openbox fluxbox i3 xfwm4 mutter kwin; do + if command -v "$wm" &>/dev/null; then WM="$wm"; break; fi + done +fi + +# Full desktop sessions need their own dbus +case "$RESOLVED" in + *startplasma*|*gnome-session*|*xfce4-session*) NEEDS_DBUS=true;; +esac + +if [ -n "$WM" ]; then + if $NEEDS_DBUS && command -v dbus-run-session &>/dev/null; then + dbus-run-session "$WM" & + else + "$WM" & + fi + sleep 3 +fi + +x11vnc -display :%d -rfbport %d -nopw -forever -shared -noxdamage +`, displayNum, displayNum, displayNum, resolution, displayNum, port) +} + // parseWindowIndex extracts and validates the window index from the URL. func parseWindowIndex(r *http.Request) (int, bool) { indexStr := chi.URLParam(r, "index") @@ -239,3 +464,95 @@ func (s *Server) handleWindowKeys(w http.ResponseWriter, r *http.Request) { writeJSON(w, http.StatusOK, map[string]bool{"ok": true}) } + +// handleWindowResolution changes the desktop resolution by restarting Xvfb + x11vnc. +func (s *Server) handleWindowResolution(w http.ResponseWriter, r *http.Request) { + session := chi.URLParam(r, "session") + if errMsg := validate.ValidateName(session, "Session name"); errMsg != "" { + writeError(w, http.StatusBadRequest, errMsg) + return + } + + index, ok := parseWindowIndex(r) + if !ok { + writeError(w, http.StatusBadRequest, "Invalid window index") + return + } + + var body struct { + Resolution string `json:"resolution"` + } + if err := json.NewDecoder(r.Body).Decode(&body); err != nil { + writeError(w, http.StatusBadRequest, "Invalid JSON body") + return + } + + if errMsg := validate.ValidateResolution(body.Resolution); errMsg != "" { + writeError(w, http.StatusBadRequest, errMsg) + return + } + + server := serverFromRequest(r) + + // Read existing VNC and websockify ports from window options + portStr, err := s.tmux.GetWindowOption(session, index, "@rk_vnc_port", server) + if err != nil { + writeError(w, http.StatusBadRequest, "Window is not a desktop window or VNC port not set") + return + } + port, err := strconv.Atoi(portStr) + if err != nil { + writeError(w, http.StatusInternalServerError, "Invalid VNC port value") + return + } + // Derive display number from port (same logic as creation) + displayNum := port - 5900 + if displayNum < 0 { + displayNum = port % 1000 + } + + // Send C-c to kill the running bash -c (which kills Xvfb, x11vnc, WM), + // then send the full startup script at the new resolution. + if err := s.tmux.SendKeys(session, index, "C-c", server); err != nil { + slog.Error("failed to send C-c", "err", err) + } + time.Sleep(1 * time.Second) + + script := desktopStartupScript(displayNum, port, body.Resolution) + scriptFile := fmt.Sprintf("/tmp/rk-desktop-%d.sh", port) + if err := os.WriteFile(scriptFile, []byte(script), 0700); err != nil { + writeError(w, http.StatusInternalServerError, "Failed to write startup script") + return + } + if err := s.tmux.SendKeys(session, index, scriptFile, server); err != nil { + writeError(w, http.StatusInternalServerError, err.Error()) + return + } + + writeJSON(w, http.StatusOK, map[string]bool{"ok": true}) +} + +// handleDesktopInfo returns the websockify port for a desktop window. +func (s *Server) handleDesktopInfo(w http.ResponseWriter, r *http.Request) { + session := chi.URLParam(r, "session") + if errMsg := validate.ValidateName(session, "Session name"); errMsg != "" { + writeError(w, http.StatusBadRequest, errMsg) + return + } + + index, ok := parseWindowIndex(r) + if !ok { + writeError(w, http.StatusBadRequest, "Invalid window index") + return + } + + server := serverFromRequest(r) + + wsPortStr, err := s.tmux.GetWindowOption(session, index, "@rk_ws_port", server) + if err != nil { + writeError(w, http.StatusNotFound, "Not a desktop window or websockify port not set") + return + } + + writeJSON(w, http.StatusOK, map[string]string{"wsPort": wsPortStr}) +} diff --git a/app/backend/api/windows_test.go b/app/backend/api/windows_test.go index 14c7ad59..440fb026 100644 --- a/app/backend/api/windows_test.go +++ b/app/backend/api/windows_test.go @@ -283,6 +283,138 @@ func TestWindowSplitInvalidSession(t *testing.T) { } } +func TestWindowCreateDesktop(t *testing.T) { + ops := &mockTmuxOps{ + listWindowsResult: []tmux.WindowInfo{ + {Index: 0, Name: "zsh", Type: "terminal", WorktreePath: "/home/user"}, + }, + } + router := newTestRouter(&mockSessionFetcher{}, ops) + + body := `{"name":"dev","type":"desktop"}` + req := httptest.NewRequest(http.MethodPost, "/api/sessions/run-kit/windows", strings.NewReader(body)) + req.Header.Set("Content-Type", "application/json") + rec := httptest.NewRecorder() + router.ServeHTTP(rec, req) + + if rec.Code != http.StatusCreated { + t.Errorf("status = %d, want %d; body = %s", rec.Code, http.StatusCreated, rec.Body.String()) + } + + if !ops.createWindowCalled { + t.Error("CreateWindow was not called") + } + // Desktop window should have desktop: prefix + if ops.createWindowName != "desktop:dev" { + t.Errorf("name = %q, want %q", ops.createWindowName, "desktop:dev") + } + // SendKeys should have been called with the startup script + if !ops.sendKeysCalled { + t.Error("SendKeys was not called (expected startup script)") + } +} + +func TestWindowCreateDesktopDefaultResolution(t *testing.T) { + ops := &mockTmuxOps{ + listWindowsResult: []tmux.WindowInfo{ + {Index: 0, Name: "zsh", Type: "terminal", WorktreePath: "/home/user"}, + {Index: 1, Name: "desktop:dev", Type: "desktop", WorktreePath: "/home/user"}, + }, + } + router := newTestRouter(&mockSessionFetcher{}, ops) + + body := `{"name":"test","type":"desktop"}` + req := httptest.NewRequest(http.MethodPost, "/api/sessions/run-kit/windows", strings.NewReader(body)) + req.Header.Set("Content-Type", "application/json") + rec := httptest.NewRecorder() + router.ServeHTTP(rec, req) + + if rec.Code != http.StatusCreated { + t.Errorf("status = %d, want %d; body = %s", rec.Code, http.StatusCreated, rec.Body.String()) + } + + // The startup script should contain the default resolution 1920x1080 + if !strings.Contains(ops.sendKeysKeys, "1920x1080") { + t.Errorf("startup script does not contain default resolution 1920x1080: %s", ops.sendKeysKeys) + } +} + +func TestWindowCreateDesktopCustomResolution(t *testing.T) { + ops := &mockTmuxOps{ + listWindowsResult: []tmux.WindowInfo{ + {Index: 0, Name: "zsh", Type: "terminal", WorktreePath: "/home/user"}, + {Index: 1, Name: "desktop:hires", Type: "desktop", WorktreePath: "/home/user"}, + }, + } + router := newTestRouter(&mockSessionFetcher{}, ops) + + body := `{"name":"hires","type":"desktop","resolution":"2560x1440"}` + req := httptest.NewRequest(http.MethodPost, "/api/sessions/run-kit/windows", strings.NewReader(body)) + req.Header.Set("Content-Type", "application/json") + rec := httptest.NewRecorder() + router.ServeHTTP(rec, req) + + if rec.Code != http.StatusCreated { + t.Errorf("status = %d, want %d; body = %s", rec.Code, http.StatusCreated, rec.Body.String()) + } + + if !strings.Contains(ops.sendKeysKeys, "2560x1440") { + t.Errorf("startup script does not contain custom resolution 2560x1440: %s", ops.sendKeysKeys) + } +} + +func TestWindowCreateDesktopInvalidResolution(t *testing.T) { + router := newTestRouter(&mockSessionFetcher{}, &mockTmuxOps{}) + + body := `{"name":"bad","type":"desktop","resolution":"foo; rm -rf /"}` + req := httptest.NewRequest(http.MethodPost, "/api/sessions/run-kit/windows", strings.NewReader(body)) + req.Header.Set("Content-Type", "application/json") + rec := httptest.NewRecorder() + router.ServeHTTP(rec, req) + + if rec.Code != http.StatusBadRequest { + t.Errorf("status = %d, want %d", rec.Code, http.StatusBadRequest) + } +} + +func TestWindowResolutionChange(t *testing.T) { + ops := &mockTmuxOps{ + getWindowOptionResult: "59234", + } + router := newTestRouter(&mockSessionFetcher{}, ops) + + body := `{"resolution":"2560x1440"}` + req := httptest.NewRequest(http.MethodPost, "/api/sessions/run-kit/windows/1/resolution", strings.NewReader(body)) + req.Header.Set("Content-Type", "application/json") + rec := httptest.NewRecorder() + router.ServeHTTP(rec, req) + + if rec.Code != http.StatusOK { + t.Errorf("status = %d, want %d; body = %s", rec.Code, http.StatusOK, rec.Body.String()) + } + + if !ops.sendKeysCalled { + t.Error("SendKeys was not called (expected restart script)") + } + if !strings.Contains(ops.sendKeysKeys, "2560x1440") { + t.Errorf("restart script does not contain new resolution: %s", ops.sendKeysKeys) + } +} + +func TestWindowResolutionChangeInvalidResolution(t *testing.T) { + router := newTestRouter(&mockSessionFetcher{}, &mockTmuxOps{}) + + body := `{"resolution":"bad"}` + req := httptest.NewRequest(http.MethodPost, "/api/sessions/run-kit/windows/1/resolution", strings.NewReader(body)) + req.Header.Set("Content-Type", "application/json") + rec := httptest.NewRecorder() + router.ServeHTTP(rec, req) + + if rec.Code != http.StatusBadRequest { + t.Errorf("status = %d, want %d", rec.Code, http.StatusBadRequest) + } +} + func TestWindowSplitInvalidJSON(t *testing.T) { router := newTestRouter(&mockSessionFetcher{}, &mockTmuxOps{}) diff --git a/app/backend/internal/daemon/daemon.go b/app/backend/internal/daemon/daemon.go index 96ce8c19..c145d694 100644 --- a/app/backend/internal/daemon/daemon.go +++ b/app/backend/internal/daemon/daemon.go @@ -133,11 +133,11 @@ func Stop() error { select { case <-ctx.Done(): // Timeout — kill forcefully with a fresh short context. + // kill-session may return an error if the tmux server exits when + // the last session is killed — that's still a successful stop. killCtx, killCancel := context.WithTimeout(context.Background(), 2*time.Second) defer killCancel() - if err := runTmux(killCtx, "kill-session", "-t", SessionName); err != nil { - return fmt.Errorf("killing daemon session after timeout: %w", err) - } + _ = runTmux(killCtx, "kill-session", "-t", SessionName) return nil case <-time.After(stopPollInterval): if !isRunningCtx(ctx) { diff --git a/app/backend/internal/tmux/tmux.go b/app/backend/internal/tmux/tmux.go index 254b741d..e665efbe 100644 --- a/app/backend/internal/tmux/tmux.go +++ b/app/backend/internal/tmux/tmux.go @@ -146,6 +146,7 @@ const ( type WindowInfo struct { Index int `json:"index"` Name string `json:"name"` + Type string `json:"type"` WorktreePath string `json:"worktreePath"` Activity string `json:"activity"` // "active" or "idle" IsActiveWindow bool `json:"isActiveWindow"` @@ -272,9 +273,15 @@ func parseWindows(lines []string, nowUnix int64) []WindowInfo { isActive := strings.TrimSpace(parts[4]) == "1" paneCmd := strings.TrimSpace(parts[5]) + winType := "terminal" + if strings.HasPrefix(parts[1], "desktop:") { + winType = "desktop" + } + windows = append(windows, WindowInfo{ Index: index, Name: parts[1], + Type: winType, WorktreePath: parts[2], Activity: activity, IsActiveWindow: isActive, @@ -592,3 +599,30 @@ func KillServer(server string) error { } return err } + +// GetWindowOption reads a tmux user window option (e.g. @rk_vnc_port) from a specific window. +// Returns the value and nil on success, or empty string and error if not set or command fails. +func GetWindowOption(session string, windowIndex int, key string, server string) (string, error) { + ctx, cancel := withTimeout() + defer cancel() + + target := fmt.Sprintf("%s:%d", session, windowIndex) + lines, err := tmuxExecServer(ctx, server, "show-options", "-wv", "-t", target, key) + if err != nil { + return "", err + } + if len(lines) == 0 { + return "", fmt.Errorf("window option %s not set", key) + } + return strings.TrimSpace(lines[0]), nil +} + +// SetWindowOption sets a tmux user window option (e.g. @rk_vnc_port) on a specific window. +func SetWindowOption(session string, windowIndex int, key, value string, server string) error { + ctx, cancel := withTimeout() + defer cancel() + + target := fmt.Sprintf("%s:%d", session, windowIndex) + _, err := tmuxExecServer(ctx, server, "set-option", "-w", "-t", target, key, value) + return err +} diff --git a/app/backend/internal/tmux/tmux_test.go b/app/backend/internal/tmux/tmux_test.go index 5b907981..cdee956c 100644 --- a/app/backend/internal/tmux/tmux_test.go +++ b/app/backend/internal/tmux/tmux_test.go @@ -144,7 +144,7 @@ func TestParseWindows(t *testing.T) { }, now: fakeNow, want: []WindowInfo{ - {Index: 0, Name: "dev", WorktreePath: "/home/user/project", Activity: "active", IsActiveWindow: true, PaneCommand: "claude", ActivityTimestamp: fakeNow - 1}, + {Index: 0, Name: "dev", Type: "terminal", WorktreePath: "/home/user/project", Activity: "active", IsActiveWindow: true, PaneCommand: "claude", ActivityTimestamp: fakeNow - 1}, }, }, { @@ -154,7 +154,7 @@ func TestParseWindows(t *testing.T) { }, now: fakeNow, want: []WindowInfo{ - {Index: 0, Name: "dev", WorktreePath: "/home/user/project", Activity: "idle", IsActiveWindow: false, PaneCommand: "zsh", ActivityTimestamp: fakeNow - ActivityThresholdSeconds - 100}, + {Index: 0, Name: "dev", Type: "terminal", WorktreePath: "/home/user/project", Activity: "idle", IsActiveWindow: false, PaneCommand: "zsh", ActivityTimestamp: fakeNow - ActivityThresholdSeconds - 100}, }, }, { @@ -165,8 +165,8 @@ func TestParseWindows(t *testing.T) { }, now: fakeNow, want: []WindowInfo{ - {Index: 0, Name: "dev", WorktreePath: "/home/user/project", Activity: "active", IsActiveWindow: true, PaneCommand: "claude", ActivityTimestamp: fakeNow}, - {Index: 2, Name: "build", WorktreePath: "/tmp/build", Activity: "active", IsActiveWindow: false, PaneCommand: "make", ActivityTimestamp: fakeNow}, + {Index: 0, Name: "dev", Type: "terminal", WorktreePath: "/home/user/project", Activity: "active", IsActiveWindow: true, PaneCommand: "claude", ActivityTimestamp: fakeNow}, + {Index: 2, Name: "build", Type: "terminal", WorktreePath: "/tmp/build", Activity: "active", IsActiveWindow: false, PaneCommand: "make", ActivityTimestamp: fakeNow}, }, }, { @@ -183,7 +183,7 @@ func TestParseWindows(t *testing.T) { }, now: fakeNow, want: []WindowInfo{ - {Index: 1, Name: "good", WorktreePath: "/home/user", Activity: "active", IsActiveWindow: true, PaneCommand: "zsh", ActivityTimestamp: fakeNow}, + {Index: 1, Name: "good", Type: "terminal", WorktreePath: "/home/user", Activity: "active", IsActiveWindow: true, PaneCommand: "zsh", ActivityTimestamp: fakeNow}, }, }, { @@ -193,7 +193,7 @@ func TestParseWindows(t *testing.T) { }, now: fakeNow, want: []WindowInfo{ - {Index: 0, Name: "edge", WorktreePath: "/path", Activity: "active", IsActiveWindow: false, PaneCommand: "bash", ActivityTimestamp: fakeNow - ActivityThresholdSeconds}, + {Index: 0, Name: "edge", Type: "terminal", WorktreePath: "/path", Activity: "active", IsActiveWindow: false, PaneCommand: "bash", ActivityTimestamp: fakeNow - ActivityThresholdSeconds}, }, }, { @@ -203,7 +203,7 @@ func TestParseWindows(t *testing.T) { }, now: fakeNow, want: []WindowInfo{ - {Index: 0, Name: "past", WorktreePath: "/path", Activity: "idle", IsActiveWindow: false, PaneCommand: "vim", ActivityTimestamp: fakeNow - ActivityThresholdSeconds - 1}, + {Index: 0, Name: "past", Type: "terminal", WorktreePath: "/path", Activity: "idle", IsActiveWindow: false, PaneCommand: "vim", ActivityTimestamp: fakeNow - ActivityThresholdSeconds - 1}, }, }, { @@ -213,7 +213,7 @@ func TestParseWindows(t *testing.T) { }, now: fakeNow, want: []WindowInfo{ - {Index: 0, Name: "work", WorktreePath: "/home/user/code", Activity: "active", IsActiveWindow: true, PaneCommand: "node", ActivityTimestamp: fakeNow}, + {Index: 0, Name: "work", Type: "terminal", WorktreePath: "/home/user/code", Activity: "active", IsActiveWindow: true, PaneCommand: "node", ActivityTimestamp: fakeNow}, }, }, { @@ -223,7 +223,7 @@ func TestParseWindows(t *testing.T) { }, now: 1710300100, want: []WindowInfo{ - {Index: 0, Name: "ts", WorktreePath: "/path", Activity: "idle", IsActiveWindow: false, PaneCommand: "zsh", ActivityTimestamp: 1710300000}, + {Index: 0, Name: "ts", Type: "terminal", WorktreePath: "/path", Activity: "idle", IsActiveWindow: false, PaneCommand: "zsh", ActivityTimestamp: 1710300000}, }, }, } @@ -247,6 +247,9 @@ func TestParseWindows(t *testing.T) { if got[i].Name != tt.want[i].Name { t.Errorf("window[%d].Name = %q, want %q", i, got[i].Name, tt.want[i].Name) } + if got[i].Type != tt.want[i].Type { + t.Errorf("window[%d].Type = %q, want %q", i, got[i].Type, tt.want[i].Type) + } if got[i].WorktreePath != tt.want[i].WorktreePath { t.Errorf("window[%d].WorktreePath = %q, want %q", i, got[i].WorktreePath, tt.want[i].WorktreePath) } @@ -267,6 +270,37 @@ func TestParseWindows(t *testing.T) { } } +func TestParseWindowsDesktopType(t *testing.T) { + const fakeNow int64 = 1700000000 + + tests := []struct { + name string + winName string + wantType string + }{ + {"desktop prefix sets type desktop", "desktop:dev", "desktop"}, + {"desktop prefix with suffix", "desktop:my-env", "desktop"}, + {"plain name sets type terminal", "zsh", "terminal"}, + {"no prefix sets type terminal", "build", "terminal"}, + {"desktop in middle is terminal", "my-desktop-win", "terminal"}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + lines := []string{ + windowLine(0, tt.winName, "/home/user", fakeNow, 1, "bash"), + } + got := parseWindows(lines, fakeNow) + if len(got) != 1 { + t.Fatalf("expected 1 window, got %d", len(got)) + } + if got[0].Type != tt.wantType { + t.Errorf("Type = %q, want %q", got[0].Type, tt.wantType) + } + }) + } +} + func TestSanitizeEnv(t *testing.T) { tests := []struct { name string diff --git a/app/backend/internal/validate/validate.go b/app/backend/internal/validate/validate.go index 13891a52..ddee8073 100644 --- a/app/backend/internal/validate/validate.go +++ b/app/backend/internal/validate/validate.go @@ -26,8 +26,13 @@ func ValidateName(name, label string) string { if forbiddenChars.MatchString(name) { return fmt.Sprintf("%s contains forbidden characters", label) } - if strings.Contains(name, ":") || strings.Contains(name, ".") { - return fmt.Sprintf("%s cannot contain colons or periods", label) + if strings.Contains(name, ".") { + return fmt.Sprintf("%s cannot contain periods", label) + } + // Allow colon only in "desktop:" prefix (used for desktop window type detection). + // Bare colons elsewhere are forbidden because tmux uses : as session:window separator. + if strings.Contains(name, ":") && !strings.HasPrefix(name, "desktop:") { + return fmt.Sprintf("%s cannot contain colons (except desktop: prefix)", label) } return "" } @@ -104,6 +109,21 @@ func ValidatePath(path, label string) string { return "" } +// resolutionPattern matches valid resolution strings: {width}x{height} with 3-5 digit numbers. +var resolutionPattern = regexp.MustCompile(`^\d{3,5}x\d{3,5}$`) + +// ValidateResolution validates a display resolution string (e.g. "1920x1080"). +// Returns empty string if valid, error message if invalid. +func ValidateResolution(res string) string { + if res == "" { + return "Resolution cannot be empty" + } + if !resolutionPattern.MatchString(res) { + return "Resolution must be in WIDTHxHEIGHT format (e.g. 1920x1080)" + } + return "" +} + // SanitizeFilename sanitizes a user-provided filename for safe disk storage. func SanitizeFilename(name string) string { // Strip null bytes diff --git a/app/backend/internal/validate/validate_test.go b/app/backend/internal/validate/validate_test.go index 633cb695..13f5fb97 100644 --- a/app/backend/internal/validate/validate_test.go +++ b/app/backend/internal/validate/validate_test.go @@ -151,6 +151,45 @@ func TestExpandTilde(t *testing.T) { } } +func TestValidateResolution(t *testing.T) { + tests := []struct { + name string + input string + wantErr bool + }{ + {"valid 1920x1080", "1920x1080", false}, + {"valid 800x600", "800x600", false}, + {"valid 2560x1440", "2560x1440", false}, + {"valid 1280x720", "1280x720", false}, + {"valid small 320x240", "320x240", false}, + {"valid 5 digits", "10000x10000", false}, + {"empty string", "", true}, + {"just text", "foo", true}, + {"shell injection", "1920x1080; rm -rf /", true}, + {"missing height", "1920x", true}, + {"missing width", "x1080", true}, + {"too few digits width", "12x1080", true}, + {"too few digits height", "1920x10", true}, + {"too many digits", "123456x1080", true}, + {"float resolution", "1920.5x1080", true}, + {"negative", "-1920x1080", true}, + {"spaces", "1920 x 1080", true}, + {"uppercase X", "1920X1080", true}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result := ValidateResolution(tt.input) + if tt.wantErr && result == "" { + t.Error("expected error but got none") + } + if !tt.wantErr && result != "" { + t.Errorf("expected no error but got: %s", result) + } + }) + } +} + func TestSanitizeFilename(t *testing.T) { tests := []struct { name string diff --git a/app/frontend/package.json b/app/frontend/package.json index 7f16a736..a8b7cffb 100644 --- a/app/frontend/package.json +++ b/app/frontend/package.json @@ -12,6 +12,7 @@ "test:e2e": "playwright test" }, "dependencies": { + "@novnc/novnc": "^1.5.0", "@tanstack/react-router": "^1.114.0", "@xterm/addon-clipboard": "^0.2.0", "@xterm/addon-fit": "^0.10.0", diff --git a/app/frontend/pnpm-lock.yaml b/app/frontend/pnpm-lock.yaml index 03a2de41..4921422b 100644 --- a/app/frontend/pnpm-lock.yaml +++ b/app/frontend/pnpm-lock.yaml @@ -8,6 +8,9 @@ importers: .: dependencies: + '@novnc/novnc': + specifier: ^1.5.0 + version: 1.5.0 '@tanstack/react-router': specifier: ^1.114.0 version: 1.166.6(react-dom@19.2.4(react@19.2.4))(react@19.2.4) @@ -1117,6 +1120,9 @@ packages: resolution: {integrity: sha512-cXu86tF4VQVfwz8W1SPbhoRyHJkti6mjH/XJIxp40jhO4j2k1m4KYrEykxqWPkFF3vrK4rgQppBh//AwyGSXPA==} engines: {node: '>=18'} + '@novnc/novnc@1.5.0': + resolution: {integrity: sha512-4yGHOtUCnEJUCsgEt/L78eeJu00kthurLBWXFiaXfonNx0pzbs6R/3gJb1byZe6iAE8V9MF0syQb0xIL8MSOtQ==} + '@open-draft/deferred-promise@2.2.0': resolution: {integrity: sha512-CecwLWx3rhxVQF6V4bAgPS5t+So2sTbPgAzafKkVizyi7tlwpcFpdFqq+wqF2OwNBmqFuu6tOyouTuxgpMfzmA==} @@ -4040,6 +4046,8 @@ snapshots: outvariant: 1.4.3 strict-event-emitter: 0.5.1 + '@novnc/novnc@1.5.0': {} + '@open-draft/deferred-promise@2.2.0': {} '@open-draft/logger@0.3.0': diff --git a/app/frontend/src/api/client.ts b/app/frontend/src/api/client.ts index 405b0764..4a8f219b 100644 --- a/app/frontend/src/api/client.ts +++ b/app/frontend/src/api/client.ts @@ -109,6 +109,42 @@ export async function createWindow( return res.json(); } +export async function createDesktopWindow( + session: string, + name?: string, + resolution?: string, +): Promise<{ ok: boolean }> { + const body: Record = { + name: name ?? "desktop", + type: "desktop", + }; + if (resolution) body.resolution = resolution; + const res = await fetch(withServer(`/api/sessions/${encodeURIComponent(session)}/windows`), { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify(body), + }); + if (!res.ok) await throwOnError(res); + return res.json(); +} + +export async function changeDesktopResolution( + session: string, + windowIndex: number, + resolution: string, +): Promise<{ ok: boolean }> { + const res = await fetch( + withServer(`/api/sessions/${encodeURIComponent(session)}/windows/${windowIndex}/resolution`), + { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ resolution }), + }, + ); + if (!res.ok) await throwOnError(res); + return res.json(); +} + export async function killWindow( session: string, index: number, diff --git a/app/frontend/src/app.tsx b/app/frontend/src/app.tsx index d5f4158f..80b6c169 100644 --- a/app/frontend/src/app.tsx +++ b/app/frontend/src/app.tsx @@ -8,13 +8,15 @@ import { useDialogState } from "@/hooks/use-dialog-state"; import { TopBar } from "@/components/top-bar"; import { Sidebar } from "@/components/sidebar"; import { TerminalClient } from "@/components/terminal-client"; +import { DesktopClient, type TouchMode } from "@/components/desktop-client"; import { BottomBar } from "@/components/bottom-bar"; +import { DesktopBottomBar } from "@/components/desktop-bottom-bar"; import type { PaletteAction } from "@/components/command-palette"; import { Dialog } from "@/components/dialog"; import { Dashboard } from "@/components/dashboard"; import { KeyboardShortcuts } from "@/components/keyboard-shortcuts"; -import { selectWindow, createWindow, splitWindow, closePane, reloadTmuxConfig, initTmuxConf, getHealth, createServer, killServer as killServerApi } from "@/api/client"; +import { selectWindow, createWindow, createDesktopWindow, changeDesktopResolution, splitWindow, closePane, reloadTmuxConfig, initTmuxConf, getHealth, createServer, killServer as killServerApi } from "@/api/client"; import { useSessionContext } from "@/contexts/session-context"; import { useBrowserTitle } from "@/hooks/use-browser-title"; @@ -107,10 +109,18 @@ function AppShell() { const [composeOpen, setComposeOpen] = useState(false); const [scrollLocked, setScrollLocked] = useState(false); const [hostname, setHostname] = useState(""); + const rfbInstanceRef = useRef(null); const [showCreateServerDialog, setShowCreateServerDialog] = useState(false); const [createServerName, setCreateServerName] = useState(""); const [showKillServerConfirm, setShowKillServerConfirm] = useState(false); const [showKeyboardShortcuts, setShowKeyboardShortcuts] = useState(false); + const [touchMode, setTouchMode] = useState(() => { + try { + const stored = localStorage.getItem("rk-desktop-touch-mode"); + if (stored === "direct" || stored === "trackpad") return stored; + } catch { /* noop */ } + return "direct"; + }); // Fetch hostname once on mount (guarded for StrictMode double-invoke) const didFetchHostnameRef = useRef(false); @@ -210,13 +220,18 @@ function AppShell() { return currentSession.windows.find((w) => w.isActiveWindow) ?? null; }, [currentSession]); + // Track whether the current window is a desktop (via ref to avoid effect deps) + const isDesktopRef = useRef(false); + isDesktopRef.current = currentWindow?.type === "desktop"; + useEffect(() => { - if (!activeWindow || !sessionName) return; + if (!activeWindow || !sessionName || !windowIndex) return; if (String(activeWindow.index) !== windowIndex) { - // Skip if user recently navigated (e.g. clicked sidebar) or a dialog is open if (dialogOpenRef.current) return; const elapsed = Date.now() - userNavTimestampRef.current; if (elapsed < 3000) return; + // Desktop windows connect via VNC, not tmux attach — skip activeWindow sync + if (isDesktopRef.current) return; navigate({ to: "/$server/$session/$window", params: { server, session: sessionName, window: String(activeWindow.index) }, @@ -281,6 +296,18 @@ function AppShell() { [], ); + // Create a desktop window in a session + const handleCreateDesktopWindow = useCallback( + async (session: string) => { + try { + await createDesktopWindow(session); + } catch { + // SSE will reflect + } + }, + [], + ); + // Theme const { preference: themePreference, resolved: themeResolved, themeDark, themeLight } = useTheme(); const { setTheme } = useThemeActions(); @@ -383,6 +410,13 @@ function AppShell() { if (sessionName) handleCreateWindow(sessionName); }, }, + { + id: "create-desktop", + label: "New Desktop Window", + onSelect: () => { + if (sessionName) handleCreateDesktopWindow(sessionName); + }, + }, ] : []), ...(currentWindow @@ -434,12 +468,12 @@ function AppShell() { ] : []), ], - [sessionName, currentWindow, handleCreateWindow, dialogs], + [sessionName, currentWindow, handleCreateWindow, handleCreateDesktopWindow, dialogs], ); const viewActions: PaletteAction[] = useMemo( () => [ - ...(sessionName + ...(sessionName && currentWindow?.type !== "desktop" ? [ { id: "text-input", @@ -448,13 +482,23 @@ function AppShell() { }, ] : []), + ...(sessionName && currentWindow?.type === "desktop" + ? [ + { id: "resolution-720x1280", label: "Desktop resolution: 720x1280 (portrait)", onSelect: () => changeDesktopResolution(sessionName, currentWindow.index, "720x1280").catch(() => {}) }, + { id: "resolution-1080x1920", label: "Desktop resolution: 1080x1920 (portrait)", onSelect: () => changeDesktopResolution(sessionName, currentWindow.index, "1080x1920").catch(() => {}) }, + { id: "resolution-1440x2560", label: "Desktop resolution: 1440x2560 (portrait)", onSelect: () => changeDesktopResolution(sessionName, currentWindow.index, "1440x2560").catch(() => {}) }, + { id: "resolution-1280x720", label: "Desktop resolution: 1280x720 (landscape)", onSelect: () => changeDesktopResolution(sessionName, currentWindow.index, "1280x720").catch(() => {}) }, + { id: "resolution-1920x1080", label: "Desktop resolution: 1920x1080 (landscape)", onSelect: () => changeDesktopResolution(sessionName, currentWindow.index, "1920x1080").catch(() => {}) }, + { id: "resolution-2560x1440", label: "Desktop resolution: 2560x1440 (landscape)", onSelect: () => changeDesktopResolution(sessionName, currentWindow.index, "2560x1440").catch(() => {}) }, + ] + : []), { id: "toggle-fixed-width", label: fixedWidth ? "View: Full Width" : "View: Fixed Width (900px)", onSelect: toggleFixedWidth, }, ], - [sessionName, fixedWidth, toggleFixedWidth], + [sessionName, currentWindow, fixedWidth, toggleFixedWidth], ); const configActions: PaletteAction[] = useMemo( @@ -540,6 +584,7 @@ function AppShell() { onToggleDrawer={() => setDrawerOpen(!drawerOpen)} onCreateSession={dialogs.openCreateDialog} onCreateWindow={handleCreateWindow} + onCreateDesktopWindow={handleCreateDesktopWindow} onOpenCompose={() => setComposeOpen((v) => !v)} /> @@ -589,31 +634,63 @@ function AppShell() { style={fixedWidth ? { maxWidth: 900, width: "100%", marginInline: "auto" } : undefined} > {sessionName && windowIndex ? ( - <> -
- navigate({ to: "/$server", params: { server }, replace: true })} - focusRef={focusTerminalRef} - scrollLocked={scrollLocked} - /> -
- {/* Bottom Bar — only on terminal pages */} -
- setComposeOpen((v) => !v)} onFocusTerminal={() => focusTerminalRef.current?.()} onScrollLockChange={setScrollLocked} /> + currentWindow?.type === "desktop" ? ( + <> +
+ navigate({ to: "/$server", params: { server }, replace: true })} + onRfbRef={(rfb) => { rfbInstanceRef.current = rfb; }} + /> +
+
+ +
+ + ) : currentWindow?.type === "terminal" ? ( + <> +
+ navigate({ to: "/$server", params: { server }, replace: true })} + focusRef={focusTerminalRef} + scrollLocked={scrollLocked} + /> +
+ {/* Bottom Bar — only on terminal pages */} +
+ setComposeOpen((v) => !v)} onFocusTerminal={() => focusTerminalRef.current?.()} onScrollLockChange={setScrollLocked} /> +
+ + ) : ( + /* Window type not yet known (waiting for SSE) — render nothing to avoid + TerminalClient connecting to a desktop window's relay */ +
+ Connecting...
- + ) ) : ( )}
diff --git a/app/frontend/src/components/breadcrumb-dropdown.tsx b/app/frontend/src/components/breadcrumb-dropdown.tsx index 60b535c3..8cbbdceb 100644 --- a/app/frontend/src/components/breadcrumb-dropdown.tsx +++ b/app/frontend/src/components/breadcrumb-dropdown.tsx @@ -7,24 +7,26 @@ type Props = { icon?: string; onNavigate?: (href: string) => void; action?: { label: string; onAction: () => void }; + actions?: { label: string; onAction: () => void }[]; triggerClassName?: string; }; -export function BreadcrumbDropdown({ items, label, icon, onNavigate, action, triggerClassName }: Props) { +export function BreadcrumbDropdown({ items, label, icon, onNavigate, action, actions, triggerClassName }: Props) { const [open, setOpen] = useState(false); const [focusedIndex, setFocusedIndex] = useState(-1); const containerRef = useRef(null); const buttonRef = useRef(null); - const actionRef = useRef(null); const itemRefs = useRef<(HTMLButtonElement | null)[]>([]); - // When action exists, index 0 = action button, indices 1..N = items. - // When no action, indices 0..N-1 = items directly. - const offset = action ? 1 : 0; + // Merge single action and actions array into one list for offset calculation. + const allActions = actions ?? (action ? [action] : []); + const offset = allActions.length; const totalCount = items.length + offset; + const actionRefs = useRef<(HTMLButtonElement | null)[]>([]); + function getFocusableRef(index: number): HTMLButtonElement | null { - if (action && index === 0) return actionRef.current; + if (index < offset) return actionRefs.current[index] ?? null; return itemRefs.current[index - offset] ?? null; } @@ -103,21 +105,24 @@ export function BreadcrumbDropdown({ items, label, icon, onNavigate, action, tri aria-label={label ? `Switch ${label}` : "Switch"} className="absolute top-full left-0 mt-1 bg-bg-primary border border-border rounded-lg shadow-2xl py-1 min-w-[160px] max-w-[240px] z-50" > - {action && ( + {allActions.length > 0 && ( <> - + {allActions.map((act, ai) => ( + + ))}
)} diff --git a/app/frontend/src/components/dashboard.test.tsx b/app/frontend/src/components/dashboard.test.tsx index 1ccd0acc..36d5e0ef 100644 --- a/app/frontend/src/components/dashboard.test.tsx +++ b/app/frontend/src/components/dashboard.test.tsx @@ -12,6 +12,7 @@ const sessions: ProjectSession[] = [ { index: 0, name: "main", + type: "terminal", worktreePath: "~/code/run-kit", activity: "active", isActiveWindow: true, @@ -24,6 +25,7 @@ const sessions: ProjectSession[] = [ { index: 1, name: "scratch", + type: "terminal", worktreePath: "~/code/run-kit", activity: "idle", isActiveWindow: false, @@ -38,6 +40,7 @@ const sessions: ProjectSession[] = [ { index: 0, name: "dev", + type: "terminal", worktreePath: "~/code/ao-server", activity: "idle", isActiveWindow: true, diff --git a/app/frontend/src/components/dashboard.tsx b/app/frontend/src/components/dashboard.tsx index 0fc68cb0..45a180fb 100644 --- a/app/frontend/src/components/dashboard.tsx +++ b/app/frontend/src/components/dashboard.tsx @@ -7,6 +7,7 @@ type DashboardProps = { onNavigate: (session: string, windowIndex: number) => void; onCreateSession: () => void; onCreateWindow: (session: string) => void; + onCreateDesktopWindow?: (session: string) => void; }; export function Dashboard({ @@ -14,6 +15,7 @@ export function Dashboard({ onNavigate, onCreateSession, onCreateWindow, + onCreateDesktopWindow, }: DashboardProps) { const [expanded, setExpanded] = useState>({}); @@ -101,6 +103,11 @@ export function Dashboard({ {win.name} + {win.type === "desktop" && ( + + Desktop + + )} {win.fabStage && ( {win.fabStage} @@ -141,16 +148,27 @@ export function Dashboard({ ); })} - {/* New Window button */} - + {/* New Window buttons */} +
+ + +
)} diff --git a/app/frontend/src/components/desktop-bottom-bar.tsx b/app/frontend/src/components/desktop-bottom-bar.tsx new file mode 100644 index 00000000..01d898c0 --- /dev/null +++ b/app/frontend/src/components/desktop-bottom-bar.tsx @@ -0,0 +1,297 @@ +import { useState, useCallback, useRef, useEffect } from "react"; +import { useModifierState } from "@/hooks/use-modifier-state"; +import { ArrowPad } from "@/components/arrow-pad"; +import { changeDesktopResolution } from "@/api/client"; +import type { TouchMode } from "@/components/desktop-client"; + +type DesktopBottomBarProps = { + rfbRef: React.RefObject; + sessionName: string; + windowIndex: number; + hostname?: string; + touchMode: TouchMode; + onTouchModeChange: (mode: TouchMode) => void; +}; + +const KEYSYM = { + Escape: 0xff1b, Tab: 0xff09, + Ctrl: 0xffe3, Alt: 0xffe9, + Left: 0xff51, Up: 0xff52, Right: 0xff53, Down: 0xff54, + F1: 0xffbe, F2: 0xffbf, F3: 0xffc0, F4: 0xffc1, F5: 0xffc2, F6: 0xffc3, + F7: 0xffc4, F8: 0xffc5, F9: 0xffc6, F10: 0xffc7, F11: 0xffc8, F12: 0xffc9, + PgUp: 0xff55, PgDn: 0xff56, Home: 0xff50, End: 0xff57, Ins: 0xff63, Del: 0xffff, +} as const; + +const FN_KEYS = [ + { label: "F1", sym: KEYSYM.F1 }, { label: "F2", sym: KEYSYM.F2 }, { label: "F3", sym: KEYSYM.F3 }, { label: "F4", sym: KEYSYM.F4 }, + { label: "F5", sym: KEYSYM.F5 }, { label: "F6", sym: KEYSYM.F6 }, { label: "F7", sym: KEYSYM.F7 }, { label: "F8", sym: KEYSYM.F8 }, + { label: "F9", sym: KEYSYM.F9 }, { label: "F10", sym: KEYSYM.F10 }, { label: "F11", sym: KEYSYM.F11 }, { label: "F12", sym: KEYSYM.F12 }, +]; + +const EXT_KEYS = [ + { label: "PgUp", sym: KEYSYM.PgUp }, { label: "PgDn", sym: KEYSYM.PgDn }, + { label: "Home", sym: KEYSYM.Home }, { label: "End", sym: KEYSYM.End }, + { label: "Ins", sym: KEYSYM.Ins }, { label: "Del", sym: KEYSYM.Del }, +]; + +const KBD_CLASS = + "min-h-[36px] min-w-[36px] flex items-center justify-center px-1 py-0 text-xs border border-border rounded select-none transition-colors hover:border-text-secondary active:bg-bg-card outline-none"; + +const MODIFIER_LABELS: Record = { ctrl: "Control", alt: "Option" }; + +const RESOLUTIONS = [ + { label: "720\u00D71280 (portrait)", value: "720x1280" }, + { label: "1080\u00D71920 (portrait)", value: "1080x1920" }, + { label: "1280\u00D7720", value: "1280x720" }, + { label: "1920\u00D71080", value: "1920x1080" }, + { label: "2560\u00D71440", value: "2560x1440" }, +]; + +const preventFocusSteal = (e: React.MouseEvent) => e.preventDefault(); + +export function DesktopBottomBar({ rfbRef, sessionName, windowIndex, hostname, touchMode, onTouchModeChange }: DesktopBottomBarProps) { + const mods = useModifierState(); + const [menuOpen, setMenuOpen] = useState(false); + const [fnOpen, setFnOpen] = useState(false); + const [kbdActive, setKbdActive] = useState(false); + const menuRef = useRef(null); + const fnRef = useRef(null); + const kbdInputRef = useRef(null); + + // Close popups on outside click / escape + useEffect(() => { + if (!menuOpen && !fnOpen) return; + const handleClick = (e: MouseEvent) => { + if (menuOpen && menuRef.current && !menuRef.current.contains(e.target as Node)) setMenuOpen(false); + if (fnOpen && fnRef.current && !fnRef.current.contains(e.target as Node)) setFnOpen(false); + }; + const handleKey = (e: KeyboardEvent) => { if (e.key === "Escape") { setMenuOpen(false); setFnOpen(false); } }; + document.addEventListener("mousedown", handleClick); + document.addEventListener("keydown", handleKey); + return () => { document.removeEventListener("mousedown", handleClick); document.removeEventListener("keydown", handleKey); }; + }, [menuOpen, fnOpen]); + + // Send keysym to noVNC with modifier state + const sendKey = useCallback((keysym: number) => { + const rfb = rfbRef.current; + if (!rfb || !("sendKey" in rfb)) return; + const send = (rfb as unknown as { sendKey: (k: number, c: string | null, d?: boolean) => void }).sendKey; + const snapshot = mods.consume(); + if (snapshot.ctrl) send.call(rfb, KEYSYM.Ctrl, null, true); + if (snapshot.alt) send.call(rfb, KEYSYM.Alt, null, true); + send.call(rfb, keysym, null, true); + send.call(rfb, keysym, null, false); + if (snapshot.alt) send.call(rfb, KEYSYM.Alt, null, false); + if (snapshot.ctrl) send.call(rfb, KEYSYM.Ctrl, null, false); + }, [rfbRef, mods]); + + const sendArrow = useCallback((code: string) => { + const map: Record = { A: KEYSYM.Up, B: KEYSYM.Down, C: KEYSYM.Right, D: KEYSYM.Left }; + if (map[code]) sendKey(map[code]); + }, [sendKey]); + + // Hidden textarea handlers for mobile keyboard + const handleKbdKeyDown = useCallback((e: React.KeyboardEvent) => { + if (e.key.length > 1) { + const map: Record = { + Enter: 0xff0d, Backspace: 0xff08, Tab: 0xff09, Escape: 0xff1b, + Delete: 0xffff, ArrowLeft: 0xff51, ArrowUp: 0xff52, ArrowRight: 0xff53, ArrowDown: 0xff54, + }; + if (map[e.key]) { e.preventDefault(); sendKey(map[e.key]); } + } + }, [sendKey]); + + const handleKbdTextInput = useCallback((e: React.FormEvent) => { + const text = e.currentTarget.value; + if (!text) return; + for (const char of text) sendKey(char.charCodeAt(0)); + e.currentTarget.value = ""; + }, [sendKey]); + + // Desktop menu actions + const handleClipboardPaste = useCallback(async () => { + try { const text = await navigator.clipboard.readText(); if (text && rfbRef.current) rfbRef.current.clipboardPasteFrom = text; } catch { /* denied */ } + setMenuOpen(false); + }, [rfbRef]); + + const handleResolution = useCallback(async (res: string) => { + setMenuOpen(false); + try { await changeDesktopResolution(sessionName, windowIndex, res); } catch { /* best-effort */ } + }, [sessionName, windowIndex]); + + const handleFullscreen = useCallback(() => { + setMenuOpen(false); + if (document.fullscreenElement) document.exitFullscreen().catch(() => {}); + else document.documentElement.requestFullscreen().catch(() => {}); + }, []); + + const handleToggleTouchMode = useCallback(() => { + const next = touchMode === "trackpad" ? "direct" : "trackpad"; + onTouchModeChange(next); + if (rfbRef.current) rfbRef.current.showDotCursor = next === "trackpad"; + try { localStorage.setItem("rk-desktop-touch-mode", next); } catch { /* noop */ } + setMenuOpen(false); + }, [touchMode, onTouchModeChange, rfbRef]); + + return ( +
+ {/* Hidden textarea for mobile keyboard input */} +