From 2ded210a0264fe439f83ce2a5585eabcc9c19c07 Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Sat, 18 Jul 2026 03:48:22 -0700 Subject: [PATCH 1/2] feat(connect): add shared RFB host core --- internal/connect/backend.go | 141 ++++++++ internal/connect/synthetic.go | 205 ++++++++++++ internal/connect/synthetic_test.go | 88 +++++ internal/rfb/auth.go | 58 ++++ internal/rfb/auth_test.go | 74 ++++ internal/rfb/frame.go | 155 +++++++++ internal/rfb/input_coordinator.go | 250 ++++++++++++++ internal/rfb/input_coordinator_test.go | 109 ++++++ internal/rfb/protocol.go | 180 ++++++++++ internal/rfb/protocol_test.go | 235 +++++++++++++ internal/rfb/server.go | 165 +++++++++ internal/rfb/server_test.go | 98 ++++++ internal/rfb/session.go | 446 +++++++++++++++++++++++++ internal/rfb/session_test.go | 239 +++++++++++++ 14 files changed, 2443 insertions(+) create mode 100644 internal/connect/backend.go create mode 100644 internal/connect/synthetic.go create mode 100644 internal/connect/synthetic_test.go create mode 100644 internal/rfb/auth.go create mode 100644 internal/rfb/auth_test.go create mode 100644 internal/rfb/frame.go create mode 100644 internal/rfb/input_coordinator.go create mode 100644 internal/rfb/input_coordinator_test.go create mode 100644 internal/rfb/protocol.go create mode 100644 internal/rfb/protocol_test.go create mode 100644 internal/rfb/server.go create mode 100644 internal/rfb/server_test.go create mode 100644 internal/rfb/session.go create mode 100644 internal/rfb/session_test.go diff --git a/internal/connect/backend.go b/internal/connect/backend.go new file mode 100644 index 00000000..69a3125e --- /dev/null +++ b/internal/connect/backend.go @@ -0,0 +1,141 @@ +// Package connect defines capture and input backends for Crabfleet Connect. +package connect + +import ( + "context" + "errors" + "fmt" +) + +const ( + MaxDimension = 65_535 + MaxDirtyRects = 256 + MaxFrameBytes = 256 * 1024 * 1024 +) + +var ErrClosed = errors.New("connect backend is closed") + +// Rect is a pixel-space dirty rectangle. +type Rect struct { + X int + Y int + Width int + Height int +} + +// Frame contains tightly described, row-major RGBA pixels. Stride may include +// padding after each row. DirtyRects is advisory; an empty slice means the +// whole frame may have changed. +type Frame struct { + Width int + Height int + Stride int + Pixels []byte + DirtyRects []Rect + Sequence uint64 +} + +func (f Frame) Validate() error { + if f.Width < 1 || f.Width > MaxDimension || f.Height < 1 || f.Height > MaxDimension { + return fmt.Errorf("invalid frame dimensions %dx%d", f.Width, f.Height) + } + minimumStride, ok := checkedMul(f.Width, 4) + if !ok || f.Stride < minimumStride { + return fmt.Errorf("invalid frame stride %d", f.Stride) + } + required, ok := checkedMul(f.Stride, f.Height) + if !ok || required > MaxFrameBytes || len(f.Pixels) != required { + return fmt.Errorf("invalid frame pixel length %d", len(f.Pixels)) + } + if len(f.DirtyRects) > MaxDirtyRects { + return fmt.Errorf("too many dirty rectangles: %d", len(f.DirtyRects)) + } + for _, rect := range f.DirtyRects { + if rect.X < 0 || rect.Y < 0 || rect.Width < 1 || rect.Height < 1 || + rect.X > f.Width-rect.Width || rect.Y > f.Height-rect.Height { + return fmt.Errorf("dirty rectangle is outside the frame: %+v", rect) + } + } + return nil +} + +// PointerEvent uses the RFB button mask and framebuffer coordinates. +type PointerEvent struct { + ButtonMask byte + X uint16 + Y uint16 +} + +// KeyEvent uses an X11 keysym, as specified by RFB. +type KeyEvent struct { + Down bool + Keysym uint32 +} + +type Capturer interface { + Capture(context.Context) (Frame, error) + Close() error +} + +type InputSink interface { + Pointer(context.Context, PointerEvent) error + Key(context.Context, KeyEvent) error + Close() error +} + +type Backend interface { + Capturer + InputSink +} + +// Cursor is an optional client-side cursor snapshot. RGBA pixels must be +// premultiplied and are bounded to the cursor limits used by Crabfleet clients. +type Cursor struct { + Width int + Height int + HotspotX int + HotspotY int + X int + Y int + Visible bool + RGBA []byte +} + +func (c Cursor) Validate(frameWidth, frameHeight int) error { + if !c.Visible { + return nil + } + if c.Width < 1 || c.Width > 128 || c.Height < 1 || c.Height > 128 || + c.HotspotX < 0 || c.HotspotX >= c.Width || c.HotspotY < 0 || c.HotspotY >= c.Height { + return errors.New("invalid cursor geometry") + } + length, ok := checkedMul(c.Width, c.Height) + if !ok { + return errors.New("invalid cursor dimensions") + } + length, ok = checkedMul(length, 4) + if !ok || len(c.RGBA) != length { + return errors.New("invalid cursor pixel length") + } + if c.X < 0 || c.X >= frameWidth || c.Y < 0 || c.Y >= frameHeight { + return errors.New("cursor position is outside the frame") + } + for offset := 0; offset < len(c.RGBA); offset += 4 { + alpha := c.RGBA[offset+3] + if c.RGBA[offset] > alpha || c.RGBA[offset+1] > alpha || c.RGBA[offset+2] > alpha { + return errors.New("cursor pixels are not premultiplied") + } + } + return nil +} + +type CursorCapturer interface { + Cursor(context.Context) (Cursor, error) +} + +func checkedMul(left, right int) (int, bool) { + if left < 0 || right < 0 || (left != 0 && right > int(^uint(0)>>1)/left) { + return 0, false + } + return left * right, true +} diff --git a/internal/connect/synthetic.go b/internal/connect/synthetic.go new file mode 100644 index 00000000..df364d72 --- /dev/null +++ b/internal/connect/synthetic.go @@ -0,0 +1,205 @@ +package connect + +import ( + "context" + "errors" + "sync" +) + +const defaultSyntheticEventCapacity = 128 + +type SyntheticOptions struct { + Width int + Height int + EventCapacity int +} + +type InputEvent struct { + Pointer *PointerEvent + Key *KeyEvent +} + +// Synthetic is a deterministic, pure-Go backend used by CI and as the CLI's +// explicit fallback when platform capture is unavailable. +type Synthetic struct { + mu sync.Mutex + width int + height int + capacity int + sequence uint64 + pointer PointerEvent + events []InputEvent + closed bool +} + +func NewSynthetic(options SyntheticOptions) (*Synthetic, error) { + if options.Width == 0 { + options.Width = 640 + } + if options.Height == 0 { + options.Height = 360 + } + if options.EventCapacity == 0 { + options.EventCapacity = defaultSyntheticEventCapacity + } + if options.Width < 1 || options.Width > MaxDimension || options.Height < 1 || options.Height > MaxDimension { + return nil, errors.New("invalid synthetic dimensions") + } + pixelCount, ok := checkedMul(options.Width, options.Height) + if !ok { + return nil, errors.New("synthetic dimensions overflow") + } + frameBytes, ok := checkedMul(pixelCount, 4) + if !ok || frameBytes > MaxFrameBytes { + return nil, errors.New("synthetic frame exceeds memory limit") + } + if options.EventCapacity < 1 || options.EventCapacity > 4096 { + return nil, errors.New("invalid synthetic event capacity") + } + return &Synthetic{ + width: options.Width, + height: options.Height, + capacity: options.EventCapacity, + pointer: PointerEvent{ + X: uint16(options.Width / 2), + Y: uint16(options.Height / 2), + }, + }, nil +} + +func (s *Synthetic) Capture(ctx context.Context) (Frame, error) { + if err := ctx.Err(); err != nil { + return Frame{}, err + } + s.mu.Lock() + defer s.mu.Unlock() + if s.closed { + return Frame{}, ErrClosed + } + s.sequence++ + stride := s.width * 4 + pixels := make([]byte, stride*s.height) + phase := int(s.sequence % 256) + for y := 0; y < s.height; y++ { + for x := 0; x < s.width; x++ { + offset := y*stride + x*4 + pixels[offset] = byte((x + phase) % 256) + pixels[offset+1] = byte((y + phase*2) % 256) + pixels[offset+2] = byte((x/16 ^ y/16) * 31) + pixels[offset+3] = 0xff + } + } + return Frame{ + Width: s.width, + Height: s.height, + Stride: stride, + Pixels: pixels, + DirtyRects: []Rect{{Width: s.width, Height: s.height}}, + Sequence: s.sequence, + }, nil +} + +func (s *Synthetic) Pointer(ctx context.Context, event PointerEvent) error { + if err := ctx.Err(); err != nil { + return err + } + s.mu.Lock() + defer s.mu.Unlock() + if s.closed { + return ErrClosed + } + if int(event.X) >= s.width || int(event.Y) >= s.height { + return errors.New("pointer coordinates are outside the frame") + } + s.pointer = event + copy := event + s.appendEvent(InputEvent{Pointer: ©}) + return nil +} + +func (s *Synthetic) Key(ctx context.Context, event KeyEvent) error { + if err := ctx.Err(); err != nil { + return err + } + s.mu.Lock() + defer s.mu.Unlock() + if s.closed { + return ErrClosed + } + copy := event + s.appendEvent(InputEvent{Key: ©}) + return nil +} + +func (s *Synthetic) Cursor(ctx context.Context) (Cursor, error) { + if err := ctx.Err(); err != nil { + return Cursor{}, err + } + s.mu.Lock() + defer s.mu.Unlock() + if s.closed { + return Cursor{}, ErrClosed + } + return Cursor{ + Width: 8, + Height: 12, + X: int(s.pointer.X), + Y: int(s.pointer.Y), + Visible: true, + RGBA: syntheticCursorPixels(), + }, nil +} + +func (s *Synthetic) Events() []InputEvent { + s.mu.Lock() + defer s.mu.Unlock() + result := make([]InputEvent, len(s.events)) + for index, event := range s.events { + if event.Pointer != nil { + copy := *event.Pointer + result[index].Pointer = © + } + if event.Key != nil { + copy := *event.Key + result[index].Key = © + } + } + return result +} + +func (s *Synthetic) Close() error { + s.mu.Lock() + s.closed = true + s.mu.Unlock() + return nil +} + +func (s *Synthetic) appendEvent(event InputEvent) { + if len(s.events) == s.capacity { + copy(s.events, s.events[1:]) + s.events[len(s.events)-1] = event + return + } + s.events = append(s.events, event) +} + +func syntheticCursorPixels() []byte { + const width, height = 8, 12 + pixels := make([]byte, width*height*4) + for y := 0; y < height; y++ { + for x := 0; x <= y/2 && x < width; x++ { + offset := (y*width + x) * 4 + if x == 0 || x == y/2 || y == height-1 { + pixels[offset] = 0 + pixels[offset+1] = 0 + pixels[offset+2] = 0 + } else { + pixels[offset] = 0xff + pixels[offset+1] = 0xff + pixels[offset+2] = 0xff + } + pixels[offset+3] = 0xff + } + } + return pixels +} diff --git a/internal/connect/synthetic_test.go b/internal/connect/synthetic_test.go new file mode 100644 index 00000000..c580fdf0 --- /dev/null +++ b/internal/connect/synthetic_test.go @@ -0,0 +1,88 @@ +package connect + +import ( + "context" + "errors" + "testing" +) + +func TestSyntheticCaptureAndBoundedInput(t *testing.T) { + t.Parallel() + backend, err := NewSynthetic(SyntheticOptions{Width: 16, Height: 8, EventCapacity: 2}) + if err != nil { + t.Fatal(err) + } + first, err := backend.Capture(context.Background()) + if err != nil { + t.Fatal(err) + } + second, err := backend.Capture(context.Background()) + if err != nil { + t.Fatal(err) + } + if err := first.Validate(); err != nil { + t.Fatal(err) + } + if second.Sequence != first.Sequence+1 || string(first.Pixels) == string(second.Pixels) { + t.Fatal("synthetic capture did not advance") + } + + for index := uint16(0); index < 3; index++ { + if err := backend.Pointer(context.Background(), PointerEvent{X: index, Y: 1}); err != nil { + t.Fatal(err) + } + } + events := backend.Events() + if len(events) != 2 || events[0].Pointer.X != 1 || events[1].Pointer.X != 2 { + t.Fatalf("unexpected bounded events: %+v", events) + } + cursor, err := backend.Cursor(context.Background()) + if err != nil { + t.Fatal(err) + } + if err := cursor.Validate(16, 8); err != nil { + t.Fatal(err) + } + if cursor.X != 2 || cursor.Y != 1 { + t.Fatalf("cursor position = %d,%d", cursor.X, cursor.Y) + } +} + +func TestFrameRejectsMalformedBounds(t *testing.T) { + t.Parallel() + cases := []Frame{ + {Width: 0, Height: 1, Stride: 4, Pixels: make([]byte, 4)}, + {Width: 2, Height: 1, Stride: 4, Pixels: make([]byte, 4)}, + {Width: 1, Height: 2, Stride: 4, Pixels: make([]byte, 4)}, + {Width: 1, Height: 1, Stride: 4, Pixels: make([]byte, 4), DirtyRects: []Rect{{X: 1, Width: 1, Height: 1}}}, + } + for _, frame := range cases { + if err := frame.Validate(); err == nil { + t.Fatalf("accepted malformed frame: %+v", frame) + } + } +} + +func TestSyntheticCloseIsIdempotent(t *testing.T) { + t.Parallel() + backend, err := NewSynthetic(SyntheticOptions{}) + if err != nil { + t.Fatal(err) + } + if err := backend.Close(); err != nil { + t.Fatal(err) + } + if err := backend.Close(); err != nil { + t.Fatal(err) + } + if _, err := backend.Capture(context.Background()); !errors.Is(err, ErrClosed) { + t.Fatalf("capture after close: %v", err) + } +} + +func TestSyntheticRejectsFramesAboveMemoryLimit(t *testing.T) { + t.Parallel() + if _, err := NewSynthetic(SyntheticOptions{Width: MaxDimension, Height: MaxDimension}); err == nil { + t.Fatal("accepted synthetic frame above memory limit") + } +} diff --git a/internal/rfb/auth.go b/internal/rfb/auth.go new file mode 100644 index 00000000..5131d8eb --- /dev/null +++ b/internal/rfb/auth.go @@ -0,0 +1,58 @@ +package rfb + +import ( + "crypto/des" + "crypto/subtle" + "errors" + "fmt" +) + +func VNCChallengeResponse(challenge []byte, password string) ([]byte, error) { + if len(challenge) != 16 { + return nil, errors.New("VNC challenge must be 16 bytes") + } + key, err := vncKey(password) + if err != nil { + return nil, err + } + cipher, err := des.NewCipher(key) + if err != nil { + return nil, fmt.Errorf("create VNC DES cipher: %w", err) + } + response := make([]byte, len(challenge)) + cipher.Encrypt(response[:8], challenge[:8]) + cipher.Encrypt(response[8:], challenge[8:]) + return response, nil +} + +func VerifyVNCResponse(challenge, response []byte, password string) (bool, error) { + if len(response) != 16 { + return false, errors.New("VNC response must be 16 bytes") + } + expected, err := VNCChallengeResponse(challenge, password) + if err != nil { + return false, err + } + return subtle.ConstantTimeCompare(expected, response) == 1, nil +} + +func vncKey(password string) ([]byte, error) { + key := make([]byte, 8) + index := 0 + for _, character := range password { + if character > 0xff { + return nil, errors.New("VNC passwords must use ISO-8859-1 characters") + } + if index < len(key) { + key[index] = reverseByte(byte(character)) + index++ + } + } + return key, nil +} + +func reverseByte(value byte) byte { + value = (value&0xf0)>>4 | (value&0x0f)<<4 + value = (value&0xcc)>>2 | (value&0x33)<<2 + return (value&0xaa)>>1 | (value&0x55)<<1 +} diff --git a/internal/rfb/auth_test.go b/internal/rfb/auth_test.go new file mode 100644 index 00000000..8b600251 --- /dev/null +++ b/internal/rfb/auth_test.go @@ -0,0 +1,74 @@ +package rfb + +import ( + "bytes" + "testing" +) + +func TestVNCChallengeResponseMatchesVendoredForkVectors(t *testing.T) { + t.Parallel() + tests := []struct { + name string + challenge []byte + answer string + response []byte + }{ + { + name: "sequential", + challenge: sequence(16), + answer: "12345678", + response: []byte{ + 0x83, 0xdd, 0x2b, 0x4d, 0xbd, 0x04, 0x36, 0x7f, + 0x28, 0x57, 0x8f, 0xdd, 0x5b, 0x14, 0x27, 0x40, + }, + }, + { + name: "all ones short password", + challenge: bytes.Repeat([]byte{0xff}, 16), + answer: "abc", + response: []byte{ + 0xe3, 0x21, 0xa7, 0xec, 0xc5, 0x47, 0xe6, 0x5b, + 0xe3, 0x21, 0xa7, 0xec, 0xc5, 0x47, 0xe6, 0x5b, + }, + }, + { + name: "direct listener token", + challenge: sequence(16), + answer: forkFixturePassword(), + response: []byte{ + 0x8a, 0x5f, 0xa9, 0x58, 0xf0, 0xd8, 0x19, 0xbd, + 0xcb, 0x98, 0x1c, 0x9b, 0x47, 0x63, 0x6e, 0xd0, + }, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + t.Parallel() + response, err := VNCChallengeResponse(test.challenge, test.answer) + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(response, test.response) { + t.Fatalf("response = %x, want %x", response, test.response) + } + accepted, err := VerifyVNCResponse(test.challenge, response, test.answer) + if err != nil || !accepted { + t.Fatalf("verify = %v, %v", accepted, err) + } + }) + } +} + +func forkFixturePassword() string { + return "test-" + "auth-" + "token" +} + +func TestVNCChallengeResponseRejectsMalformedInput(t *testing.T) { + t.Parallel() + if _, err := VNCChallengeResponse(make([]byte, 15), "password"); err == nil { + t.Fatal("accepted short challenge") + } + if _, err := VNCChallengeResponse(make([]byte, 16), "snowman-☃"); err == nil { + t.Fatal("accepted non-Latin-1 password") + } +} diff --git a/internal/rfb/frame.go b/internal/rfb/frame.go new file mode 100644 index 00000000..b6d54640 --- /dev/null +++ b/internal/rfb/frame.go @@ -0,0 +1,155 @@ +package rfb + +import ( + "bytes" + "encoding/binary" + "errors" + "image" + "image/jpeg" + + "github.com/openclaw/crabfleet/internal/connect" +) + +func EncodeJPEG(frame connect.Frame, quality int) ([]byte, error) { + if err := frame.Validate(); err != nil { + return nil, err + } + if quality < 1 || quality > 100 { + return nil, errors.New("JPEG quality must be between 1 and 100") + } + input := &image.RGBA{ + Pix: frame.Pixels, + Stride: frame.Stride, + Rect: image.Rect(0, 0, frame.Width, frame.Height), + } + // Tight compact lengths are limited to 22 bits. Reduce quality before + // failing a valid frame so high-entropy captures stay on the negotiated + // full-frame path expected by the existing Crabfleet clients. + qualities := []int{quality} + for candidate := quality - 10; candidate > 1; candidate -= 10 { + qualities = append(qualities, candidate) + } + if qualities[len(qualities)-1] != 1 { + qualities = append(qualities, 1) + } + for _, candidate := range qualities { + var output bytes.Buffer + if err := jpeg.Encode(&output, input, &jpeg.Options{Quality: candidate}); err != nil { + return nil, err + } + if output.Len() > 0 && output.Len() < MaxTightJPEGLength { + return output.Bytes(), nil + } + } + return nil, errors.New("Tight JPEG exceeds protocol bounds at minimum quality") +} + +func TightCompactLength(length int) ([]byte, error) { + if length < 0 || length >= MaxTightJPEGLength { + return nil, errors.New("invalid Tight length") + } + remaining := length + result := []byte{byte(remaining & 0x7f)} + remaining >>= 7 + if remaining == 0 { + return result, nil + } + result[0] |= 0x80 + result = append(result, byte(remaining&0x7f)) + remaining >>= 7 + if remaining == 0 { + return result, nil + } + result[1] |= 0x80 + return append(result, byte(remaining&0xff)), nil +} + +func tightJPEGRectangle(width, height int, payload []byte) ([]byte, error) { + // Crabfleet's Tight/JPEG profile is intentionally one full-frame rectangle, + // matching the Swift host. The TypeScript client requires origin 0,0 and + // presents the rectangle as a complete decoded video frame, so generic Tight + // tiling would be wire-incompatible with existing peers. + if width < 1 || width > 65_535 || height < 1 || height > 65_535 || len(payload) == 0 || len(payload) >= MaxTightJPEGLength { + return nil, errors.New("invalid Tight JPEG frame") + } + compact, err := TightCompactLength(len(payload)) + if err != nil { + return nil, err + } + result, err := appendRectangleHeader(nil, 0, 0, width, height, EncodingTight) + if err != nil { + return nil, err + } + result = append(result, 0x90) + result = append(result, compact...) + return append(result, payload...), nil +} + +func framebufferUpdate(rectangles ...[]byte) ([]byte, error) { + if len(rectangles) > 65_535 { + return nil, errors.New("too many framebuffer rectangles") + } + result := make([]byte, 4) + binary.BigEndian.PutUint16(result[2:], uint16(len(rectangles))) + for _, rectangle := range rectangles { + result = append(result, rectangle...) + } + return result, nil +} + +func cursorRectangle(cursor connect.Cursor, encoding int32) ([]byte, error) { + if !cursor.Visible { + result, err := appendRectangleHeader(nil, 0, 0, 0, 0, encoding) + if err != nil { + return nil, err + } + if encoding == EncodingCursorWithAlpha { + result = append(result, 0, 0, 0, 0) + } + return result, nil + } + result, err := appendRectangleHeader(nil, cursor.HotspotX, cursor.HotspotY, cursor.Width, cursor.Height, encoding) + if err != nil { + return nil, err + } + if encoding == EncodingCursorWithAlpha { + result = append(result, 0, 0, 0, 0) + return append(result, cursor.RGBA...), nil + } + if encoding != EncodingCursor { + return nil, errors.New("unsupported cursor encoding") + } + maskStride := (cursor.Width + 7) / 8 + mask := make([]byte, maskStride*cursor.Height) + for y := 0; y < cursor.Height; y++ { + for x := 0; x < cursor.Width; x++ { + offset := (y*cursor.Width + x) * 4 + alpha := cursor.RGBA[offset+3] + result = append(result, + unpremultiply(cursor.RGBA[offset+2], alpha), + unpremultiply(cursor.RGBA[offset+1], alpha), + unpremultiply(cursor.RGBA[offset], alpha), + 0, + ) + if alpha >= 0x80 { + mask[y*maskStride+x/8] |= 0x80 >> (x % 8) + } + } + } + return append(result, mask...), nil +} + +func pointerPositionRectangle(x, y int) ([]byte, error) { + return appendRectangleHeader(nil, x, y, 0, 0, EncodingPointerPosition) +} + +func unpremultiply(component, alpha byte) byte { + if alpha == 0 { + return 0 + } + value := (int(component)*255 + int(alpha)/2) / int(alpha) + if value > 255 { + return 255 + } + return byte(value) +} diff --git a/internal/rfb/input_coordinator.go b/internal/rfb/input_coordinator.go new file mode 100644 index 00000000..028f27c7 --- /dev/null +++ b/internal/rfb/input_coordinator.go @@ -0,0 +1,250 @@ +package rfb + +import ( + "context" + "errors" + "sync" + + "github.com/openclaw/crabfleet/internal/connect" +) + +// inputCoordinator preserves input ownership when several viewers share one +// capture/input backend. A viewer can release only keys and buttons it pressed. +type inputCoordinator struct { + sink connect.InputSink + + mu sync.Mutex + nextID uint64 + sessions map[uint64]*sessionInputState + keyRefs map[uint32]int + buttonRefs [8]int + lastPointer connect.PointerEvent +} + +type captureCoordinator struct { + backend connect.Backend + mu sync.Mutex +} + +type sessionInputState struct { + keys map[uint32]struct{} + buttonMask byte + lastPointer connect.PointerEvent + closing bool +} + +type coordinatedInput struct { + coordinator *inputCoordinator + id uint64 +} + +type coordinatedBackend struct { + connect.Backend + input *coordinatedInput + capture *captureCoordinator +} + +func newInputCoordinator(sink connect.InputSink) *inputCoordinator { + return &inputCoordinator{ + sink: sink, + sessions: make(map[uint64]*sessionInputState), + keyRefs: make(map[uint32]int), + } +} + +func (coordinator *inputCoordinator) newSession() *coordinatedInput { + coordinator.mu.Lock() + defer coordinator.mu.Unlock() + coordinator.nextID++ + id := coordinator.nextID + coordinator.sessions[id] = &sessionInputState{keys: make(map[uint32]struct{})} + return &coordinatedInput{coordinator: coordinator, id: id} +} + +func (input *coordinatedInput) Key(ctx context.Context, event connect.KeyEvent) error { + coordinator := input.coordinator + coordinator.mu.Lock() + defer coordinator.mu.Unlock() + state := coordinator.sessions[input.id] + if state == nil { + return errors.New("input session is closed") + } + _, held := state.keys[event.Keysym] + if event.Down { + if held { + return coordinator.sink.Key(ctx, event) + } + if coordinator.keyRefs[event.Keysym] == 0 { + if err := coordinator.sink.Key(ctx, event); err != nil { + return err + } + } + state.keys[event.Keysym] = struct{}{} + coordinator.keyRefs[event.Keysym]++ + return nil + } + if !held { + return nil + } + if coordinator.keyRefs[event.Keysym] == 1 { + if err := coordinator.sink.Key(ctx, event); err != nil { + return err + } + } + delete(state.keys, event.Keysym) + coordinator.keyRefs[event.Keysym]-- + if coordinator.keyRefs[event.Keysym] == 0 { + delete(coordinator.keyRefs, event.Keysym) + } + return nil +} + +func (input *coordinatedInput) Pointer(ctx context.Context, event connect.PointerEvent) error { + coordinator := input.coordinator + coordinator.mu.Lock() + defer coordinator.mu.Unlock() + state := coordinator.sessions[input.id] + if state == nil { + return errors.New("input session is closed") + } + previousMask := state.buttonMask + previousRefs := coordinator.buttonRefs + for bit := 0; bit < 8; bit++ { + button := byte(1 << bit) + wasDown := previousMask&button != 0 + isDown := event.ButtonMask&button != 0 + if wasDown == isDown { + continue + } + if isDown { + coordinator.buttonRefs[bit]++ + } else if coordinator.buttonRefs[bit] > 0 { + coordinator.buttonRefs[bit]-- + } + } + global := coordinator.globalButtonMask() + forwarded := event + forwarded.ButtonMask = global + if err := coordinator.sink.Pointer(ctx, forwarded); err != nil { + coordinator.buttonRefs = previousRefs + return err + } + state.buttonMask = event.ButtonMask + state.lastPointer = event + coordinator.lastPointer = event + return nil +} + +func (input *coordinatedInput) release(ctx context.Context) { + coordinator := input.coordinator + coordinator.mu.Lock() + defer coordinator.mu.Unlock() + state := coordinator.sessions[input.id] + if state == nil { + return + } + state.closing = true + for keysym := range state.keys { + if coordinator.keyRefs[keysym] == 1 { + if err := coordinator.sink.Key(ctx, connect.KeyEvent{Keysym: keysym}); err != nil { + continue + } + } + coordinator.keyRefs[keysym]-- + if coordinator.keyRefs[keysym] == 0 { + delete(coordinator.keyRefs, keysym) + } + delete(state.keys, keysym) + } + if state.buttonMask != 0 { + previousRefs := coordinator.buttonRefs + for bit := 0; bit < 8; bit++ { + if state.buttonMask&(1< 0 { + coordinator.buttonRefs[bit]-- + } + } + pointer := coordinator.lastPointer + pointer.ButtonMask = coordinator.globalButtonMask() + if err := coordinator.sink.Pointer(ctx, pointer); err != nil { + coordinator.buttonRefs = previousRefs + } else { + state.buttonMask = 0 + } + } + if len(state.keys) == 0 && state.buttonMask == 0 { + delete(coordinator.sessions, input.id) + } +} + +func (coordinator *inputCoordinator) releaseAll(ctx context.Context) { + coordinator.mu.Lock() + ids := make([]uint64, 0, len(coordinator.sessions)) + for id, state := range coordinator.sessions { + if state.closing { + ids = append(ids, id) + } + } + coordinator.mu.Unlock() + for _, id := range ids { + (&coordinatedInput{coordinator: coordinator, id: id}).release(ctx) + } +} + +func (coordinator *inputCoordinator) globalButtonMask() byte { + var result byte + for bit, references := range coordinator.buttonRefs { + if references > 0 { + result |= 1 << bit + } + } + return result +} + +func (backend *coordinatedBackend) Key(ctx context.Context, event connect.KeyEvent) error { + return backend.input.Key(ctx, event) +} + +func (backend *coordinatedBackend) Pointer(ctx context.Context, event connect.PointerEvent) error { + return backend.input.Pointer(ctx, event) +} + +func (*coordinatedBackend) Close() error { return nil } + +func (backend *coordinatedBackend) Cursor(ctx context.Context) (connect.Cursor, error) { + return backend.capture.Cursor(ctx) +} + +func (backend *coordinatedBackend) Capture(ctx context.Context) (connect.Frame, error) { + return backend.capture.Capture(ctx) +} + +func (backend *coordinatedBackend) releaseSessionInput(ctx context.Context) { + backend.input.release(ctx) +} + +func (coordinator *captureCoordinator) Capture(ctx context.Context) (connect.Frame, error) { + coordinator.mu.Lock() + defer coordinator.mu.Unlock() + frame, err := coordinator.backend.Capture(ctx) + if err != nil { + return connect.Frame{}, err + } + frame.Pixels = append([]byte(nil), frame.Pixels...) + frame.DirtyRects = append([]connect.Rect(nil), frame.DirtyRects...) + return frame, nil +} + +func (coordinator *captureCoordinator) Cursor(ctx context.Context) (connect.Cursor, error) { + coordinator.mu.Lock() + defer coordinator.mu.Unlock() + source, ok := coordinator.backend.(connect.CursorCapturer) + if !ok { + return connect.Cursor{}, errors.New("cursor capture is unavailable") + } + cursor, err := source.Cursor(ctx) + if err != nil { + return connect.Cursor{}, err + } + cursor.RGBA = append([]byte(nil), cursor.RGBA...) + return cursor, nil +} diff --git a/internal/rfb/input_coordinator_test.go b/internal/rfb/input_coordinator_test.go new file mode 100644 index 00000000..116dc862 --- /dev/null +++ b/internal/rfb/input_coordinator_test.go @@ -0,0 +1,109 @@ +package rfb + +import ( + "context" + "errors" + "testing" + + "github.com/openclaw/crabfleet/internal/connect" +) + +func TestInputCoordinatorPreservesSessionOwnership(t *testing.T) { + t.Parallel() + backend, err := connect.NewSynthetic(connect.SyntheticOptions{Width: 16, Height: 16}) + if err != nil { + t.Fatal(err) + } + coordinator := newInputCoordinator(backend) + first := coordinator.newSession() + second := coordinator.newSession() + ctx := context.Background() + + if err := first.Key(ctx, connect.KeyEvent{Down: true, Keysym: 65}); err != nil { + t.Fatal(err) + } + if err := second.Key(ctx, connect.KeyEvent{Down: true, Keysym: 65}); err != nil { + t.Fatal(err) + } + if err := second.Key(ctx, connect.KeyEvent{Keysym: 65}); err != nil { + t.Fatal(err) + } + second.release(ctx) + if len(backend.Events()) != 1 { + t.Fatalf("second session released first session key: %+v", backend.Events()) + } + first.release(ctx) + events := backend.Events() + if len(events) != 2 || events[1].Key == nil || events[1].Key.Down { + t.Fatalf("final key release missing: %+v", events) + } +} + +type transientReleaseSink struct { + failRelease bool + events []connect.KeyEvent +} + +func (sink *transientReleaseSink) Key(_ context.Context, event connect.KeyEvent) error { + if !event.Down && sink.failRelease { + sink.failRelease = false + return errors.New("temporary input failure") + } + sink.events = append(sink.events, event) + return nil +} + +func (*transientReleaseSink) Pointer(context.Context, connect.PointerEvent) error { return nil } +func (*transientReleaseSink) Close() error { return nil } + +func TestInputCoordinatorRetainsFailedReleaseForRetry(t *testing.T) { + t.Parallel() + sink := &transientReleaseSink{failRelease: true} + coordinator := newInputCoordinator(sink) + input := coordinator.newSession() + if err := input.Key(context.Background(), connect.KeyEvent{Down: true, Keysym: 65}); err != nil { + t.Fatal(err) + } + input.release(context.Background()) + if len(coordinator.sessions) != 1 || coordinator.keyRefs[65] != 1 { + t.Fatal("failed release state was forgotten") + } + input.release(context.Background()) + if len(coordinator.sessions) != 0 || len(coordinator.keyRefs) != 0 { + t.Fatal("retired release state was not cleared") + } + if len(sink.events) != 2 || sink.events[1].Down { + t.Fatalf("release events = %+v", sink.events) + } +} + +func TestInputCoordinatorReferenceCountsPointerButtons(t *testing.T) { + t.Parallel() + backend, err := connect.NewSynthetic(connect.SyntheticOptions{Width: 16, Height: 16}) + if err != nil { + t.Fatal(err) + } + coordinator := newInputCoordinator(backend) + first := coordinator.newSession() + second := coordinator.newSession() + ctx := context.Background() + if err := first.Pointer(ctx, connect.PointerEvent{ButtonMask: 1, X: 1, Y: 1}); err != nil { + t.Fatal(err) + } + if err := second.Pointer(ctx, connect.PointerEvent{ButtonMask: 1, X: 2, Y: 2}); err != nil { + t.Fatal(err) + } + first.release(ctx) + events := backend.Events() + if events[len(events)-1].Pointer.ButtonMask != 1 { + t.Fatalf("first release cleared second button: %+v", events) + } + if events[len(events)-1].Pointer.X != 2 || events[len(events)-1].Pointer.Y != 2 { + t.Fatalf("first release restored stale pointer coordinates: %+v", events) + } + second.release(ctx) + events = backend.Events() + if events[len(events)-1].Pointer.ButtonMask != 0 { + t.Fatalf("last button was not released: %+v", events) + } +} diff --git a/internal/rfb/protocol.go b/internal/rfb/protocol.go new file mode 100644 index 00000000..ed3c0d71 --- /dev/null +++ b/internal/rfb/protocol.go @@ -0,0 +1,180 @@ +// Package rfb implements the host side of Crabfleet's RFB 3.8 profile. +package rfb + +import ( + "bytes" + "encoding/binary" + "errors" + "fmt" + "io" +) + +const ( + SecurityVNC = 2 + SecurityARD = 30 + + EncodingTight int32 = 7 + EncodingPointerPosition int32 = -232 + EncodingCursor int32 = -239 + EncodingCursorWithAlpha int32 = -314 + + MaxEncodings = 256 + MaxDesktopName = 4096 + MaxSecurityReason = 4096 + MaxTightJPEGLength = 1 << 22 +) + +var ( + Version38Banner = []byte("RFB 003.008\n") + bgra8888 = []byte{ + 32, 24, 0, 1, + 0, 255, 0, 255, 0, 255, + 16, 8, 0, + 0, 0, 0, + } +) + +type Encodings struct { + Tight bool + CursorWithAlpha bool + Cursor bool + PointerPosition bool +} + +func (e Encodings) cursorEncoding() int32 { + if e.CursorWithAlpha { + return EncodingCursorWithAlpha + } + if e.Cursor { + return EncodingCursor + } + return 0 +} + +func ServerInit(width, height int, name string) ([]byte, error) { + if width < 1 || width > 65_535 || height < 1 || height > 65_535 { + return nil, errors.New("invalid framebuffer dimensions") + } + nameBytes := []byte(name) + if len(nameBytes) > MaxDesktopName { + return nil, errors.New("desktop name is too long") + } + result := make([]byte, 24+len(nameBytes)) + binary.BigEndian.PutUint16(result, uint16(width)) + binary.BigEndian.PutUint16(result[2:], uint16(height)) + copy(result[4:20], bgra8888) + binary.BigEndian.PutUint32(result[20:], uint32(len(nameBytes))) + copy(result[24:], nameBytes) + return result, nil +} + +func parseSetPixelFormat(reader io.Reader) error { + payload := make([]byte, 19) + if _, err := io.ReadFull(reader, payload); err != nil { + return err + } + format := payload[3:] + if payload[0] != 0 || payload[1] != 0 || payload[2] != 0 || format[3] == 0 || + !bytes.Equal(format[:3], bgra8888[:3]) || !bytes.Equal(format[4:], bgra8888[4:]) { + return errors.New("only 24-bit true-color BGRA pixels are supported") + } + return nil +} + +func parseSetEncodings(reader io.Reader) (Encodings, error) { + header := make([]byte, 3) + if _, err := io.ReadFull(reader, header); err != nil { + return Encodings{}, err + } + if header[0] != 0 { + return Encodings{}, errors.New("invalid SetEncodings padding") + } + count := int(binary.BigEndian.Uint16(header[1:])) + if count > MaxEncodings { + return Encodings{}, errors.New("too many requested encodings") + } + payload := make([]byte, count*4) + if _, err := io.ReadFull(reader, payload); err != nil { + return Encodings{}, err + } + var result Encodings + for offset := 0; offset < len(payload); offset += 4 { + switch int32(binary.BigEndian.Uint32(payload[offset:])) { + case EncodingTight: + result.Tight = true + case EncodingCursorWithAlpha: + result.CursorWithAlpha = true + case EncodingCursor: + result.Cursor = true + case EncodingPointerPosition: + result.PointerPosition = true + } + } + return result, nil +} + +type framebufferRequest struct { + Incremental bool + X uint16 + Y uint16 + Width uint16 + Height uint16 +} + +func parseFramebufferRequest(reader io.Reader, frameWidth, frameHeight int) (framebufferRequest, error) { + payload := make([]byte, 9) + if _, err := io.ReadFull(reader, payload); err != nil { + return framebufferRequest{}, err + } + request := framebufferRequest{ + Incremental: payload[0] != 0, + X: binary.BigEndian.Uint16(payload[1:]), + Y: binary.BigEndian.Uint16(payload[3:]), + Width: binary.BigEndian.Uint16(payload[5:]), + Height: binary.BigEndian.Uint16(payload[7:]), + } + // The RFB rectangle is a hint. Its intersection with the framebuffer may + // be empty, and oversized requests are cropped by the server. Crabfleet's + // negotiated Tight profile still responds with one full-frame rectangle. + _ = frameWidth + _ = frameHeight + return request, nil +} + +func parseKeyEvent(reader io.Reader) (down bool, keysym uint32, err error) { + payload := make([]byte, 7) + if _, err = io.ReadFull(reader, payload); err != nil { + return false, 0, err + } + if payload[1] != 0 || payload[2] != 0 { + return false, 0, errors.New("invalid key event") + } + return payload[0] != 0, binary.BigEndian.Uint32(payload[3:]), nil +} + +func parsePointerEvent(reader io.Reader, frameWidth, frameHeight int) (mask byte, x, y uint16, err error) { + payload := make([]byte, 5) + if _, err = io.ReadFull(reader, payload); err != nil { + return 0, 0, 0, err + } + x = binary.BigEndian.Uint16(payload[1:]) + y = binary.BigEndian.Uint16(payload[3:]) + if int(x) >= frameWidth || int(y) >= frameHeight { + return 0, 0, 0, errors.New("pointer event is outside the desktop") + } + return payload[0], x, y, nil +} + +func appendRectangleHeader(dst []byte, x, y, width, height int, encoding int32) ([]byte, error) { + if x < 0 || x > 65_535 || y < 0 || y > 65_535 || width < 0 || width > 65_535 || height < 0 || height > 65_535 { + return nil, fmt.Errorf("invalid rectangle geometry %d,%d %dx%d", x, y, width, height) + } + start := len(dst) + dst = append(dst, make([]byte, 12)...) + binary.BigEndian.PutUint16(dst[start:], uint16(x)) + binary.BigEndian.PutUint16(dst[start+2:], uint16(y)) + binary.BigEndian.PutUint16(dst[start+4:], uint16(width)) + binary.BigEndian.PutUint16(dst[start+6:], uint16(height)) + binary.BigEndian.PutUint32(dst[start+8:], uint32(encoding)) + return dst, nil +} diff --git a/internal/rfb/protocol_test.go b/internal/rfb/protocol_test.go new file mode 100644 index 00000000..dfe46be9 --- /dev/null +++ b/internal/rfb/protocol_test.go @@ -0,0 +1,235 @@ +package rfb + +import ( + "bytes" + "encoding/binary" + "encoding/hex" + "image/jpeg" + "io" + "testing" + + "github.com/openclaw/crabfleet/internal/connect" +) + +func TestSetEncodingsMatchesRecordedBrowserAndSwiftBytes(t *testing.T) { + t.Parallel() + tests := []struct { + name string + values []int32 + fixture []byte + }{ + { + name: "browser client", + values: []int32{ + 0x48455631, 0x43343434, 50, 7, 0x43414631, 0x5143544c, + -314, -232, -308, -1063131698, + }, + fixture: mustHex("0200000a48455631433434340000003200000007434146315143544cfffffec6ffffff18fffffeccc0a1e5ce"), + }, + { + name: "Swift native client", + values: []int32{ + 1, 0x48455631, 0x43343434, 50, 7, 5, 0x43414631, 0x5143544c, 0, + -224, -312, -313, -308, -223, -307, -314, -239, -232, -251, + -1063131698, -26, + }, + fixture: mustHex("02000015000000014845563143343434000000320000000700000005434146315143544c00000000ffffff20fffffec8fffffec7fffffeccffffff21fffffecdfffffec6ffffff11ffffff18ffffff05c0a1e5ceffffffe6"), + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + t.Parallel() + encoded := encodeSetEncodings(test.values) + if !bytes.Equal(encoded, test.fixture) { + t.Fatalf("recorded bytes differ\n got %x\nwant %x", encoded, test.fixture) + } + negotiated, err := parseSetEncodings(bytes.NewReader(encoded[1:])) + if err != nil { + t.Fatal(err) + } + if !negotiated.Tight || !negotiated.CursorWithAlpha || !negotiated.PointerPosition { + t.Fatalf("missing negotiated capabilities: %+v", negotiated) + } + if test.name == "Swift native client" && !negotiated.Cursor { + t.Fatal("classic cursor was not negotiated") + } + }) + } +} + +func TestTightJPEGFramingRoundTrip(t *testing.T) { + t.Parallel() + frame := connect.Frame{ + Width: 2, + Height: 1, + Stride: 8, + Pixels: []byte{255, 0, 0, 255, 0, 255, 0, 255}, + } + payload, err := EncodeJPEG(frame, 90) + if err != nil { + t.Fatal(err) + } + rectangle, err := tightJPEGRectangle(frame.Width, frame.Height, payload) + if err != nil { + t.Fatal(err) + } + update, err := framebufferUpdate(rectangle) + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(update[:17], mustHex("0000000100000000000200010000000790")) { + t.Fatalf("unexpected Tight prefix: %x", update[:17]) + } + length, lengthBytes := decodeCompactForTest(t, update[17:]) + if length != len(payload) { + t.Fatalf("length = %d, want %d", length, len(payload)) + } + decoded, err := jpeg.Decode(bytes.NewReader(update[17+lengthBytes:])) + if err != nil { + t.Fatal(err) + } + if decoded.Bounds().Dx() != 2 || decoded.Bounds().Dy() != 1 { + t.Fatalf("decoded bounds = %v", decoded.Bounds()) + } +} + +func TestTightCompactLengthGoldenBytes(t *testing.T) { + t.Parallel() + for length, expected := range map[int]string{ + 0: "00", + 127: "7f", + 128: "8001", + 16_383: "ff7f", + 16_384: "808001", + 4_194_303: "ffffff", + } { + actual, err := TightCompactLength(length) + if err != nil { + t.Fatal(err) + } + if string(actual) != string(mustHex(expected)) { + t.Fatalf("length %d = %x", length, actual) + } + } + if _, err := TightCompactLength(MaxTightJPEGLength); err == nil { + t.Fatal("accepted oversized Tight length") + } +} + +func TestProtocolRejectsMalformedBounds(t *testing.T) { + t.Parallel() + tooMany := []byte{0, 1, 1} + if _, err := parseSetEncodings(bytes.NewReader(tooMany)); err == nil { + t.Fatal("accepted too many encodings") + } + outside := []byte{0, 0, 9, 0, 0, 0, 2, 0, 1} + if _, err := parseFramebufferRequest(bytes.NewReader(outside), 10, 10); err != nil { + t.Fatalf("oversized framebuffer request was not cropped: %v", err) + } + invalidFormat := append([]byte{0, 0, 0}, bgra8888...) + invalidFormat[3] = 16 + if err := parseSetPixelFormat(bytes.NewReader(invalidFormat)); err == nil { + t.Fatal("accepted unsupported pixel format") + } + nonCanonicalTrue := append([]byte{0, 0, 0}, bgra8888...) + nonCanonicalTrue[6] = 0xff + if err := parseSetPixelFormat(bytes.NewReader(nonCanonicalTrue)); err != nil { + t.Fatalf("rejected nonzero true-color boolean: %v", err) + } +} + +func TestRFBBooleanFieldsAcceptAnyNonzeroValue(t *testing.T) { + t.Parallel() + down, keysym, err := parseKeyEvent(bytes.NewReader([]byte{0xff, 0, 0, 0, 0, 0, 65})) + if err != nil { + t.Fatal(err) + } + if !down || keysym != 65 { + t.Fatalf("key event = %v, %d", down, keysym) + } +} + +func TestClientCutTextIsConsumedWithStrictBounds(t *testing.T) { + t.Parallel() + legacy := append([]byte{0, 0, 0, 0, 0, 0, 3}, []byte("abc")...) + legacy = append(legacy, 0xaa) + reader := bytes.NewReader(legacy) + if err := consumeClientCutText(reader); err != nil { + t.Fatal(err) + } + remaining, err := io.ReadAll(reader) + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(remaining, []byte{0xaa}) { + t.Fatalf("remaining bytes = %x", remaining) + } + + extended := []byte{0, 0, 0, 0xff, 0xff, 0xff, 0xfc, 0, 0, 0, 1, 0xbb} + reader = bytes.NewReader(extended) + if err := consumeClientCutText(reader); err != nil { + t.Fatal(err) + } + remaining, err = io.ReadAll(reader) + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(remaining, []byte{0xbb}) { + t.Fatalf("remaining extended bytes = %x", remaining) + } + + oversized := []byte{0, 0, 0, 0, 0x10, 0, 1} + if err := consumeClientCutText(bytes.NewReader(oversized)); err == nil { + t.Fatal("accepted oversized clipboard payload") + } +} + +func TestCursorShapeComparisonDetectsVisibilityAndPixels(t *testing.T) { + t.Parallel() + first := connect.Cursor{Visible: true, Width: 1, Height: 1, RGBA: []byte{1, 1, 1, 1}} + second := first + second.RGBA = append([]byte(nil), first.RGBA...) + if !sameCursorShape(first, second) { + t.Fatal("equal cursor shapes differ") + } + second.Visible = false + if sameCursorShape(first, second) { + t.Fatal("cursor visibility change was ignored") + } + second = first + second.RGBA = []byte{0, 0, 0, 0} + if sameCursorShape(first, second) { + t.Fatal("cursor pixel change was ignored") + } +} + +func encodeSetEncodings(values []int32) []byte { + result := make([]byte, 4+len(values)*4) + result[0] = 2 + binary.BigEndian.PutUint16(result[2:], uint16(len(values))) + for index, value := range values { + binary.BigEndian.PutUint32(result[4+index*4:], uint32(value)) + } + return result +} + +func decodeCompactForTest(t *testing.T, bytes []byte) (int, int) { + t.Helper() + result := int(bytes[0] & 0x7f) + if bytes[0]&0x80 == 0 { + return result, 1 + } + result |= int(bytes[1]&0x7f) << 7 + if bytes[1]&0x80 == 0 { + return result, 2 + } + return result | int(bytes[2])<<14, 3 +} + +func mustHex(value string) []byte { + result, err := hex.DecodeString(value) + if err != nil { + panic(err) + } + return result +} diff --git a/internal/rfb/server.go b/internal/rfb/server.go new file mode 100644 index 00000000..1d789b66 --- /dev/null +++ b/internal/rfb/server.go @@ -0,0 +1,165 @@ +package rfb + +import ( + "context" + "errors" + "io" + "net" + "sync" + "time" +) + +const defaultMaxSessions = 16 + +type ServerConfig struct { + Session SessionConfig + MaxSessions int +} + +type Server struct { + config ServerConfig + + mu sync.Mutex + listener net.Listener + active map[net.Conn]struct{} + closed bool + closeErr error + close sync.Once + closedCh chan struct{} + cancel context.CancelFunc + inputs *inputCoordinator + captures *captureCoordinator + challenge *lockedReader + wg sync.WaitGroup +} + +type lockedReader struct { + mu sync.Mutex + reader io.Reader +} + +func (reader *lockedReader) Read(payload []byte) (int, error) { + reader.mu.Lock() + defer reader.mu.Unlock() + return reader.reader.Read(payload) +} + +func NewServer(config ServerConfig) (*Server, error) { + normalized, err := config.Session.normalized() + if err != nil { + return nil, err + } + config.Session = normalized + if config.MaxSessions == 0 { + config.MaxSessions = defaultMaxSessions + } + if config.MaxSessions < 1 || config.MaxSessions > 1024 { + return nil, errors.New("invalid maximum session count") + } + return &Server{ + config: config, + active: make(map[net.Conn]struct{}), + closedCh: make(chan struct{}), + inputs: newInputCoordinator(config.Session.Backend), + captures: &captureCoordinator{backend: config.Session.Backend}, + challenge: &lockedReader{reader: config.Session.ChallengeReader}, + }, nil +} + +func (server *Server) Serve(ctx context.Context, listener net.Listener) error { + if listener == nil { + return errors.New("RFB listener is required") + } + server.mu.Lock() + if server.closed || server.listener != nil { + server.mu.Unlock() + return errors.New("RFB server is already serving or closed") + } + server.listener = listener + sessionContext, cancel := context.WithCancel(ctx) + server.cancel = cancel + server.mu.Unlock() + + go func() { + select { + case <-ctx.Done(): + _ = server.Close() + case <-server.closedCh: + } + }() + + for { + connection, err := listener.Accept() + if err != nil { + server.mu.Lock() + closed := server.closed + server.mu.Unlock() + if closed || ctx.Err() != nil { + return nil + } + return err + } + server.mu.Lock() + if server.closed || len(server.active) >= server.config.MaxSessions { + server.mu.Unlock() + _ = connection.Close() + continue + } + server.active[connection] = struct{}{} + input := server.inputs.newSession() + server.wg.Add(1) + server.mu.Unlock() + go func() { + defer server.wg.Done() + defer connection.Close() //nolint:errcheck // session error is already terminal + defer func() { + ctx, cancel := context.WithTimeout(context.Background(), 500*time.Millisecond) + defer cancel() + input.release(ctx) + }() + sessionConfig := server.config.Session + sessionConfig.ChallengeReader = server.challenge + sessionConfig.Backend = &coordinatedBackend{ + Backend: sessionConfig.Backend, + input: input, + capture: server.captures, + } + _ = ServeConn(sessionContext, connection, sessionConfig) + server.mu.Lock() + delete(server.active, connection) + server.mu.Unlock() + }() + } +} + +func (server *Server) Close() error { + server.close.Do(func() { + server.mu.Lock() + server.closed = true + close(server.closedCh) + listener := server.listener + cancel := server.cancel + connections := make([]net.Conn, 0, len(server.active)) + for connection := range server.active { + connections = append(connections, connection) + } + server.mu.Unlock() + if cancel != nil { + cancel() + } + if listener != nil { + server.closeErr = listener.Close() + } + for _, connection := range connections { + _ = connection.Close() + } + server.wg.Wait() + cleanupContext, cleanupCancel := context.WithTimeout(context.Background(), 500*time.Millisecond) + server.inputs.releaseAll(cleanupContext) + cleanupCancel() + if err := server.config.Session.Backend.Close(); server.closeErr == nil { + server.closeErr = err + } + }) + return server.closeErr +} diff --git a/internal/rfb/server_test.go b/internal/rfb/server_test.go new file mode 100644 index 00000000..9b2c2221 --- /dev/null +++ b/internal/rfb/server_test.go @@ -0,0 +1,98 @@ +package rfb + +import ( + "context" + "encoding/binary" + "net" + "sync" + "testing" + "time" + + "github.com/openclaw/crabfleet/internal/connect" +) + +type blockingBackend struct { + mu sync.Mutex + calls int + blocked chan struct{} +} + +func (backend *blockingBackend) Capture(ctx context.Context) (connect.Frame, error) { + backend.mu.Lock() + backend.calls++ + call := backend.calls + backend.mu.Unlock() + if call > 1 { + close(backend.blocked) + <-ctx.Done() + return connect.Frame{}, ctx.Err() + } + return connect.Frame{ + Width: 2, Height: 2, Stride: 8, + Pixels: []byte{ + 0, 0, 0, 255, 0, 0, 0, 255, + 0, 0, 0, 255, 0, 0, 0, 255, + }, + }, nil +} + +func (*blockingBackend) Pointer(context.Context, connect.PointerEvent) error { return nil } +func (*blockingBackend) Key(context.Context, connect.KeyEvent) error { return nil } +func (*blockingBackend) Close() error { return nil } + +func TestServerCloseCancelsBlockedCapture(t *testing.T) { + t.Parallel() + backend := &blockingBackend{blocked: make(chan struct{})} + server, err := NewServer(ServerConfig{Session: SessionConfig{ + Backend: backend, Password: sessionFixturePassword(), ChallengeReader: &repeatReader{}, + }}) + if err != nil { + t.Fatal(err) + } + listener, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + serveDone := make(chan error, 1) + go func() { serveDone <- server.Serve(context.Background(), listener) }() + client, err := net.Dial("tcp", listener.Addr().String()) + if err != nil { + t.Fatal(err) + } + completeHandshake(t, client, sessionFixturePassword()) + init := readExactly(t, client, 24) + _ = readExactly(t, client, int(binary.BigEndian.Uint32(init[20:]))) + assertWrite(t, client, encodeSetEncodings([]int32{EncodingTight})) + assertWrite(t, client, []byte{3, 0, 0, 0, 0, 0, 0, 2, 0, 2}) + select { + case <-backend.blocked: + case <-time.After(time.Second): + t.Fatal("capture did not block") + } + closed := make(chan error, 1) + go func() { closed <- server.Close() }() + select { + case err := <-closed: + if err != nil { + t.Fatal(err) + } + case <-time.After(time.Second): + t.Fatal("server Close blocked behind capture") + } + _ = client.Close() + if err := <-serveDone; err != nil { + t.Fatal(err) + } + if err := server.Close(); err != nil { + t.Fatal(err) + } +} + +type repeatReader struct{} + +func (*repeatReader) Read(payload []byte) (int, error) { + for index := range payload { + payload[index] = byte(index) + } + return len(payload), nil +} diff --git a/internal/rfb/session.go b/internal/rfb/session.go new file mode 100644 index 00000000..241ce85b --- /dev/null +++ b/internal/rfb/session.go @@ -0,0 +1,446 @@ +package rfb + +import ( + "bytes" + "context" + "crypto/rand" + "encoding/binary" + "errors" + "fmt" + "io" + "net" + "time" + + "github.com/openclaw/crabfleet/internal/connect" +) + +const ( + defaultHandshakeTimeout = 10 * time.Second + defaultMediaTimeout = 5 * time.Second + defaultJPEGQuality = 80 + maximumPressedKeys = 32 +) + +type SessionConfig struct { + Backend connect.Backend + Password string + DesktopName string + ChallengeReader io.Reader + HandshakeTimeout time.Duration + MediaTimeout time.Duration + JPEGQuality int +} + +func (config SessionConfig) normalized() (SessionConfig, error) { + if config.Backend == nil { + return config, errors.New("RFB backend is required") + } + if config.Password == "" { + return config, errors.New("RFB password is required") + } + if _, err := vncKey(config.Password); err != nil { + return config, err + } + passwordLength := 0 + for range config.Password { + passwordLength++ + } + if passwordLength > 8 { + return config, errors.New("VNC passwords are limited to eight ISO-8859-1 characters") + } + if len([]byte(config.DesktopName)) > MaxDesktopName { + return config, errors.New("desktop name is too long") + } + if config.DesktopName == "" { + config.DesktopName = "Crabfleet Connect" + } + if config.ChallengeReader == nil { + config.ChallengeReader = rand.Reader + } + if config.HandshakeTimeout == 0 { + config.HandshakeTimeout = defaultHandshakeTimeout + } + if config.MediaTimeout == 0 { + config.MediaTimeout = defaultMediaTimeout + } + if config.JPEGQuality == 0 { + config.JPEGQuality = defaultJPEGQuality + } + if config.HandshakeTimeout < 0 || config.MediaTimeout < 0 || config.JPEGQuality < 1 || config.JPEGQuality > 100 { + return config, errors.New("invalid RFB session limits") + } + return config, nil +} + +func ServeConn(ctx context.Context, connection net.Conn, config SessionConfig) error { + config, err := config.normalized() + if err != nil { + return err + } + if connection == nil { + return errors.New("RFB connection is required") + } + stopWatch := make(chan struct{}) + go func() { + select { + case <-ctx.Done(): + _ = connection.Close() + case <-stopWatch: + } + }() + defer close(stopWatch) + + if config.HandshakeTimeout > 0 { + if err := connection.SetDeadline(time.Now().Add(config.HandshakeTimeout)); err != nil { + return err + } + } + handshakeContext, cancelHandshake := context.WithTimeout(ctx, config.HandshakeTimeout) + frame, err := handshake(handshakeContext, connection, config) + cancelHandshake() + if err != nil { + return fmt.Errorf("RFB handshake: %w", err) + } + if err := connection.SetDeadline(time.Time{}); err != nil { + return err + } + if err := ctx.Err(); err != nil { + return err + } + return messageLoop(ctx, connection, config, frame) +} + +func handshake(ctx context.Context, connection net.Conn, config SessionConfig) (connect.Frame, error) { + if err := writeFull(connection, Version38Banner); err != nil { + return connect.Frame{}, err + } + clientBanner := make([]byte, len(Version38Banner)) + if _, err := io.ReadFull(connection, clientBanner); err != nil { + return connect.Frame{}, err + } + if string(clientBanner) != string(Version38Banner) { + return connect.Frame{}, errors.New("unsupported RFB version") + } + if err := writeFull(connection, []byte{2, SecurityARD, SecurityVNC}); err != nil { + return connect.Frame{}, err + } + selection := []byte{0} + if _, err := io.ReadFull(connection, selection); err != nil { + return connect.Frame{}, err + } + if selection[0] == SecurityARD { + err := sendSecurityFailure(connection, "ARD host authentication is not implemented by Crabfleet Connect yet.") + if err != nil { + return connect.Frame{}, err + } + return connect.Frame{}, errors.New("ARD host authentication is deferred") + } + if selection[0] != SecurityVNC { + return connect.Frame{}, errors.New("unsupported RFB security selection") + } + challenge := make([]byte, 16) + if _, err := io.ReadFull(config.ChallengeReader, challenge); err != nil { + return connect.Frame{}, fmt.Errorf("generate VNC challenge: %w", err) + } + if err := writeFull(connection, challenge); err != nil { + return connect.Frame{}, err + } + response := make([]byte, 16) + if _, err := io.ReadFull(connection, response); err != nil { + return connect.Frame{}, err + } + accepted, err := VerifyVNCResponse(challenge, response, config.Password) + if err != nil { + return connect.Frame{}, err + } + if !accepted { + if err := sendSecurityFailure(connection, "Authentication failed."); err != nil { + return connect.Frame{}, err + } + return connect.Frame{}, errors.New("VNC authentication failed") + } + if err := writeFull(connection, []byte{0, 0, 0, 0}); err != nil { + return connect.Frame{}, err + } + clientInit := []byte{0} + if _, err := io.ReadFull(connection, clientInit); err != nil { + return connect.Frame{}, err + } + if clientInit[0] == 0 { + return connect.Frame{}, errors.New("exclusive ClientInit is not supported") + } + frame, err := config.Backend.Capture(ctx) + if err != nil { + return connect.Frame{}, fmt.Errorf("initial capture: %w", err) + } + if err := frame.Validate(); err != nil { + return connect.Frame{}, err + } + if _, err := EncodeJPEG(frame, config.JPEGQuality); err != nil { + return connect.Frame{}, fmt.Errorf("initial Tight JPEG: %w", err) + } + serverInit, err := ServerInit(frame.Width, frame.Height, config.DesktopName) + if err != nil { + return connect.Frame{}, err + } + if err := writeFull(connection, serverInit); err != nil { + return connect.Frame{}, err + } + return frame, nil +} + +func sendSecurityFailure(writer io.Writer, reason string) error { + reasonBytes := []byte(reason) + if len(reasonBytes) > MaxSecurityReason { + return errors.New("security failure reason is too long") + } + result := make([]byte, 8+len(reasonBytes)) + binary.BigEndian.PutUint32(result, 1) + binary.BigEndian.PutUint32(result[4:], uint32(len(reasonBytes))) + copy(result[8:], reasonBytes) + return writeFull(writer, result) +} + +func messageLoop(ctx context.Context, connection net.Conn, config SessionConfig, initialFrame connect.Frame) error { + width, height := initialFrame.Width, initialFrame.Height + var encodings Encodings + var negotiated bool + var lastCursorShape *connect.Cursor + pressedKeys := make(map[uint32]struct{}) + var lastPointer connect.PointerEvent + defer func() { releaseInput(config.Backend, pressedKeys, lastPointer) }() + for { + messageType := []byte{0} + if _, err := io.ReadFull(connection, messageType); err != nil { + return err + } + switch messageType[0] { + case 0: + if err := parseSetPixelFormat(connection); err != nil { + return err + } + case 2: + next, err := parseSetEncodings(connection) + if err != nil { + return err + } + encodings = next + negotiated = true + lastCursorShape = nil + case 3: + if _, err := parseFramebufferRequest(connection, width, height); err != nil { + return err + } + if !negotiated || !encodings.Tight { + return errors.New("the client did not offer Tight encoding") + } + frame, err := config.Backend.Capture(ctx) + if err != nil { + return fmt.Errorf("capture framebuffer: %w", err) + } + if err := frame.Validate(); err != nil { + return err + } + if frame.Width != width || frame.Height != height { + return errors.New("framebuffer size changed without resize negotiation") + } + payload, err := EncodeJPEG(frame, config.JPEGQuality) + if err != nil { + return err + } + video, err := tightJPEGRectangle(width, height, payload) + if err != nil { + return err + } + rectangles := [][]byte{video} + var nextCursorShape *connect.Cursor + if source, ok := config.Backend.(connect.CursorCapturer); ok && encodings.cursorEncoding() != 0 { + cursor, cursorErr := source.Cursor(ctx) + if cursorErr == nil { + if err := cursor.Validate(width, height); err != nil { + return err + } + if lastCursorShape == nil || !sameCursorShape(*lastCursorShape, cursor) { + shape, err := cursorRectangle(cursor, encodings.cursorEncoding()) + if err != nil { + return err + } + rectangles = append(rectangles, shape) + copy := cursor + copy.RGBA = append([]byte(nil), cursor.RGBA...) + nextCursorShape = © + } + if cursor.Visible && encodings.PointerPosition { + position, err := pointerPositionRectangle(cursor.X, cursor.Y) + if err != nil { + return err + } + rectangles = append(rectangles, position) + } + } + } + update, err := framebufferUpdate(rectangles...) + if err != nil { + return err + } + if err := writeMedia(connection, update, config.MediaTimeout); err != nil { + if !errors.Is(err, errMediaDropped) { + return err + } + // The media frame was dropped before any bytes reached the wire. + // Preserve request/response pacing with a legal empty update. If + // even that control response cannot be sent, framing cannot recover. + if err := writeMedia(connection, []byte{0, 0, 0, 0}, config.MediaTimeout); err != nil { + return fmt.Errorf("send empty update after media drop: %w", err) + } + continue + } + if nextCursorShape != nil { + lastCursorShape = nextCursorShape + } + case 4: + down, keysym, err := parseKeyEvent(connection) + if err != nil { + return err + } + if down { + if _, alreadyPressed := pressedKeys[keysym]; !alreadyPressed && len(pressedKeys) >= maximumPressedKeys { + return errors.New("too many simultaneously pressed keys") + } + } + if err := config.Backend.Key(ctx, connect.KeyEvent{Down: down, Keysym: keysym}); err != nil { + return fmt.Errorf("inject key: %w", err) + } + if down { + pressedKeys[keysym] = struct{}{} + } else { + delete(pressedKeys, keysym) + } + case 5: + mask, x, y, err := parsePointerEvent(connection, width, height) + if err != nil { + return err + } + if err := config.Backend.Pointer(ctx, connect.PointerEvent{ButtonMask: mask, X: x, Y: y}); err != nil { + return fmt.Errorf("inject pointer: %w", err) + } + lastPointer = connect.PointerEvent{ButtonMask: mask, X: x, Y: y} + case 6: + if err := consumeClientCutText(connection); err != nil { + return err + } + default: + return fmt.Errorf("unsupported client message %d", messageType[0]) + } + } +} + +func sameCursorShape(left, right connect.Cursor) bool { + return left.Visible == right.Visible && left.Width == right.Width && left.Height == right.Height && + left.HotspotX == right.HotspotX && left.HotspotY == right.HotspotY && bytes.Equal(left.RGBA, right.RGBA) +} + +func releaseInput(backend connect.InputSink, keys map[uint32]struct{}, pointer connect.PointerEvent) { + if releaser, ok := backend.(interface{ releaseSessionInput(context.Context) }); ok { + ctx, cancel := context.WithTimeout(context.Background(), 500*time.Millisecond) + defer cancel() + releaser.releaseSessionInput(ctx) + return + } + + deadline := time.Now().Add(500 * time.Millisecond) + for keysym := range keys { + for attempt := 0; attempt < 3; attempt++ { + remaining := time.Until(deadline) + if remaining <= 0 { + return + } + if remaining > 10*time.Millisecond { + remaining = 10 * time.Millisecond + } + ctx, cancel := context.WithTimeout(context.Background(), remaining) + err := backend.Key(ctx, connect.KeyEvent{Down: false, Keysym: keysym}) + cancel() + if err == nil { + break + } + } + } + if pointer.ButtonMask != 0 && time.Now().Before(deadline) { + pointer.ButtonMask = 0 + for attempt := 0; attempt < 3; attempt++ { + ctx, cancel := context.WithDeadline(context.Background(), deadline) + err := backend.Pointer(ctx, pointer) + cancel() + if err == nil { + break + } + } + } +} + +const ( + maxLegacyClipboardBytes = 1 * 1024 * 1024 + maxExtendedClipboardBody = 4 + maxLegacyClipboardBytes + 65_536 +) + +func consumeClientCutText(reader io.Reader) error { + header := make([]byte, 7) + if _, err := io.ReadFull(reader, header); err != nil { + return err + } + if header[0] != 0 || header[1] != 0 || header[2] != 0 { + return errors.New("invalid ClientCutText padding") + } + length := int64(int32(binary.BigEndian.Uint32(header[3:]))) + limit := int64(maxLegacyClipboardBytes) + if length < 0 { + length = -length + limit = maxExtendedClipboardBody + } + if length > limit { + return errors.New("ClientCutText payload is too large") + } + _, err := io.CopyN(io.Discard, reader, length) + return err +} + +var errMediaDropped = errors.New("media write deadline expired before transmission") + +func writeMedia(connection net.Conn, payload []byte, timeout time.Duration) error { + if timeout > 0 { + if err := connection.SetWriteDeadline(time.Now().Add(timeout)); err != nil { + return err + } + defer connection.SetWriteDeadline(time.Time{}) //nolint:errcheck // the write result remains authoritative + } + written := 0 + for written < len(payload) { + count, err := connection.Write(payload[written:]) + written += count + if err != nil { + if timeoutError, ok := err.(net.Error); ok && timeoutError.Timeout() && written == 0 { + return errMediaDropped + } + return err + } + if count == 0 { + return io.ErrShortWrite + } + } + return nil +} + +func writeFull(writer io.Writer, payload []byte) error { + for len(payload) > 0 { + count, err := writer.Write(payload) + if err != nil { + return err + } + if count == 0 { + return io.ErrShortWrite + } + payload = payload[count:] + } + return nil +} diff --git a/internal/rfb/session_test.go b/internal/rfb/session_test.go new file mode 100644 index 00000000..9585bd71 --- /dev/null +++ b/internal/rfb/session_test.go @@ -0,0 +1,239 @@ +package rfb + +import ( + "bytes" + "context" + "encoding/binary" + "errors" + "image/jpeg" + "io" + "net" + "testing" + "time" + + "github.com/openclaw/crabfleet/internal/connect" +) + +func TestSyntheticBackendEndToEnd(t *testing.T) { + t.Parallel() + backend, err := connect.NewSynthetic(connect.SyntheticOptions{Width: 32, Height: 18}) + if err != nil { + t.Fatal(err) + } + server, client := net.Pipe() + serverDone := make(chan error, 1) + go func() { + serverDone <- ServeConn(context.Background(), server, SessionConfig{ + Backend: backend, + Password: sessionFixturePassword(), + DesktopName: "Synthetic Linux", + ChallengeReader: bytes.NewReader(sequence(16)), + HandshakeTimeout: time.Second, + MediaTimeout: time.Second, + }) + }() + defer server.Close() + + assertRead(t, client, Version38Banner) + assertWrite(t, client, Version38Banner) + assertRead(t, client, []byte{2, SecurityARD, SecurityVNC}) + assertWrite(t, client, []byte{SecurityVNC}) + challenge := readExactly(t, client, 16) + response, err := VNCChallengeResponse(challenge, sessionFixturePassword()) + if err != nil { + t.Fatal(err) + } + assertWrite(t, client, response) + assertRead(t, client, []byte{0, 0, 0, 0}) + assertWrite(t, client, []byte{1}) + + serverInit := readExactly(t, client, 24) + if binary.BigEndian.Uint16(serverInit) != 32 || binary.BigEndian.Uint16(serverInit[2:]) != 18 { + t.Fatalf("server dimensions = %dx%d", binary.BigEndian.Uint16(serverInit), binary.BigEndian.Uint16(serverInit[2:])) + } + nameLength := binary.BigEndian.Uint32(serverInit[20:]) + assertRead(t, client, []byte("Synthetic Linux")[:nameLength]) + + assertWrite(t, client, encodeSetEncodings([]int32{ + 0x48455631, EncodingTight, EncodingCursorWithAlpha, EncodingPointerPosition, + })) + assertWrite(t, client, []byte{3, 0, 0, 0, 0, 0, 0, 32, 0, 18}) + + updateHeader := readExactly(t, client, 4) + if !bytes.Equal(updateHeader[:2], []byte{0, 0}) || binary.BigEndian.Uint16(updateHeader[2:]) != 3 { + t.Fatalf("update header = %x", updateHeader) + } + videoHeader := readExactly(t, client, 13) + if int32(binary.BigEndian.Uint32(videoHeader[8:])) != EncodingTight || videoHeader[12] != 0x90 { + t.Fatalf("video header = %x", videoHeader) + } + jpegLength := readCompactFromConn(t, client) + jpegPayload := readExactly(t, client, jpegLength) + if _, err := jpeg.Decode(bytes.NewReader(jpegPayload)); err != nil { + t.Fatalf("decode JPEG: %v", err) + } + cursorHeader := readExactly(t, client, 16) + if int32(binary.BigEndian.Uint32(cursorHeader[8:])) != EncodingCursorWithAlpha || binary.BigEndian.Uint32(cursorHeader[12:]) != 0 { + t.Fatalf("cursor header = %x", cursorHeader) + } + cursorWidth := int(binary.BigEndian.Uint16(cursorHeader[4:])) + cursorHeight := int(binary.BigEndian.Uint16(cursorHeader[6:])) + _ = readExactly(t, client, cursorWidth*cursorHeight*4) + pointerHeader := readExactly(t, client, 12) + if int32(binary.BigEndian.Uint32(pointerHeader[8:])) != EncodingPointerPosition { + t.Fatalf("pointer header = %x", pointerHeader) + } + + assertWrite(t, client, []byte{5, 1, 0, 7, 0, 8}) + assertWrite(t, client, []byte{4, 1, 0, 0, 0, 0, 0, 65}) + if err := client.Close(); err != nil { + t.Fatal(err) + } + if err := <-serverDone; err == nil || (!errors.Is(err, io.EOF) && !errors.Is(err, net.ErrClosed)) { + t.Fatalf("session result = %v", err) + } + events := backend.Events() + if len(events) != 4 || events[0].Pointer == nil || events[1].Key == nil || + events[2].Key == nil || events[2].Key.Down || events[3].Pointer == nil || events[3].Pointer.ButtonMask != 0 { + t.Fatalf("input events = %+v", events) + } +} + +func TestHandshakeRejectsSecurityNoneAndARDStub(t *testing.T) { + t.Parallel() + for _, selection := range []byte{1, SecurityARD} { + selection := selection + t.Run(string(rune(selection)), func(t *testing.T) { + t.Parallel() + backend, err := connect.NewSynthetic(connect.SyntheticOptions{Width: 4, Height: 4}) + if err != nil { + t.Fatal(err) + } + server, client := net.Pipe() + done := make(chan error, 1) + go func() { + done <- ServeConn(context.Background(), server, SessionConfig{ + Backend: backend, Password: "fake", HandshakeTimeout: time.Second, + }) + }() + assertRead(t, client, Version38Banner) + assertWrite(t, client, Version38Banner) + assertRead(t, client, []byte{2, SecurityARD, SecurityVNC}) + assertWrite(t, client, []byte{selection}) + if selection == SecurityARD { + status := readExactly(t, client, 4) + if binary.BigEndian.Uint32(status) != 1 { + t.Fatalf("ARD failure status = %x", status) + } + length := binary.BigEndian.Uint32(readExactly(t, client, 4)) + _ = readExactly(t, client, int(length)) + } + _ = client.Close() + if err := <-done; err == nil { + t.Fatal("security selection was accepted") + } + }) + } +} + +func TestSessionRejectsMalformedSetEncodings(t *testing.T) { + t.Parallel() + backend, err := connect.NewSynthetic(connect.SyntheticOptions{Width: 4, Height: 4}) + if err != nil { + t.Fatal(err) + } + server, client := net.Pipe() + done := make(chan error, 1) + go func() { + done <- ServeConn(context.Background(), server, SessionConfig{ + Backend: backend, Password: "fake", ChallengeReader: bytes.NewReader(sequence(16)), HandshakeTimeout: time.Second, + }) + }() + completeHandshake(t, client, "fake") + serverInit := readExactly(t, client, 24) + nameLength := binary.BigEndian.Uint32(serverInit[20:]) + _ = readExactly(t, client, int(nameLength)) + assertWrite(t, client, []byte{2, 0, 1, 1}) + _ = client.Close() + if err := <-done; err == nil { + t.Fatal("malformed SetEncodings was accepted") + } +} + +func TestSessionConfigurationRejectsLongVNCPassword(t *testing.T) { + t.Parallel() + backend, err := connect.NewSynthetic(connect.SyntheticOptions{Width: 4, Height: 4}) + if err != nil { + t.Fatal(err) + } + if _, err := (SessionConfig{Backend: backend, Password: forkFixturePassword()}).normalized(); err == nil { + t.Fatal("accepted VNC password longer than eight characters") + } +} + +func completeHandshake(t *testing.T, client net.Conn, password string) { + t.Helper() + assertRead(t, client, Version38Banner) + assertWrite(t, client, Version38Banner) + assertRead(t, client, []byte{2, SecurityARD, SecurityVNC}) + assertWrite(t, client, []byte{SecurityVNC}) + challenge := readExactly(t, client, 16) + response, err := VNCChallengeResponse(challenge, password) + if err != nil { + t.Fatal(err) + } + assertWrite(t, client, response) + assertRead(t, client, []byte{0, 0, 0, 0}) + assertWrite(t, client, []byte{1}) +} + +func assertRead(t *testing.T, reader io.Reader, expected []byte) { + t.Helper() + actual := readExactly(t, reader, len(expected)) + if !bytes.Equal(actual, expected) { + t.Fatalf("read %x, want %x", actual, expected) + } +} + +func assertWrite(t *testing.T, writer io.Writer, payload []byte) { + t.Helper() + if err := writeFull(writer, payload); err != nil { + t.Fatal(err) + } +} + +func readExactly(t *testing.T, reader io.Reader, count int) []byte { + t.Helper() + result := make([]byte, count) + if _, err := io.ReadFull(reader, result); err != nil { + t.Fatal(err) + } + return result +} + +func readCompactFromConn(t *testing.T, reader io.Reader) int { + t.Helper() + first := readExactly(t, reader, 1)[0] + result := int(first & 0x7f) + if first&0x80 == 0 { + return result + } + second := readExactly(t, reader, 1)[0] + result |= int(second&0x7f) << 7 + if second&0x80 == 0 { + return result + } + return result | int(readExactly(t, reader, 1)[0])<<14 +} + +func sequence(count int) []byte { + result := make([]byte, count) + for index := range result { + result[index] = byte(index) + } + return result +} + +func sessionFixturePassword() string { + return "1234" + "5678" +} From e5d4ced529020357d1d80fca210e428d532a9109 Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Sat, 18 Jul 2026 06:06:42 -0700 Subject: [PATCH 2/2] feat(connect): add Linux X11 host --- CHANGELOG.md | 1 + README.md | 19 + cmd/crabfleet-connect/README.md | 23 + cmd/crabfleet-connect/main.go | 130 ++++++ cmd/crabfleet-connect/main_test.go | 62 +++ docs/macos-native-client.md | 21 + docs/screen-recording-indicator.md | 16 +- go.mod | 4 +- go.sum | 2 + internal/connect/platform_linux.go | 7 + internal/connect/platform_other.go | 9 + internal/connect/x11_cursor.go | 42 ++ internal/connect/x11_cursor_test.go | 36 ++ internal/connect/x11_keymap.go | 358 +++++++++++++++ internal/connect/x11_keymap_test.go | 188 ++++++++ internal/connect/x11_linux.go | 656 ++++++++++++++++++++++++++++ internal/rfb/server.go | 15 +- internal/rfb/server_test.go | 19 +- 18 files changed, 1589 insertions(+), 19 deletions(-) create mode 100644 cmd/crabfleet-connect/README.md create mode 100644 cmd/crabfleet-connect/main.go create mode 100644 cmd/crabfleet-connect/main_test.go create mode 100644 internal/connect/platform_linux.go create mode 100644 internal/connect/platform_other.go create mode 100644 internal/connect/x11_cursor.go create mode 100644 internal/connect/x11_cursor_test.go create mode 100644 internal/connect/x11_keymap.go create mode 100644 internal/connect/x11_keymap_test.go create mode 100644 internal/connect/x11_linux.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 382b431a..8262f1e7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,7 @@ ## Unreleased +- Add the Crabfleet Connect foundation with a shared Go RFB 3.8 host core, per-run VNC-DES authentication, Tight/JPEG and client-side cursor/input support, a CI-safe synthetic backend, and a Linux X11 MIT-SHM/XFixes/XTest backend plus cross-compiled CLI, while documenting deferred codecs, Wayland, ARD, audio, and hardware validation. - Warn without blocking Share This Mac when Tailscale reports a numerically suffixed duplicate registration, while continuing to advertise the current `Self` node address. - Blend the macOS title bar into the desktop deck, align its unified top band, and add hover, focus, refresh-progress, and empty-state interaction polish. - Keep Share This Mac permission status and start availability synchronized with Screen Recording and Accessibility grants made in System Settings while the sheet is open. diff --git a/README.md b/README.md index 24887ca9..282a20d8 100644 --- a/README.md +++ b/README.md @@ -427,6 +427,25 @@ Full documentation available at [docs.crabfleet.ai](https://docs.crabfleet.ai): - [API](https://docs.crabfleet.ai/api) – REST and WebSocket APIs - [Spec](https://docs.crabfleet.ai/spec) – Complete product specification +## Crabfleet Connect for Linux + +`crabfleet-connect` is the first cross-platform host foundation for sharing a +Linux machine to the native macOS viewer or browser client. It provides a Go +RFB 3.8 server, a fresh per-run VNC password, Tight/JPEG frames, client-side +cursor updates, remote input, a synthetic test backend, and a Linux X11 +MIT-SHM/XFixes/XTest backend. The X11 backend cross-compiles in CI but has not +been validated on physical Linux hardware yet. + +The CLI listens on loopback by default because VNC-DES does not encrypt RFB +traffic. Remote use requires an explicit `--bind` on an already protected +private path. + +This increment does not provide ARD host authentication, H.264 or HEVC +encoding, Wayland/PipeWire capture, multi-group XKB input, audio, clipboard +synchronization, or service packaging. See +[`cmd/crabfleet-connect/README.md`](cmd/crabfleet-connect/README.md) +for the exact boundary and run command. + ## Security - All state-changing operations require authentication diff --git a/cmd/crabfleet-connect/README.md b/cmd/crabfleet-connect/README.md new file mode 100644 index 00000000..febf8fbe --- /dev/null +++ b/cmd/crabfleet-connect/README.md @@ -0,0 +1,23 @@ +# Crabfleet Connect foundation + +`crabfleet-connect` is the first Linux host foundation for sharing a machine to Crabfleet's macOS viewer or browser client. + +Real in this increment: + +- RFB 3.8 server handshake and direct-listener VNC-DES authentication with an eight-character per-run share password; Security None is never offered. +- Tight/JPEG full-frame updates, client-side cursor pseudo-encodings, pointer/key input, strict protocol bounds, bounded sessions, and idempotent teardown. +- A pure-Go synthetic capture/input backend used by tests and as an explicit fallback. +- A Linux X11 backend implemented with MIT-SHM `XShmGetImage` capture, XFixes cursor images, and XTest input with key-level modifier mapping. It compiles for Linux; it has not been validated on physical Linux hardware in this track. Multi-group XKB layouts fail closed to the synthetic fallback rather than risking incorrect input. + +Deferred: + +- ARD host authentication. Type 30 remains in the Track H-compatible security offer but fails closed; VNC-DES type 2 is the working MVP. +- H.264/HEVC hardware encoding, Wayland/PipeWire capture, multi-group XKB input, audio, clipboard synchronization, packaging/service installation, and real-hardware validation. + +Run: + +```sh +go run ./cmd/crabfleet-connect --display :0 --port 5900 +``` + +The process prints its per-run share password and listener address. If native capture cannot initialize, it says why and uses the synthetic test pattern. Use `--synthetic` to force that backend. The listener defaults to `127.0.0.1` because VNC-DES authenticates but does not encrypt the RFB session. Use `--bind` to select a private interface only when the network path is already protected, such as through an authenticated tunnel; `--bind 0.0.0.0` is an explicit insecure exposure. diff --git a/cmd/crabfleet-connect/main.go b/cmd/crabfleet-connect/main.go new file mode 100644 index 00000000..2ffa1ff0 --- /dev/null +++ b/cmd/crabfleet-connect/main.go @@ -0,0 +1,130 @@ +package main + +import ( + "context" + "crypto/rand" + "errors" + "flag" + "fmt" + "io" + "net" + "os" + "os/signal" + "strconv" + "strings" + "syscall" + + "github.com/openclaw/crabfleet/internal/connect" + "github.com/openclaw/crabfleet/internal/rfb" +) + +var version = "dev" + +func main() { + ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) + defer stop() + if err := run(ctx, os.Args[1:], os.Stdout, os.Stderr); err != nil { + fmt.Fprintln(os.Stderr, "crabfleet-connect:", err) + os.Exit(1) + } +} + +func run(ctx context.Context, arguments []string, stdout, stderr io.Writer) error { + flags := flag.NewFlagSet("crabfleet-connect", flag.ContinueOnError) + flags.SetOutput(stderr) + display := flags.String("display", "", "X11 display to capture (defaults to DISPLAY)") + bind := flags.String("bind", "127.0.0.1", "listener address; use a private interface explicitly for remote access") + port := flags.Int("port", 5900, "TCP port for the direct RFB listener") + synthetic := flags.Bool("synthetic", false, "force the synthetic test-pattern backend") + showVersion := flags.Bool("version", false, "print version and exit") + if err := flags.Parse(arguments); err != nil { + if errors.Is(err, flag.ErrHelp) { + return nil + } + return err + } + if flags.NArg() != 0 { + return fmt.Errorf("unexpected arguments: %v", flags.Args()) + } + if *showVersion { + fmt.Fprintf(stdout, "crabfleet-connect %s\n", version) + return nil + } + if *port < 1 || *port > 65_535 { + return errors.New("port must be between 1 and 65535") + } + if strings.TrimSpace(*bind) == "" { + return errors.New("bind address must not be empty") + } + + backend, description, err := selectBackend(*synthetic, *display, stderr) + if err != nil { + return err + } + password, err := generateSharePassword() + if err != nil { + _ = backend.Close() + return err + } + hostname, err := os.Hostname() + if err != nil || hostname == "" { + hostname = "Linux" + } + server, err := rfb.NewServer(rfb.ServerConfig{Session: rfb.SessionConfig{ + Backend: backend, + Password: password, + DesktopName: "Crabfleet Connect (" + hostname + ")", + }}) + if err != nil { + _ = backend.Close() + return err + } + listener, err := net.Listen("tcp", net.JoinHostPort(*bind, strconv.Itoa(*port))) + if err != nil { + _ = server.Close() + return fmt.Errorf("listen on port %d: %w", *port, err) + } + fmt.Fprintf(stdout, "Crabfleet Connect %s\n", version) + fmt.Fprintf(stdout, "Backend: %s\n", description) + fmt.Fprintf(stdout, "Listening: %s\n", listener.Addr()) + fmt.Fprintf(stdout, "Share password: %s\n", password) + return server.Serve(ctx, listener) +} + +func selectBackend(forceSynthetic bool, display string, stderr io.Writer) (connect.Backend, string, error) { + if !forceSynthetic { + backend, err := connect.NewPlatformBackend(display) + if err == nil { + return backend, "Linux X11 (MIT-SHM capture + XTest input)", nil + } + fmt.Fprintf(stderr, "Native capture unavailable (%v); using synthetic test pattern.\n", err) + } + backend, err := connect.NewSynthetic(connect.SyntheticOptions{}) + if err != nil { + return nil, "", fmt.Errorf("create synthetic backend: %w", err) + } + return backend, "synthetic test pattern", nil +} + +func generateSharePassword() (string, error) { + const alphabet = "ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz23456789" + const length = 8 + result := make([]byte, 0, length) + limit := byte(256 - (256 % len(alphabet))) + buffer := make([]byte, 32) + for len(result) < length { + if _, err := io.ReadFull(rand.Reader, buffer); err != nil { + return "", fmt.Errorf("generate share password: %w", err) + } + for _, value := range buffer { + if value >= limit { + continue + } + result = append(result, alphabet[int(value)%len(alphabet)]) + if len(result) == length { + break + } + } + } + return string(result), nil +} diff --git a/cmd/crabfleet-connect/main_test.go b/cmd/crabfleet-connect/main_test.go new file mode 100644 index 00000000..6679b576 --- /dev/null +++ b/cmd/crabfleet-connect/main_test.go @@ -0,0 +1,62 @@ +package main + +import ( + "bytes" + "context" + "strings" + "testing" +) + +func TestVersion(t *testing.T) { + t.Parallel() + var stdout, stderr bytes.Buffer + if err := run(context.Background(), []string{"--version"}, &stdout, &stderr); err != nil { + t.Fatal(err) + } + if stdout.String() != "crabfleet-connect dev\n" || stderr.Len() != 0 { + t.Fatalf("stdout=%q stderr=%q", stdout.String(), stderr.String()) + } +} + +func TestHelpIsSuccessful(t *testing.T) { + t.Parallel() + var stdout, stderr bytes.Buffer + if err := run(context.Background(), []string{"--help"}, &stdout, &stderr); err != nil { + t.Fatal(err) + } + if stdout.Len() != 0 || !strings.Contains(stderr.String(), "Usage of crabfleet-connect:") { + t.Fatalf("stdout=%q stderr=%q", stdout.String(), stderr.String()) + } +} + +func TestGenerateSharePassword(t *testing.T) { + t.Parallel() + first, err := generateSharePassword() + if err != nil { + t.Fatal(err) + } + second, err := generateSharePassword() + if err != nil { + t.Fatal(err) + } + const alphabet = "ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz23456789" + if len(first) != 8 || strings.Trim(first, alphabet) != "" || first == second { + t.Fatalf("generated passwords %q and %q", first, second) + } +} + +func TestRejectsInvalidPort(t *testing.T) { + t.Parallel() + var stdout, stderr bytes.Buffer + if err := run(context.Background(), []string{"--port", "0"}, &stdout, &stderr); err == nil { + t.Fatal("accepted invalid port") + } +} + +func TestRejectsEmptyBindAddress(t *testing.T) { + t.Parallel() + var stdout, stderr bytes.Buffer + if err := run(context.Background(), []string{"--bind="}, &stdout, &stderr); err == nil { + t.Fatal("accepted empty bind address") + } +} diff --git a/docs/macos-native-client.md b/docs/macos-native-client.md index 887f57d9..64235b82 100644 --- a/docs/macos-native-client.md +++ b/docs/macos-native-client.md @@ -58,6 +58,27 @@ full UTF-8 text with servers that negotiate it; against servers without the extension, standard cut text must encode losslessly as ISO-8859-1 and unsupported text is rejected instead of silently becoming empty data. +## Linux Connect foundation + +The native viewer can also connect directly to the first +`crabfleet-connect` Linux host foundation. That Go host speaks RFB 3.8 with a +fresh per-run VNC-DES password, Tight/JPEG full-frame updates, client-side +cursor rectangles, and pointer/key input. Its synthetic backend provides the +portable CI and protocol-test path. The Linux-only backend implements X11 +capture with MIT-SHM `XShmGetImage`, cursor images with XFixes, and input with +XTest; it cross-compiles for amd64 and arm64 but has not been exercised on +physical Linux hardware in this increment. + +The Connect listener defaults to loopback because VNC-DES does not encrypt RFB +traffic. A remote listener requires an explicit private bind on a separately +protected network path. + +The Linux host still advertises the direct-listener ARD security type for wire +compatibility, but ARD host authentication fails closed and viewers must select +VNC password authentication. H.264/HEVC encoding, Wayland/PipeWire, audio, +multi-group XKB input, clipboard synchronization, service packaging, and +real-hardware validation remain follow-up work. + ## Share This Mac The host path is deliberately app-owned. It does not start, configure, proxy, diff --git a/docs/screen-recording-indicator.md b/docs/screen-recording-indicator.md index c7604f4c..830cc667 100644 --- a/docs/screen-recording-indicator.md +++ b/docs/screen-recording-indicator.md @@ -10,13 +10,13 @@ Sonoma, **blue on macOS 26 Tahoe**) and why macOS periodically re-prompts for sc For any third-party app running an `SCStream`, this is **mandatory and unsuppressible** — there is no app-side API (`SCStreamConfiguration`/`SCContentFilter` do not affect it), and it is independent of display-vs-window capture, audio capture, or picker choice. It exists as a - privacy guarantee. The only Apple-sanctioned hide is the *external-display* exemption + privacy guarantee. The only Apple-sanctioned hide is the _external-display_ exemption (`system-override suppress-sw-camera-indication-on-external-displays=on`, Apple Support 118449), which does not cover a host sharing its built-in/primary display. 2. **The recurring "…bypass the system private window picker and directly access your screen and - audio" prompt** — macOS 15+ periodic TCC re-authorization for SCK used *without* the system - `SCContentSharingPicker`. This is **not** an indicator; it's a consent nag, and it *is* + audio" prompt** — macOS 15+ periodic TCC re-authorization for SCK used _without_ the system + `SCContentSharingPicker`. This is **not** an indicator; it's a consent nag, and it _is_ solvable (below). Do not conflate the two: the prompt is fixable, the indicator largely is not for third-party apps. @@ -28,18 +28,18 @@ the third-party indicator. Jump Desktop appears exempt because its capture lands **Remote Desktop / Remote Management** TCC class (`kTCCServiceRemoteDesktop`) — the same unattended-access bucket as Apple's Screen Sharing — rather than plain "Screen Recording". Jump also holds Apple's restricted entitlement **`com.apple.developer.persistent-content-capture`**, -which removes the recurring re-auth prompt for VNC-style apps. (Jump migrated *toward* SCK, not +which removes the recurring re-auth prompt for VNC-style apps. (Jump migrated _toward_ SCK, not away from it; the legacy `CGDisplayStream`/`CGWindowListCreateImage` path is deprecated and -triggers *more* consent alerts on Sonoma+.) +triggers _more_ consent alerts on Sonoma+.) -**Unverified:** whether the Remote Desktop grant actually removes the *indicator* on macOS 26 for +**Unverified:** whether the Remote Desktop grant actually removes the _indicator_ on macOS 26 for a third-party app. The prompt removal is documented; the indicator removal is a plausible side effect of the permission class but must be confirmed on-device before relying on it. ## Options for Crabfleet (ranked) 1. **`persistent-content-capture` entitlement + Remote Desktop grant** — the real "Jump playbook". - Removes the recurring prompt; may drop the indicator (verify on-device). Cost: Apple-*gated* + Removes the recurring prompt; may drop the indicator (verify on-device). Cost: Apple-_gated_ restricted entitlement (applied for per signing identity — **open-source status does not waive this**), a dedicated App ID, per-executable provisioning profiles, notarization, and users must grant under Privacy → Remote Desktop, not Screen Recording. Medium effort, external approval @@ -47,7 +47,7 @@ side effect of the permission class but must be confirmed on-device before relyi 2. **Stay on SCK, adopt `SCContentSharingPicker`** — removes the "bypass the private window picker" prompt with no entitlement, but adds a picker step (undesirable for an unattended host) and does **not** remove the indicator. Low effort; honest fallback. -3. **Legacy `CGDisplayStream`/`CGWindowListCreateImage`** — rejected: deprecated, *more* prompts, +3. **Legacy `CGDisplayStream`/`CGWindowListCreateImage`** — rejected: deprecated, _more_ prompts, still lights the indicator. 4. **System extension / DriverKit virtual display** — could sidestep the built-in-display indicator by capturing a synthetic display, but very high cost (system-extension approval, DriverKit diff --git a/go.mod b/go.mod index 39b92ae9..66f19d12 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,7 @@ go 1.25.0 require ( github.com/alecthomas/kong v1.15.0 github.com/coder/websocket v1.8.15 + github.com/jezek/xgb v1.3.1 golang.org/x/crypto v0.54.0 + golang.org/x/sys v0.47.0 ) - -require golang.org/x/sys v0.47.0 // indirect diff --git a/go.sum b/go.sum index 83b76c50..567f6594 100644 --- a/go.sum +++ b/go.sum @@ -8,6 +8,8 @@ github.com/coder/websocket v1.8.15 h1:6B2JPeOGlpff2Uz6vOEH1Vzpi0iUz20A+lPVhPHtNU github.com/coder/websocket v1.8.15/go.mod h1:NX3SzP+inril6yawo5CQXx8+fk145lPDC6pumgx0mVg= github.com/hexops/gotextdiff v1.0.3 h1:gitA9+qJrrTCsiCl7+kh75nPqQt1cx4ZkudSTLoUqJM= github.com/hexops/gotextdiff v1.0.3/go.mod h1:pSWU5MAI3yDq+fZBTazCSJysOMbxWL1BSow5/V2vxeg= +github.com/jezek/xgb v1.3.1 h1:NQCAEfQyzN+3RjWUSHBuVIxQcy2YfG3/mNvKfs/0rEg= +github.com/jezek/xgb v1.3.1/go.mod h1:nrhwO0FX/enq75I7Y7G8iN1ubpSGZEiA3v9e9GyRFlk= golang.org/x/crypto v0.54.0 h1:YLIA59K4fiNzHzjnZt2tUJQjQtUWfWbeHBqKtk3eScw= golang.org/x/crypto v0.54.0/go.mod h1:KWL8ny2AZdGR2cWmzeHrp2azQPGogOv+HeQaVEXC2dk= golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs= diff --git a/internal/connect/platform_linux.go b/internal/connect/platform_linux.go new file mode 100644 index 00000000..c3030a86 --- /dev/null +++ b/internal/connect/platform_linux.go @@ -0,0 +1,7 @@ +//go:build linux + +package connect + +func NewPlatformBackend(display string) (Backend, error) { + return NewLinuxX11(display) +} diff --git a/internal/connect/platform_other.go b/internal/connect/platform_other.go new file mode 100644 index 00000000..bf979b0e --- /dev/null +++ b/internal/connect/platform_other.go @@ -0,0 +1,9 @@ +//go:build !linux + +package connect + +import "errors" + +func NewPlatformBackend(string) (Backend, error) { + return nil, errors.New("no native capture backend is available on this platform") +} diff --git a/internal/connect/x11_cursor.go b/internal/connect/x11_cursor.go new file mode 100644 index 00000000..2a954bd3 --- /dev/null +++ b/internal/connect/x11_cursor.go @@ -0,0 +1,42 @@ +package connect + +import "errors" + +func cursorFromXFixes( + x, y, width, height, hotspotX, hotspotY int, + pixels []uint32, + frameWidth, frameHeight int, +) (Cursor, error) { + if width < 1 || width > 128 || height < 1 || height > 128 || + hotspotX < 0 || hotspotX >= width || hotspotY < 0 || hotspotY >= height || + len(pixels) != width*height || frameWidth < 1 || frameHeight < 1 { + return Cursor{}, errors.New("XFixes returned invalid cursor geometry") + } + rgba := make([]byte, len(pixels)*4) + visible := false + for index, pixel := range pixels { + alpha := byte(pixel >> 24) + red := min(byte(pixel>>16), alpha) + green := min(byte(pixel>>8), alpha) + blue := min(byte(pixel), alpha) + offset := index * 4 + rgba[offset] = red + rgba[offset+1] = green + rgba[offset+2] = blue + rgba[offset+3] = alpha + visible = visible || alpha != 0 + } + if !visible { + return Cursor{Visible: false}, nil + } + return Cursor{ + Width: width, + Height: height, + HotspotX: hotspotX, + HotspotY: hotspotY, + X: max(0, min(x, frameWidth-1)), + Y: max(0, min(y, frameHeight-1)), + Visible: true, + RGBA: rgba, + }, nil +} diff --git a/internal/connect/x11_cursor_test.go b/internal/connect/x11_cursor_test.go new file mode 100644 index 00000000..0c1f32be --- /dev/null +++ b/internal/connect/x11_cursor_test.go @@ -0,0 +1,36 @@ +package connect + +import "testing" + +func TestCursorFromXFixesConvertsPremultipliedARGB(t *testing.T) { + t.Parallel() + cursor, err := cursorFromXFixes(20, 30, 2, 1, 1, 0, []uint32{0x80402010, 0xff112233}, 100, 100) + if err != nil { + t.Fatal(err) + } + if !cursor.Visible || cursor.HotspotX != 1 || cursor.X != 20 || cursor.Y != 30 { + t.Fatalf("cursor geometry = %+v", cursor) + } + want := []byte{0x40, 0x20, 0x10, 0x80, 0x11, 0x22, 0x33, 0xff} + if string(cursor.RGBA) != string(want) { + t.Fatalf("RGBA = %x, want %x", cursor.RGBA, want) + } +} + +func TestCursorFromXFixesTreatsTransparentImageAsHidden(t *testing.T) { + t.Parallel() + cursor, err := cursorFromXFixes(0, 0, 1, 1, 0, 0, []uint32{0x00ffffff}, 1, 1) + if err != nil { + t.Fatal(err) + } + if cursor.Visible { + t.Fatal("transparent cursor was visible") + } +} + +func TestCursorFromXFixesRejectsOversizedCursor(t *testing.T) { + t.Parallel() + if _, err := cursorFromXFixes(0, 0, 129, 1, 0, 0, make([]uint32, 129), 640, 480); err == nil { + t.Fatal("accepted oversized XFixes cursor") + } +} diff --git a/internal/connect/x11_keymap.go b/internal/connect/x11_keymap.go new file mode 100644 index 00000000..1e3ed0f2 --- /dev/null +++ b/internal/connect/x11_keymap.go @@ -0,0 +1,358 @@ +package connect + +import ( + "errors" + "unicode" +) + +const ( + x11KeysymModeSwitch = 0xff7e + x11KeysymNumLock = 0xff7f + x11KeysymCapsLock = 0xffe5 + x11KeysymShiftLock = 0xffe6 + x11KeysymISOLevel3Shift = 0xfe03 +) + +type x11KeyBinding struct { + keycode byte + shift bool + mode bool + lockSensitive bool + shiftSensitive bool + keypad bool +} + +type x11LockMode byte + +const ( + x11LockNone x11LockMode = iota + x11LockCaps + x11LockShift +) + +type x11ModifierMap [8][]byte + +type x11Keymap struct { + bindings map[uint32]x11KeyBinding + modifierKeys map[byte]struct{} + modifierMasks map[byte]uint16 + shiftKeycodes map[byte]struct{} + modeKeycodes map[byte]struct{} + preferredShift byte + preferredMode byte + modeMask uint16 + numLockMask uint16 + lockMode x11LockMode +} + +func buildX11Keymap( + keysyms []uint32, + minimum byte, + count, perKeycode int, + modifiers x11ModifierMap, +) (x11Keymap, error) { + if count < 1 || perKeycode < 1 || len(keysyms) != count*perKeycode { + return x11Keymap{}, errors.New("invalid X11 keymap dimensions") + } + result := x11Keymap{ + bindings: make(map[uint32]x11KeyBinding), + modifierKeys: make(map[byte]struct{}), + modifierMasks: make(map[byte]uint16), + shiftKeycodes: make(map[byte]struct{}), + modeKeycodes: make(map[byte]struct{}), + } + symbolsByKeycode := make(map[byte][]uint32, count) + for keyOffset := 0; keyOffset < count; keyOffset++ { + keycode := byte(int(minimum) + keyOffset) + rawLevels := keysyms[keyOffset*perKeycode : (keyOffset+1)*perKeycode] + if !x11GroupsEquivalent(rawLevels) { + return x11Keymap{}, errors.New("X11 multiple keyboard groups require XKB support") + } + levels := normalizedX11Levels(rawLevels) + symbolsByKeycode[keycode] = levels + for level, keysym := range levels[:2] { + if keysym == 0 { + continue + } + pair := level &^ 1 + binding := x11KeyBinding{ + keycode: keycode, + shift: level%2 == 1, + mode: level >= 2, + lockSensitive: isX11CasePair(levels[pair], levels[pair+1]), + shiftSensitive: levels[pair] != levels[pair+1], + keypad: isX11Keypad(keysym), + } + previous, exists := result.bindings[keysym] + if !exists || bindingCost(binding) < bindingCost(previous) { + result.bindings[keysym] = binding + } + } + } + for _, keycodes := range modifiers { + for _, keycode := range keycodes { + if keycode != 0 { + result.modifierKeys[keycode] = struct{}{} + } + } + } + for modifier, keycodes := range modifiers { + for _, keycode := range keycodes { + if keycode != 0 { + result.modifierMasks[keycode] |= 1 << modifier + } + } + } + for _, keycode := range modifiers[0] { + if keycode == 0 { + continue + } + result.shiftKeycodes[keycode] = struct{}{} + if result.preferredShift == 0 { + result.preferredShift = keycode + } + } + for _, keycode := range modifiers[1] { + for _, keysym := range symbolsByKeycode[keycode] { + switch keysym { + case x11KeysymShiftLock: + if result.lockMode == x11LockNone { + result.lockMode = x11LockShift + } + case x11KeysymCapsLock: + result.lockMode = x11LockCaps + } + } + } + for modifier := 3; modifier < len(modifiers); modifier++ { + var modeKeycode byte + for _, keycode := range modifiers[modifier] { + if keycode != 0 && containsX11Keysym(symbolsByKeycode[keycode], x11KeysymModeSwitch, x11KeysymISOLevel3Shift) { + modeKeycode = keycode + break + } + } + if modeKeycode != 0 { + for _, keycode := range modifiers[modifier] { + if keycode != 0 { + result.modeKeycodes[keycode] = struct{}{} + } + } + result.modeMask |= 1 << modifier + if result.preferredMode == 0 { + result.preferredMode = modeKeycode + } + } + for _, keycode := range modifiers[modifier] { + if containsX11Keysym(symbolsByKeycode[keycode], x11KeysymNumLock) { + result.numLockMask |= 1 << modifier + break + } + } + } + for keysym, binding := range result.bindings { + _, modifier := result.modifierKeys[binding.keycode] + if (binding.shift && result.preferredShift == 0) || (binding.mode && result.preferredMode == 0) || + (modifier && (binding.shift || binding.mode)) { + delete(result.bindings, keysym) + } + } + return result, nil +} + +func normalizedX11Levels(raw []uint32) []uint32 { + levels := make([]uint32, 4) + copy(levels, raw) + levels[0], levels[1] = normalizedX11Pair(levels[:2]) + levels[2], levels[3] = normalizedX11Pair(levels[2:]) + if levels[2] == 0 && levels[3] == 0 { + levels[2], levels[3] = levels[0], levels[1] + } + return levels +} + +func x11GroupsEquivalent(raw []uint32) bool { + baseLower, baseUpper := normalizedX11Pair(raw) + for offset := 2; offset < len(raw); offset += 2 { + end := min(offset+2, len(raw)) + lower, upper := normalizedX11Pair(raw[offset:end]) + if lower == 0 && upper == 0 { + continue + } + if lower != baseLower || upper != baseUpper { + return false + } + } + return true +} + +func normalizedX11Pair(raw []uint32) (uint32, uint32) { + if len(raw) == 0 { + return 0, 0 + } + if raw[0] == 0 { + if len(raw) > 1 { + return 0, raw[1] + } + return 0, 0 + } + if len(raw) < 2 || raw[1] == 0 { + return x11ConvertCase(raw[0]) + } + return raw[0], raw[1] +} + +// x11ConvertCase mirrors Xlib XConvertCase for the legacy sets it supports and +// uses Go's Unicode tables for Latin-1 and UCS keysyms. +func x11ConvertCase(keysym uint32) (uint32, uint32) { + if keysym < 0x100 { + switch keysym { + case 0xff: + return keysym, 0x13be + case 0xb5: + return keysym, 0x07cc + case 0xdf: + return keysym, 0x01001e9e + } + return uint32(unicode.ToLower(rune(keysym))), uint32(unicode.ToUpper(rune(keysym))) + } + if keysym&0xff000000 == 0x01000000 { + value := rune(keysym & 0x00ffffff) + if value > unicode.MaxRune { + return keysym, keysym + } + lower, upper := uint32(unicode.ToLower(value)), uint32(unicode.ToUpper(value)) + if lower >= 0x100 { + lower |= 0x01000000 + } + if upper >= 0x100 { + upper |= 0x01000000 + } + return lower, upper + } + lower, upper := keysym, keysym + switch keysym >> 8 { + case 1: + switch { + case keysym == 0x01a1: + lower = 0x01b1 + case keysym >= 0x01a3 && keysym <= 0x01a6, keysym >= 0x01a9 && keysym <= 0x01ac, + keysym >= 0x01ae && keysym <= 0x01af: + lower += 0x10 + case keysym == 0x01b1: + upper = 0x01a1 + case keysym >= 0x01b3 && keysym <= 0x01b6, keysym >= 0x01b9 && keysym <= 0x01bc, + keysym >= 0x01be && keysym <= 0x01bf: + upper -= 0x10 + case keysym >= 0x01c0 && keysym <= 0x01de: + lower += 0x20 + case keysym >= 0x01e0 && keysym <= 0x01fe: + upper -= 0x20 + } + case 2: + switch { + case keysym >= 0x02a1 && keysym <= 0x02a6, keysym >= 0x02ab && keysym <= 0x02ac: + lower += 0x10 + case keysym >= 0x02b1 && keysym <= 0x02b6, keysym >= 0x02bb && keysym <= 0x02bc: + upper -= 0x10 + case keysym >= 0x02c5 && keysym <= 0x02de: + lower += 0x20 + case keysym >= 0x02e5 && keysym <= 0x02fe: + upper -= 0x20 + } + case 3: + switch { + case keysym >= 0x03a3 && keysym <= 0x03ac: + lower += 0x10 + case keysym >= 0x03b3 && keysym <= 0x03bc: + upper -= 0x10 + case keysym == 0x03bd: + lower = 0x03bf + case keysym == 0x03bf: + upper = 0x03bd + case keysym >= 0x03c0 && keysym <= 0x03de: + lower += 0x20 + case keysym >= 0x03e0 && keysym <= 0x03fe: + upper -= 0x20 + } + case 6: + switch { + case keysym >= 0x06b1 && keysym <= 0x06bf: + lower -= 0x10 + case keysym >= 0x06a1 && keysym <= 0x06af: + upper += 0x10 + case keysym >= 0x06e0 && keysym <= 0x06ff: + lower -= 0x20 + case keysym >= 0x06c0 && keysym <= 0x06df: + upper += 0x20 + } + case 7: + switch { + case keysym >= 0x07a1 && keysym <= 0x07ab: + lower += 0x10 + case keysym >= 0x07b1 && keysym <= 0x07bb && keysym != 0x07b6 && keysym != 0x07ba: + upper -= 0x10 + case keysym >= 0x07c1 && keysym <= 0x07d9: + lower += 0x20 + case keysym == 0x07f3: + upper = 0x07d2 + case keysym >= 0x07e1 && keysym <= 0x07f9: + upper -= 0x20 + } + case 0x13: + switch keysym { + case 0x13bc: + lower = 0x13bd + case 0x13bd: + upper = 0x13bc + case 0x13be: + lower = 0xff + } + } + return lower, upper +} + +func isX11CasePair(lower, upper uint32) bool { + if lower == 0 || upper == 0 || lower == upper { + return false + } + convertedLower, convertedUpper := x11ConvertCase(lower) + return convertedLower == lower && convertedUpper == upper +} + +func containsX11Keysym(keysyms []uint32, wanted ...uint32) bool { + for _, keysym := range keysyms { + for _, candidate := range wanted { + if keysym == candidate { + return true + } + } + } + return false +} + +func isX11Keypad(keysym uint32) bool { + return keysym >= 0xff80 && keysym <= 0xffbd +} + +func requiredX11PhysicalShift( + binding x11KeyBinding, + lockActive bool, + lockMode x11LockMode, + numLockActive bool, +) bool { + lockShifts := lockMode == x11LockShift || (lockMode == x11LockCaps && binding.lockSensitive) + return binding.shift != (lockActive && lockShifts) != + (binding.keypad && binding.shiftSensitive && numLockActive) +} + +func bindingCost(binding x11KeyBinding) int { + cost := 0 + if binding.shift { + cost++ + } + if binding.mode { + cost += 2 + } + return cost +} diff --git a/internal/connect/x11_keymap_test.go b/internal/connect/x11_keymap_test.go new file mode 100644 index 00000000..5f8bebd5 --- /dev/null +++ b/internal/connect/x11_keymap_test.go @@ -0,0 +1,188 @@ +package connect + +import "testing" + +func TestBuildX11KeymapPreservesRequiredLevels(t *testing.T) { + t.Parallel() + var modifiers x11ModifierMap + modifiers[0] = []byte{50} + modifiers[4] = []byte{54} + modifiers[5] = []byte{51, 53} + keymap, err := buildX11Keymap([]uint32{ + 0xffe1, 0, 0, 0, + x11KeysymModeSwitch, 0, 0, 0, + 'a', 'A', 'a', 'A', + 0xffe9, 0, 0, 0, + x11KeysymNumLock, 0, 0, 0, + }, 50, 5, 4, modifiers) + if err != nil { + t.Fatal(err) + } + checks := map[uint32]x11KeyBinding{ + 'a': {keycode: 52, lockSensitive: true, shiftSensitive: true}, + 'A': {keycode: 52, shift: true, lockSensitive: true, shiftSensitive: true}, + } + for keysym, expected := range checks { + if actual := keymap.bindings[keysym]; actual != expected { + t.Fatalf("keysym %#x binding = %+v, want %+v", keysym, actual, expected) + } + } + if keymap.preferredShift != 50 || keymap.preferredMode != 51 || keymap.modeMask != 1<<5 || keymap.numLockMask != 1<<4 { + t.Fatalf("modifier keycodes = %d/%d", keymap.preferredShift, keymap.preferredMode) + } + if _, exists := keymap.modeKeycodes[53]; !exists { + t.Fatal("did not include another keycode sharing the mode modifier slot") + } +} + +func TestBuildX11KeymapRejectsMultipleGroups(t *testing.T) { + t.Parallel() + if _, err := buildX11Keymap([]uint32{'a', 'A', 0x06c1, 0x06e1}, 20, 1, 4, x11ModifierMap{}); err == nil { + t.Fatal("accepted an XKB multi-group keymap") + } +} + +func TestBuildX11KeymapRejectsThirdGroup(t *testing.T) { + t.Parallel() + if _, err := buildX11Keymap( + []uint32{'a', 'A', 'a', 'A', 0x06c1, 0x06e1}, + 20, 1, 6, x11ModifierMap{}, + ); err == nil { + t.Fatal("accepted a differing third XKB group") + } +} + +func TestBuildX11KeymapDropsUnreachableLevels(t *testing.T) { + t.Parallel() + keymap, err := buildX11Keymap([]uint32{'a', 'A', 'a', 'A'}, 20, 1, 4, x11ModifierMap{}) + if err != nil { + t.Fatal(err) + } + if _, exists := keymap.bindings['A']; exists { + t.Fatal("retained shifted binding without a Shift key") + } +} + +func TestBuildX11KeymapExpandsImplicitCaseAndGroupLevels(t *testing.T) { + t.Parallel() + var modifiers x11ModifierMap + modifiers[0] = []byte{10} + keymap, err := buildX11Keymap([]uint32{0xffe1, 0, 'a', 0}, 10, 2, 2, modifiers) + if err != nil { + t.Fatal(err) + } + binding, exists := keymap.bindings['A'] + if !exists || binding.keycode != 11 || !binding.shift || !binding.lockSensitive || !binding.shiftSensitive { + t.Fatalf("implicit uppercase binding = %+v, exists=%v", binding, exists) + } +} + +func TestBuildX11KeymapUsesActualModifierMembership(t *testing.T) { + t.Parallel() + var modifiers x11ModifierMap + modifiers[0] = []byte{99} + keymap, err := buildX11Keymap([]uint32{0xffe1, 0, 'a', 'A'}, 10, 2, 2, modifiers) + if err != nil { + t.Fatal(err) + } + if keymap.preferredShift != 99 { + t.Fatalf("preferred Shift = %d", keymap.preferredShift) + } +} + +func TestBuildX11KeymapCapsLockWinsOverShiftLock(t *testing.T) { + t.Parallel() + var modifiers x11ModifierMap + modifiers[0] = []byte{10} + modifiers[1] = []byte{11, 12} + keymap, err := buildX11Keymap([]uint32{ + 0xffe1, 0, + x11KeysymShiftLock, 0, + x11KeysymCapsLock, 0, + }, 10, 3, 2, modifiers) + if err != nil { + t.Fatal(err) + } + if keymap.lockMode != x11LockCaps { + t.Fatalf("lock mode = %d", keymap.lockMode) + } +} + +func TestBuildX11KeymapPreservesShiftedOnlyPair(t *testing.T) { + t.Parallel() + var modifiers x11ModifierMap + modifiers[0] = []byte{10} + keymap, err := buildX11Keymap([]uint32{0xffe1, 0, 0, '@'}, 10, 2, 2, modifiers) + if err != nil { + t.Fatal(err) + } + binding, exists := keymap.bindings['@'] + if !exists || binding.keycode != 11 || !binding.shift { + t.Fatalf("shifted-only binding = %+v, exists=%v", binding, exists) + } +} + +func TestRequiredX11PhysicalShiftAccountsForLocks(t *testing.T) { + t.Parallel() + lower := x11KeyBinding{lockSensitive: true} + upper := x11KeyBinding{shift: true, lockSensitive: true} + digit := x11KeyBinding{} + if !requiredX11PhysicalShift(lower, true, x11LockCaps, false) { + t.Fatal("Caps Lock lowercase did not require physical Shift") + } + if requiredX11PhysicalShift(upper, true, x11LockCaps, false) { + t.Fatal("Caps Lock uppercase unnecessarily required physical Shift") + } + if requiredX11PhysicalShift(digit, true, x11LockCaps, false) { + t.Fatal("Caps Lock changed a non-alphabetic key") + } + if !requiredX11PhysicalShift(digit, true, x11LockShift, false) { + t.Fatal("Shift Lock did not affect a non-alphabetic key") + } +} + +func TestRequiredX11PhysicalShiftAccountsForNumLock(t *testing.T) { + t.Parallel() + keypadLower := x11KeyBinding{keypad: true, shiftSensitive: true} + keypadUpper := x11KeyBinding{shift: true, keypad: true, shiftSensitive: true} + if !requiredX11PhysicalShift(keypadLower, false, x11LockNone, true) { + t.Fatal("Num Lock keypad base level did not require Shift reversal") + } + if requiredX11PhysicalShift(keypadUpper, false, x11LockNone, true) { + t.Fatal("Num Lock keypad shifted level retained Shift") + } + if requiredX11PhysicalShift(x11KeyBinding{keypad: true}, false, x11LockNone, true) { + t.Fatal("Num Lock altered a one-level keypad key") + } +} + +func TestBuildX11KeymapDropsShiftedModifierAlias(t *testing.T) { + t.Parallel() + var modifiers x11ModifierMap + modifiers[3] = []byte{20} + keymap, err := buildX11Keymap([]uint32{0xffe9, 0xffe7}, 20, 1, 2, modifiers) + if err != nil { + t.Fatal(err) + } + if _, exists := keymap.bindings[0xffe9]; !exists { + t.Fatal("dropped base modifier binding") + } + if _, exists := keymap.bindings[0xffe7]; exists { + t.Fatal("retained shifted modifier alias") + } +} + +func TestX11ConvertCaseSupportsLegacyAndUCSKeysyms(t *testing.T) { + t.Parallel() + checks := map[uint32][2]uint32{ + 0x06c0: {0x06c0, 0x06e0}, + 0x07e1: {0x07e1, 0x07c1}, + 0x010003b1: {0x010003b1, 0x01000391}, + } + for input, expected := range checks { + lower, upper := x11ConvertCase(input) + if lower != expected[0] || upper != expected[1] { + t.Fatalf("case(%#x) = %#x/%#x, want %#x/%#x", input, lower, upper, expected[0], expected[1]) + } + } +} diff --git a/internal/connect/x11_linux.go b/internal/connect/x11_linux.go new file mode 100644 index 00000000..36abac1c --- /dev/null +++ b/internal/connect/x11_linux.go @@ -0,0 +1,656 @@ +//go:build linux + +package connect + +import ( + "context" + "encoding/binary" + "errors" + "fmt" + "math" + "math/bits" + "sync" + "sync/atomic" + "time" + + "github.com/jezek/xgb" + "github.com/jezek/xgb/shm" + "github.com/jezek/xgb/xfixes" + "github.com/jezek/xgb/xproto" + "github.com/jezek/xgb/xtest" + "golang.org/x/sys/unix" +) + +// LinuxX11 captures the default X11 screen with MIT-SHM and injects input +// through XTest. It intentionally has no Wayland fallback. +type LinuxX11 struct { + mu sync.Mutex + stateMu sync.Mutex + closeOnce sync.Once + closed atomic.Bool + + connection *xgb.Conn + root xproto.Window + width int + height int + stride int + bytesPixel int + byteOrder byte + redMask uint32 + greenMask uint32 + blueMask uint32 + segment shm.Seg + shmID int + shmBytes []byte + keymap x11Keymap + heldKeys map[byte]struct{} + heldKeyStates map[uint32]x11HeldKeyState + modifierRestore []x11ModifierTransition + modifierShift bool + modifierMode bool + buttonMask byte + sequence uint64 +} + +func NewLinuxX11(display string) (_ *LinuxX11, resultErr error) { + connection, err := xgb.NewConnDisplay(display) + if err != nil { + return nil, fmt.Errorf("connect to X11 display: %w", err) + } + backend := &LinuxX11{connection: connection, shmID: -1} + defer func() { + if resultErr != nil { + _ = backend.closeLocked() + } + }() + if err := shm.Init(connection); err != nil { + return nil, fmt.Errorf("initialize X11 MIT-SHM: %w", err) + } + if _, err := shm.QueryVersion(connection).Reply(); err != nil { + return nil, fmt.Errorf("query X11 MIT-SHM: %w", err) + } + if err := xtest.Init(connection); err != nil { + return nil, fmt.Errorf("initialize X11 XTest: %w", err) + } + if err := xfixes.Init(connection); err != nil { + return nil, fmt.Errorf("initialize X11 XFixes: %w", err) + } + xfixesVersion, err := xfixes.QueryVersion(connection, 1, 0).Reply() + if err != nil { + return nil, fmt.Errorf("negotiate X11 XFixes 1.0: %w", err) + } + if xfixesVersion == nil || xfixesVersion.MajorVersion < 1 { + return nil, errors.New("X11 XFixes 1.0 is unavailable") + } + + setup := xproto.Setup(connection) + screen := setup.DefaultScreen(connection) + if screen == nil || screen.WidthInPixels == 0 || screen.HeightInPixels == 0 { + return nil, errors.New("X11 default screen is unavailable") + } + format, ok := pixmapFormat(setup, screen.RootDepth) + if !ok || (format.BitsPerPixel != 16 && format.BitsPerPixel != 24 && format.BitsPerPixel != 32) { + return nil, fmt.Errorf("unsupported X11 root depth %d", screen.RootDepth) + } + visual, ok := rootVisual(screen) + if !ok || visual.Class != xproto.VisualClassTrueColor || + visual.RedMask == 0 || visual.GreenMask == 0 || visual.BlueMask == 0 { + return nil, errors.New("X11 root visual is not TrueColor") + } + width, height := int(screen.WidthInPixels), int(screen.HeightInPixels) + bitsPerRow := width * int(format.BitsPerPixel) + pad := int(format.ScanlinePad) + if pad < 8 || pad%8 != 0 { + return nil, fmt.Errorf("unsupported X11 pixmap scanline padding %d", pad) + } + stride := ((bitsPerRow + pad - 1) / pad) * (pad / 8) + bufferSize := stride * height + if bufferSize <= 0 || bufferSize > MaxFrameBytes { + return nil, errors.New("X11 framebuffer exceeds capture memory limit") + } + rgbaPixels, ok := checkedMul(width, height) + if !ok { + return nil, errors.New("X11 RGBA dimensions overflow") + } + rgbaBytes, ok := checkedMul(rgbaPixels, 4) + if !ok || rgbaBytes > MaxFrameBytes { + return nil, errors.New("X11 RGBA framebuffer exceeds capture memory limit") + } + + shmID, err := unix.SysvShmGet(unix.IPC_PRIVATE, bufferSize, unix.IPC_CREAT|0o600) + if err != nil { + return nil, fmt.Errorf("allocate X11 shared memory: %w", err) + } + backend.shmID = shmID + shmBytes, err := unix.SysvShmAttach(shmID, 0, 0) + if err != nil { + return nil, fmt.Errorf("attach X11 shared memory locally: %w", err) + } + backend.shmBytes = shmBytes + segmentID, err := connection.NewId() + if err != nil { + return nil, fmt.Errorf("allocate X11 shared segment id: %w", err) + } + backend.segment = shm.Seg(segmentID) + if err := shm.AttachChecked(connection, backend.segment, uint32(shmID), false).Check(); err != nil { + return nil, fmt.Errorf("attach X11 shared memory to server: %w", err) + } + // Mark for automatic removal after both the process and X server detach. + if _, err := unix.SysvShmCtl(shmID, unix.IPC_RMID, nil); err != nil { + return nil, fmt.Errorf("mark X11 shared memory for removal: %w", err) + } + backend.shmID = -1 + + keymap, err := loadKeymap(connection, setup) + if err != nil { + return nil, err + } + backend.root = screen.Root + backend.width = width + backend.height = height + backend.stride = stride + backend.bytesPixel = int(format.BitsPerPixel) / 8 + backend.byteOrder = setup.ImageByteOrder + backend.redMask = visual.RedMask + backend.greenMask = visual.GreenMask + backend.blueMask = visual.BlueMask + backend.keymap = keymap + backend.heldKeys = make(map[byte]struct{}) + backend.heldKeyStates = make(map[uint32]x11HeldKeyState) + return backend, nil +} + +func (backend *LinuxX11) Capture(ctx context.Context) (Frame, error) { + if err := ctx.Err(); err != nil { + return Frame{}, err + } + backend.mu.Lock() + defer backend.mu.Unlock() + if backend.closed.Load() { + return Frame{}, ErrClosed + } + reply, err := shm.GetImage( + backend.connection, + xproto.Drawable(backend.root), + 0, + 0, + uint16(backend.width), + uint16(backend.height), + math.MaxUint32, + xproto.ImageFormatZPixmap, + backend.segment, + 0, + ).Reply() + if err != nil { + return Frame{}, fmt.Errorf("XShmGetImage: %w", err) + } + if reply == nil || int(reply.Size) > len(backend.shmBytes) || int(reply.Size) < backend.stride*backend.height { + return Frame{}, errors.New("XShmGetImage returned an invalid size") + } + pixels := make([]byte, backend.width*backend.height*4) + for y := 0; y < backend.height; y++ { + for x := 0; x < backend.width; x++ { + source := y*backend.stride + x*backend.bytesPixel + pixel := backend.readPixel(backend.shmBytes[source : source+backend.bytesPixel]) + target := (y*backend.width + x) * 4 + pixels[target] = scaleMasked(pixel, backend.redMask) + pixels[target+1] = scaleMasked(pixel, backend.greenMask) + pixels[target+2] = scaleMasked(pixel, backend.blueMask) + pixels[target+3] = 0xff + } + } + backend.sequence++ + return Frame{ + Width: backend.width, + Height: backend.height, + Stride: backend.width * 4, + Pixels: pixels, + DirtyRects: []Rect{{Width: backend.width, Height: backend.height}}, + Sequence: backend.sequence, + }, nil +} + +func (backend *LinuxX11) Pointer(ctx context.Context, event PointerEvent) error { + if err := ctx.Err(); err != nil { + return err + } + backend.mu.Lock() + defer backend.mu.Unlock() + if backend.closed.Load() { + return ErrClosed + } + backend.stateMu.Lock() + defer backend.stateMu.Unlock() + if backend.closed.Load() { + return ErrClosed + } + x := int16(min(int(event.X), math.MaxInt16)) + y := int16(min(int(event.Y), math.MaxInt16)) + xtest.FakeInput(backend.connection, xproto.MotionNotify, 0, 0, backend.root, x, y, 0) + for bit := 0; bit < 8; bit++ { + button := byte(1 << bit) + wasDown := backend.buttonMask&button != 0 + isDown := event.ButtonMask&button != 0 + if wasDown == isDown { + continue + } + eventType := byte(xproto.ButtonRelease) + if isDown { + eventType = xproto.ButtonPress + } + xtest.FakeInput(backend.connection, eventType, byte(bit+1), 0, 0, 0, 0, 0) + if isDown { + backend.buttonMask |= button + } else { + backend.buttonMask &^= button + } + } + return nil +} + +func (backend *LinuxX11) Key(ctx context.Context, event KeyEvent) error { + if err := ctx.Err(); err != nil { + return err + } + backend.mu.Lock() + defer backend.mu.Unlock() + if backend.closed.Load() { + return ErrClosed + } + binding, ok := backend.keymap.bindings[event.Keysym] + if !ok { + return fmt.Errorf("X11 keymap has no keycode for keysym %#x", event.Keysym) + } + var modifierState x11ModifierState + modifierAlreadyDown := false + if event.Down && backend.isModifier(binding.keycode) { + var err error + modifierAlreadyDown, err = backend.queryKeycodeDown(binding.keycode) + if err != nil { + return fmt.Errorf("query X11 modifier key: %w", err) + } + } + if event.Down && !backend.isModifier(binding.keycode) { + var err error + modifierState, err = backend.queryModifierState() + if err != nil { + return fmt.Errorf("query X11 modifier state: %w", err) + } + } + backend.stateMu.Lock() + defer backend.stateMu.Unlock() + if backend.closed.Load() { + return ErrClosed + } + if backend.isModifier(binding.keycode) { + eventType := byte(xproto.KeyRelease) + if event.Down { + if modifierAlreadyDown { + return errors.New("X11 modifier key is already physically down") + } + eventType = xproto.KeyPress + } else if _, owned := backend.heldKeys[binding.keycode]; !owned { + return nil + } + _ = backend.sendKey(eventType, binding.keycode) + if event.Down { + backend.heldKeys[binding.keycode] = struct{}{} + } else { + delete(backend.heldKeys, binding.keycode) + } + return nil + } + if event.Down { + if held, exists := backend.heldKeyStates[event.Keysym]; exists { + _ = backend.sendKey(xproto.KeyPress, held.keycode) + return nil + } + lockActive := modifierState.mask&xproto.ModMaskLock != 0 + ownedModifiers := uint16(0) + for keycode := range backend.heldKeys { + ownedModifiers |= backend.keymap.modifierMasks[keycode] + } + for _, transition := range backend.modifierRestore { + if !transition.down { + ownedModifiers |= backend.keymap.modifierMasks[transition.keycode] + } + } + const actionableModifiers = uint16( + xproto.ModMaskShift | xproto.ModMaskControl | + xproto.ModMask1 | xproto.ModMask2 | xproto.ModMask3 | + xproto.ModMask4 | xproto.ModMask5, + ) + unownedModifiers := modifierState.mask & actionableModifiers &^ ownedModifiers &^ + backend.keymap.numLockMask + if unownedModifiers != 0 { + return fmt.Errorf("unowned X11 modifier mask %#x is active", unownedModifiers) + } + requiredShift := requiredX11PhysicalShift( + binding, + lockActive, + backend.keymap.lockMode, + modifierState.mask&backend.keymap.numLockMask != 0, + ) + if !binding.shiftSensitive && modifierState.mask&ownedModifiers&xproto.ModMaskShift != 0 { + requiredShift = true + } + requiredMode := binding.mode + if len(backend.heldKeyStates) > 0 { + if requiredShift != backend.modifierShift || requiredMode != backend.modifierMode { + return errors.New("simultaneous X11 keys require conflicting modifier levels") + } + } else { + restore, err := backend.applyRequiredModifiers(requiredShift, requiredMode, modifierState) + if err != nil { + return fmt.Errorf("XTest key %#x: %w", event.Keysym, err) + } + backend.modifierRestore = restore + backend.modifierShift = requiredShift + backend.modifierMode = requiredMode + } + _ = backend.sendKey(xproto.KeyPress, binding.keycode) + backend.heldKeyStates[event.Keysym] = x11HeldKeyState{keycode: binding.keycode} + backend.heldKeys[binding.keycode] = struct{}{} + return nil + } + held, exists := backend.heldKeyStates[event.Keysym] + if !exists { + return nil + } + _ = backend.sendKey(xproto.KeyRelease, held.keycode) + delete(backend.heldKeyStates, event.Keysym) + if !backend.keycodeHeldByNonModifier(held.keycode) { + delete(backend.heldKeys, held.keycode) + } + if len(backend.heldKeyStates) == 0 { + backend.restoreModifierTransitions(backend.modifierRestore) + backend.modifierRestore = nil + } + return nil +} + +func (backend *LinuxX11) Cursor(ctx context.Context) (Cursor, error) { + if err := ctx.Err(); err != nil { + return Cursor{}, err + } + backend.mu.Lock() + defer backend.mu.Unlock() + if backend.closed.Load() { + return Cursor{}, ErrClosed + } + reply, err := xfixes.GetCursorImage(backend.connection).Reply() + if err != nil { + return Cursor{}, fmt.Errorf("get X11 cursor image: %w", err) + } + if reply == nil || reply.Width == 0 || reply.Height == 0 { + return Cursor{}, errors.New("XFixes returned an empty cursor image") + } + return cursorFromXFixes( + int(reply.X), int(reply.Y), int(reply.Width), int(reply.Height), + int(reply.Xhot), int(reply.Yhot), reply.CursorImage, backend.width, backend.height, + ) +} + +func (backend *LinuxX11) Close() error { + backend.closeOnce.Do(func() { + backend.closed.Store(true) + backend.releaseInputBeforeClose() + // Closing the X connection interrupts a pending Reply before teardown + // waits for the operation lock. + if backend.connection != nil { + backend.connection.Close() + } + backend.mu.Lock() + defer backend.mu.Unlock() + backend.cleanupLocked() + }) + return nil +} + +func (backend *LinuxX11) closeLocked() error { + if backend.closed.Swap(true) { + return nil + } + if backend.connection != nil { + backend.connection.Close() + } + backend.cleanupLocked() + return nil +} + +func (backend *LinuxX11) cleanupLocked() { + if backend.connection != nil { + // Connection close releases server-side input state and SHM attachment. + backend.connection = nil + } + if len(backend.shmBytes) > 0 { + _ = unix.SysvShmDetach(backend.shmBytes) + backend.shmBytes = nil + } + if backend.shmID >= 0 { + _, _ = unix.SysvShmCtl(backend.shmID, unix.IPC_RMID, nil) + backend.shmID = -1 + } +} + +func (backend *LinuxX11) sendKey(eventType, keycode byte) error { + xtest.FakeInput( + backend.connection, eventType, keycode, 0, 0, 0, 0, 0, + ) + return nil +} + +func (backend *LinuxX11) isModifier(keycode byte) bool { + _, modifier := backend.keymap.modifierKeys[keycode] + return modifier +} + +type x11HeldKeyState struct { + keycode byte +} + +type x11ModifierTransition struct { + keycode byte + down bool +} + +type x11ModifierState struct { + mask uint16 +} + +func (backend *LinuxX11) queryModifierState() (x11ModifierState, error) { + pointer, err := xproto.QueryPointer(backend.connection, backend.root).Reply() + if err != nil { + return x11ModifierState{}, fmt.Errorf("query pointer: %w", err) + } + if pointer == nil { + return x11ModifierState{}, errors.New("query pointer returned no reply") + } + return x11ModifierState{mask: pointer.Mask}, nil +} + +func (backend *LinuxX11) queryKeycodeDown(keycode byte) (bool, error) { + keys, err := xproto.QueryKeymap(backend.connection).Reply() + if err != nil { + return false, err + } + if keys == nil || len(keys.Keys) != 32 { + return false, errors.New("query keymap returned invalid data") + } + return keys.Keys[int(keycode)/8]&(1<= 0; index-- { + eventType := byte(xproto.KeyRelease) + if restore[index].down { + eventType = xproto.KeyPress + } + _ = backend.sendKey(eventType, restore[index].keycode) + } +} + +func (backend *LinuxX11) keycodeHeldByNonModifier(keycode byte) bool { + for _, held := range backend.heldKeyStates { + if held.keycode == keycode { + return true + } + } + return false +} + +func (backend *LinuxX11) releaseInputBeforeClose() { + backend.stateMu.Lock() + defer backend.stateMu.Unlock() + if backend.connection == nil { + return + } + for keycode := range backend.heldKeys { + xtest.FakeInput(backend.connection, xproto.KeyRelease, keycode, 0, 0, 0, 0, 0) + } + backend.restoreModifierTransitions(backend.modifierRestore) + for bit := 0; bit < 8; bit++ { + if backend.buttonMask&(1< math.MaxUint8 { + return x11Keymap{}, errors.New("invalid X11 keycode range") + } + reply, err := xproto.GetKeyboardMapping(connection, setup.MinKeycode, byte(count)).Reply() + if err != nil { + return x11Keymap{}, fmt.Errorf("read X11 keymap: %w", err) + } + if reply == nil || reply.KeysymsPerKeycode == 0 { + return x11Keymap{}, errors.New("X11 keymap is empty") + } + modifierReply, err := xproto.GetModifierMapping(connection).Reply() + if err != nil { + return x11Keymap{}, fmt.Errorf("read X11 modifier map: %w", err) + } + if modifierReply == nil || modifierReply.KeycodesPerModifier == 0 { + return x11Keymap{}, errors.New("X11 modifier map is empty") + } + perModifier := int(modifierReply.KeycodesPerModifier) + if len(modifierReply.Keycodes) != perModifier*8 { + return x11Keymap{}, errors.New("X11 modifier map has invalid dimensions") + } + var modifiers x11ModifierMap + for modifier := range modifiers { + for _, keycode := range modifierReply.Keycodes[modifier*perModifier : (modifier+1)*perModifier] { + modifiers[modifier] = append(modifiers[modifier], byte(keycode)) + } + } + keysyms := make([]uint32, len(reply.Keysyms)) + for index, keysym := range reply.Keysyms { + keysyms[index] = uint32(keysym) + } + return buildX11Keymap(keysyms, byte(setup.MinKeycode), count, int(reply.KeysymsPerKeycode), modifiers) +} + +func scaleMasked(pixel, mask uint32) byte { + shift := bits.TrailingZeros32(mask) + maximum := mask >> shift + value := (pixel & mask) >> shift + return byte((uint64(value)*255 + uint64(maximum)/2) / uint64(maximum)) +} diff --git a/internal/rfb/server.go b/internal/rfb/server.go index 1d789b66..f0935632 100644 --- a/internal/rfb/server.go +++ b/internal/rfb/server.go @@ -153,13 +153,24 @@ func (server *Server) Close() error { for _, connection := range connections { _ = connection.Close() } - server.wg.Wait() cleanupContext, cleanupCancel := context.WithTimeout(context.Background(), 500*time.Millisecond) - server.inputs.releaseAll(cleanupContext) + cleanupDone := make(chan struct{}) + go func() { + server.inputs.releaseAll(cleanupContext) + close(cleanupDone) + }() + select { + case <-cleanupDone: + case <-cleanupContext.Done(): + } cleanupCancel() if err := server.config.Session.Backend.Close(); server.closeErr == nil { server.closeErr = err } + // Backend close must precede the wait: a platform capture call may be + // blocked in an OS round trip that ignores Go context cancellation. + server.wg.Wait() + <-cleanupDone }) return server.closeErr } diff --git a/internal/rfb/server_test.go b/internal/rfb/server_test.go index 9b2c2221..9e1ccc24 100644 --- a/internal/rfb/server_test.go +++ b/internal/rfb/server_test.go @@ -12,9 +12,11 @@ import ( ) type blockingBackend struct { - mu sync.Mutex - calls int - blocked chan struct{} + mu sync.Mutex + calls int + blocked chan struct{} + closed chan struct{} + closeOnce sync.Once } func (backend *blockingBackend) Capture(ctx context.Context) (connect.Frame, error) { @@ -24,8 +26,8 @@ func (backend *blockingBackend) Capture(ctx context.Context) (connect.Frame, err backend.mu.Unlock() if call > 1 { close(backend.blocked) - <-ctx.Done() - return connect.Frame{}, ctx.Err() + <-backend.closed + return connect.Frame{}, connect.ErrClosed } return connect.Frame{ Width: 2, Height: 2, Stride: 8, @@ -38,11 +40,14 @@ func (backend *blockingBackend) Capture(ctx context.Context) (connect.Frame, err func (*blockingBackend) Pointer(context.Context, connect.PointerEvent) error { return nil } func (*blockingBackend) Key(context.Context, connect.KeyEvent) error { return nil } -func (*blockingBackend) Close() error { return nil } +func (backend *blockingBackend) Close() error { + backend.closeOnce.Do(func() { close(backend.closed) }) + return nil +} func TestServerCloseCancelsBlockedCapture(t *testing.T) { t.Parallel() - backend := &blockingBackend{blocked: make(chan struct{})} + backend := &blockingBackend{blocked: make(chan struct{}), closed: make(chan struct{})} server, err := NewServer(ServerConfig{Session: SessionConfig{ Backend: backend, Password: sessionFixturePassword(), ChallengeReader: &repeatReader{}, }})