From 1b187ef2913c46916140b9495fea08db6c9aeb2d Mon Sep 17 00:00:00 2001 From: ssenerg Date: Mon, 7 Sep 2026 18:14:48 +0330 Subject: [PATCH] fix(kraken): prevent concurrent readers on a single WebSocket connection Connect stored the new connection in ws.conn and started a reader that re-read that field on every iteration. A second Connect on a live WebSocket therefore left the first reader running and pointed both readers at the same *websocket.Conn. Two readers on one gorilla connection share its buffered reader. Their frames interleave, the connection fails with errors such as "RSV2 set", "bad opcode 4" or "continuation after FIN", and the shared buffer can be corrupted badly enough to panic: panic: runtime error: slice bounds out of range [:7168] with capacity 4096 bufio.(*Reader).Read(...) github.com/gorilla/websocket.(*messageReader).Read(...) github.com/krakenfx/api-go/v2/pkg/kraken.(*WebSocket).read(...) Reaching that state is easy, because NewWebSocket registers an OnDisconnected handler that reconnects and never deregisters it: a caller that adds its own reconnecting handler gets two Connect calls per drop. With one such handler the readers accumulate on every disconnect, so a server that accepted 2 connections after the first drop accepted 3, then 5, then 7. read now takes its connection as a parameter, so a reader can never drift onto a connection that replaced the one it was started for, and Connect refuses to open a second connection while one is live or while another dial is in flight, returning the new ErrAlreadyConnected. The built-in Reconnect treats that error as success, since it means another caller has already done the work. The connection state is moved under a mutex. conn was written by Connect while a reader read it, and active was written by an incoming reader while the reader it replaced cleared it in a defer; both are reported by the race detector on every reconnect. Two smaller fixes in the same area: - Disconnect no longer panics with "send on closed channel". It closed its done channel in a defer while the handler that sends on it stays registered until the function returns, so a disconnect arriving after the one second timeout sent on a closed channel. The channel is now buffered and the send is non-blocking, which also stops the handler from blocking the reader goroutine. - Disconnect and WriteMessage no longer dereference a nil connection when nothing has been connected. Adds tests for the reconnect, duplicate-handler, concurrent-Connect and Disconnect paths. They pass under -race. --- pkg/kraken/websocket.go | 124 ++++++++++++++---- pkg/kraken/websocket_test.go | 246 +++++++++++++++++++++++++++++++++++ 2 files changed, 346 insertions(+), 24 deletions(-) create mode 100644 pkg/kraken/websocket_test.go diff --git a/pkg/kraken/websocket.go b/pkg/kraken/websocket.go index 19d63ae..9de84a2 100644 --- a/pkg/kraken/websocket.go +++ b/pkg/kraken/websocket.go @@ -5,6 +5,7 @@ import ( "crypto/tls" "crypto/x509" "encoding/json" + "errors" "fmt" "net/http" "sync" @@ -14,6 +15,18 @@ import ( "github.com/krakenfx/api-go/v2/pkg/callback" ) +// ErrAlreadyConnected is returned by [WebSocket.Connect] when a connection is +// already established, or when another caller is in the middle of establishing +// one. +// +// Opening a second connection is worse than a no-op. The reader of the first +// one keeps running, both readers end up on whichever connection was stored +// last, and two readers on a single gorilla connection share its buffered +// reader. Their frames interleave, the connection fails with errors such as +// "RSV2 set" or "bad opcode", and the shared buffer can be corrupted badly +// enough to panic with a slice bounds error. +var ErrAlreadyConnected = errors.New("already connected") + // WebSocket implements a common structure for the WebSocket APIs. type WebSocket struct { Reconnect func() @@ -25,11 +38,19 @@ type WebSocket struct { OnSent *callback.Manager[*WebSocketMessage] OnReceived *callback.Manager[*WebSocketMessage] - conn *websocket.Conn URL string - writeMux sync.Mutex Insecure bool - active bool + + // mux guards the connection state below. It is never held while a callback + // runs or while the network is touched. + mux sync.Mutex + // connecting is held for the duration of a dial so that racing callers are + // turned away before they open a connection that all but one of them would + // immediately close. + connecting bool + conn *websocket.Conn + active bool + writeMux sync.Mutex } // NewWebSocket creates a new [WebSocket] object with default values. @@ -43,7 +64,13 @@ func NewWebSocket() *WebSocket { } ws.Reconnect = func() { for { - if err := ws.Connect(); err == nil { + err := ws.Connect() + if err == nil { + return + } + // Another caller has already restored the connection, which is the + // whole purpose of this loop. Retrying would open a second one. + if errors.Is(err, ErrAlreadyConnected) { return } time.Sleep(ws.ReconnectWait) @@ -58,12 +85,26 @@ func NewWebSocket() *WebSocket { } // Connect establishes a connection. +// +// It is safe for concurrent use and returns [ErrAlreadyConnected] instead of +// opening a second connection over a live one, so that a caller with its own +// reconnect logic on top of the built-in handler cannot end up with two readers +// on the same connection. func (ws *WebSocket) Connect() error { + ws.mux.Lock() + if ws.active || ws.connecting { + ws.mux.Unlock() + return ErrAlreadyConnected + } + ws.connecting = true + insecure, url := ws.Insecure, ws.URL + ws.mux.Unlock() + dialer := &websocket.Dialer{ Proxy: http.ProxyFromEnvironment, HandshakeTimeout: 45 * time.Second, } - if ws.Insecure { + if insecure { dialer.TLSClientConfig = &tls.Config{ InsecureSkipVerify: true, VerifyPeerCertificate: func(rawCerts [][]byte, verifiedChains [][]*x509.Certificate) error { @@ -71,13 +112,26 @@ func (ws *WebSocket) Connect() error { }, } } - connection, _, err := dialer.Dial(ws.URL, nil) + connection, _, err := dialer.Dial(url, nil) + + ws.mux.Lock() + ws.connecting = false if err != nil { + ws.mux.Unlock() return fmt.Errorf("dial failed: %s", err) } + // A reader may have failed and reconnected while this dial was in flight. + if ws.active { + ws.mux.Unlock() + _ = connection.Close() + return ErrAlreadyConnected + } ws.conn = connection + ws.active = true ws.DoReconnect = true - go ws.read() + ws.mux.Unlock() + + go ws.read(connection) ws.OnConnected.Call(nil) return nil } @@ -125,15 +179,21 @@ func (m *WebSocketMessage) Map() (map[string]any, error) { return dataMapped, nil } -func (ws *WebSocket) read() { - ws.active = true - defer func() { - ws.active = false - }() +// read pumps a single connection. The connection is a parameter instead of a +// field read on every iteration, so that a reader can never drift onto a +// connection that replaced the one it was started for. +func (ws *WebSocket) read(conn *websocket.Conn) { for { - _, data, err := ws.conn.ReadMessage() + _, data, err := conn.ReadMessage() if err != nil { - _ = ws.conn.Close() + _ = conn.Close() + ws.mux.Lock() + // Only retire the connection if this reader still owns it. Clearing + // the flag here is what allows the handler below to reconnect. + if ws.conn == conn { + ws.active = false + } + ws.mux.Unlock() ws.OnDisconnected.Call(err) return } @@ -141,17 +201,29 @@ func (ws *WebSocket) read() { } } -// Disconnect stops the connection. +// Disconnect stops the connection. It is a no-op when nothing is connected. func (ws *WebSocket) Disconnect() error { + ws.mux.Lock() ws.DoReconnect = false - done := make(chan bool) - defer close(done) + conn := ws.conn + ws.mux.Unlock() + if conn == nil { + return nil + } + // Buffered, and sent without blocking: the handler runs on the reader + // goroutine, and it must neither block on a channel nobody reads after the + // wait below times out, nor send on one that has been closed. + done := make(chan struct{}, 1) + cb := ws.OnDisconnected.Recurring(func(e *callback.Event[error]) { + select { + case done <- struct{}{}: + default: + } + }) + defer ws.OnDisconnected.Deregister(cb) defer func() { - _ = ws.conn.Close() + _ = conn.Close() }() - callback := ws.OnDisconnected.Recurring(func(e *callback.Event[error]) { - done <- true - }) message := websocket.FormatCloseMessage(websocket.CloseNormalClosure, "") if err := ws.WriteMessage(websocket.CloseMessage, message); err != nil { return fmt.Errorf("write close failed: %s", err) @@ -160,12 +232,13 @@ func (ws *WebSocket) Disconnect() error { case <-done: case <-time.After(time.Second): } - ws.OnDisconnected.Deregister(callback) return nil } // IsActive returns the status of the connection. func (ws *WebSocket) IsActive() bool { + ws.mux.Lock() + defer ws.mux.Unlock() return ws.active } @@ -183,12 +256,15 @@ func (ws *WebSocket) WriteJSON(message any) error { // WriteMessage submits a raw message to the connection. func (ws *WebSocket) WriteMessage(messageType int, data []byte) error { - if ws.conn == nil { + ws.mux.Lock() + conn := ws.conn + ws.mux.Unlock() + if conn == nil { return fmt.Errorf("no connection") } ws.writeMux.Lock() defer ws.writeMux.Unlock() - if err := ws.conn.WriteMessage(messageType, data); err != nil { + if err := conn.WriteMessage(messageType, data); err != nil { return fmt.Errorf("write message failed: %s", err) } ws.OnSent.Call(NewWebSocketMessage(data)) diff --git a/pkg/kraken/websocket_test.go b/pkg/kraken/websocket_test.go new file mode 100644 index 0000000..3b72fff --- /dev/null +++ b/pkg/kraken/websocket_test.go @@ -0,0 +1,246 @@ +package kraken + +import ( + "errors" + "net/http" + "net/http/httptest" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/gorilla/websocket" + "github.com/krakenfx/api-go/v2/pkg/callback" +) + +// wsServer is a WebSocket server that counts the connections it accepts and can +// drop the live one on demand. +type wsServer struct { + server *httptest.Server + connections atomic.Int64 + mux sync.Mutex + live *websocket.Conn +} + +func newWSServer(t *testing.T) *wsServer { + t.Helper() + s := &wsServer{} + upgrader := websocket.Upgrader{} + s.server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + conn, err := upgrader.Upgrade(w, r, nil) + if err != nil { + return + } + s.connections.Add(1) + s.mux.Lock() + s.live = conn + s.mux.Unlock() + for { + if _, _, err := conn.ReadMessage(); err != nil { + _ = conn.Close() + return + } + } + })) + t.Cleanup(s.server.Close) + return s +} + +func (s *wsServer) url() string { + return "ws" + strings.TrimPrefix(s.server.URL, "http") +} + +// drop closes the connection the client is currently using. +func (s *wsServer) drop(t *testing.T) { + t.Helper() + deadline := time.After(2 * time.Second) + for { + s.mux.Lock() + conn := s.live + s.mux.Unlock() + if conn != nil { + _ = conn.Close() + return + } + select { + case <-deadline: + t.Fatal("no live connection to drop") + case <-time.After(5 * time.Millisecond): + } + } +} + +func (s *wsServer) waitForConnections(t *testing.T, want int64) { + t.Helper() + deadline := time.After(5 * time.Second) + for { + if s.connections.Load() >= want { + return + } + select { + case <-deadline: + t.Fatalf("server accepted %d connections, want %d", s.connections.Load(), want) + case <-time.After(10 * time.Millisecond): + } + } +} + +func newTestWebSocket(t *testing.T, s *wsServer) *WebSocket { + t.Helper() + ws := NewWebSocket() + ws.URL = s.url() + ws.ReconnectWait = 10 * time.Millisecond + ws.DoReconnect = true + t.Cleanup(func() { + ws.mux.Lock() + ws.DoReconnect = false + ws.mux.Unlock() + _ = ws.Disconnect() + }) + return ws +} + +func TestConnectRefusesASecondConnection(t *testing.T) { + server := newWSServer(t) + ws := newTestWebSocket(t, server) + + if err := ws.Connect(); err != nil { + t.Fatalf("first connect failed: %v", err) + } + server.waitForConnections(t, 1) + + if err := ws.Connect(); !errors.Is(err, ErrAlreadyConnected) { + t.Fatalf("second connect returned %v, want ErrAlreadyConnected", err) + } + time.Sleep(100 * time.Millisecond) + if got := server.connections.Load(); got != 1 { + t.Fatalf("server accepted %d connections, want 1", got) + } + if !ws.IsActive() { + t.Fatal("IsActive reports false while connected") + } +} + +// TestDuplicateReconnectHandlerIsHarmless covers the mistake this guard is for. +// NewWebSocket already registers an OnDisconnected handler that reconnects, so +// a caller that registers its own reconnecting handler used to open a second +// connection on every drop, leaving two readers on one connection. +func TestDuplicateReconnectHandlerIsHarmless(t *testing.T) { + server := newWSServer(t) + ws := newTestWebSocket(t, server) + ws.OnDisconnected.Recurring(func(e *callback.Event[error]) { + if ws.Reconnect != nil && !websocket.IsCloseError(e.Data, websocket.CloseNormalClosure) { + ws.Reconnect() + } + }) + + if err := ws.Connect(); err != nil { + t.Fatalf("connect failed: %v", err) + } + server.waitForConnections(t, 1) + + for i := 0; i < 3; i++ { + want := int64(i + 2) + server.drop(t) + server.waitForConnections(t, want) + // Allow a duplicate connection to appear if one is going to. + time.Sleep(200 * time.Millisecond) + if got := server.connections.Load(); got != want { + t.Fatalf("after %d drops the server accepted %d connections, want %d", i+1, got, want) + } + } +} + +func TestReconnectsAfterDrop(t *testing.T) { + server := newWSServer(t) + ws := newTestWebSocket(t, server) + var disconnects atomic.Int64 + ws.OnDisconnected.Recurring(func(e *callback.Event[error]) { + disconnects.Add(1) + }) + + if err := ws.Connect(); err != nil { + t.Fatalf("connect failed: %v", err) + } + server.waitForConnections(t, 1) + server.drop(t) + server.waitForConnections(t, 2) + + if got := disconnects.Load(); got != 1 { + t.Fatalf("%d disconnect events, want 1", got) + } +} + +// TestConcurrentConnectOpensOne covers the race between two dials in flight. +func TestConcurrentConnectOpensOne(t *testing.T) { + server := newWSServer(t) + ws := newTestWebSocket(t, server) + + var wg sync.WaitGroup + errs := make([]error, 8) + for i := range errs { + wg.Add(1) + go func() { + defer wg.Done() + errs[i] = ws.Connect() + }() + } + wg.Wait() + + var connected int + for _, err := range errs { + switch { + case err == nil: + connected++ + case errors.Is(err, ErrAlreadyConnected): + default: + t.Fatalf("unexpected connect error: %v", err) + } + } + if connected != 1 { + t.Fatalf("%d callers connected, want exactly 1", connected) + } + time.Sleep(100 * time.Millisecond) + // The losing callers must be turned away before they dial, not after. + if got := server.connections.Load(); got != 1 { + t.Fatalf("server accepted %d connections, want 1", got) + } +} + +func TestDisconnectDoesNotReconnect(t *testing.T) { + server := newWSServer(t) + ws := newTestWebSocket(t, server) + + if err := ws.Connect(); err != nil { + t.Fatalf("connect failed: %v", err) + } + server.waitForConnections(t, 1) + if err := ws.Disconnect(); err != nil { + t.Fatalf("disconnect failed: %v", err) + } + time.Sleep(300 * time.Millisecond) + if got := server.connections.Load(); got != 1 { + t.Fatalf("server accepted %d connections after disconnect, want 1", got) + } + if ws.IsActive() { + t.Fatal("IsActive reports true after disconnect") + } +} + +func TestDisconnectWithoutAConnection(t *testing.T) { + ws := NewWebSocket() + if err := ws.Disconnect(); err != nil { + t.Fatalf("Disconnect on an unconnected WebSocket returned %v, want nil", err) + } + if ws.IsActive() { + t.Fatal("IsActive reports true on an unconnected WebSocket") + } +} + +func TestWriteMessageWithoutAConnection(t *testing.T) { + ws := NewWebSocket() + if err := ws.WriteMessage(websocket.TextMessage, []byte("ping")); err == nil { + t.Fatal("WriteMessage without a connection succeeded, want an error") + } +}