diff --git a/pkg/kraken/websocket.go b/pkg/kraken/websocket.go index 19d63ae..9de84a2 100644 --- a/pkg/kraken/websocket.go +++ b/pkg/kraken/websocket.go @@ -5,6 +5,7 @@ import ( "crypto/tls" "crypto/x509" "encoding/json" + "errors" "fmt" "net/http" "sync" @@ -14,6 +15,18 @@ import ( "github.com/krakenfx/api-go/v2/pkg/callback" ) +// ErrAlreadyConnected is returned by [WebSocket.Connect] when a connection is +// already established, or when another caller is in the middle of establishing +// one. +// +// Opening a second connection is worse than a no-op. The reader of the first +// one keeps running, both readers end up on whichever connection was stored +// last, and two readers on a single gorilla connection share its buffered +// reader. Their frames interleave, the connection fails with errors such as +// "RSV2 set" or "bad opcode", and the shared buffer can be corrupted badly +// enough to panic with a slice bounds error. +var ErrAlreadyConnected = errors.New("already connected") + // WebSocket implements a common structure for the WebSocket APIs. type WebSocket struct { Reconnect func() @@ -25,11 +38,19 @@ type WebSocket struct { OnSent *callback.Manager[*WebSocketMessage] OnReceived *callback.Manager[*WebSocketMessage] - conn *websocket.Conn URL string - writeMux sync.Mutex Insecure bool - active bool + + // mux guards the connection state below. It is never held while a callback + // runs or while the network is touched. + mux sync.Mutex + // connecting is held for the duration of a dial so that racing callers are + // turned away before they open a connection that all but one of them would + // immediately close. + connecting bool + conn *websocket.Conn + active bool + writeMux sync.Mutex } // NewWebSocket creates a new [WebSocket] object with default values. @@ -43,7 +64,13 @@ func NewWebSocket() *WebSocket { } ws.Reconnect = func() { for { - if err := ws.Connect(); err == nil { + err := ws.Connect() + if err == nil { + return + } + // Another caller has already restored the connection, which is the + // whole purpose of this loop. Retrying would open a second one. + if errors.Is(err, ErrAlreadyConnected) { return } time.Sleep(ws.ReconnectWait) @@ -58,12 +85,26 @@ func NewWebSocket() *WebSocket { } // Connect establishes a connection. +// +// It is safe for concurrent use and returns [ErrAlreadyConnected] instead of +// opening a second connection over a live one, so that a caller with its own +// reconnect logic on top of the built-in handler cannot end up with two readers +// on the same connection. func (ws *WebSocket) Connect() error { + ws.mux.Lock() + if ws.active || ws.connecting { + ws.mux.Unlock() + return ErrAlreadyConnected + } + ws.connecting = true + insecure, url := ws.Insecure, ws.URL + ws.mux.Unlock() + dialer := &websocket.Dialer{ Proxy: http.ProxyFromEnvironment, HandshakeTimeout: 45 * time.Second, } - if ws.Insecure { + if insecure { dialer.TLSClientConfig = &tls.Config{ InsecureSkipVerify: true, VerifyPeerCertificate: func(rawCerts [][]byte, verifiedChains [][]*x509.Certificate) error { @@ -71,13 +112,26 @@ func (ws *WebSocket) Connect() error { }, } } - connection, _, err := dialer.Dial(ws.URL, nil) + connection, _, err := dialer.Dial(url, nil) + + ws.mux.Lock() + ws.connecting = false if err != nil { + ws.mux.Unlock() return fmt.Errorf("dial failed: %s", err) } + // A reader may have failed and reconnected while this dial was in flight. + if ws.active { + ws.mux.Unlock() + _ = connection.Close() + return ErrAlreadyConnected + } ws.conn = connection + ws.active = true ws.DoReconnect = true - go ws.read() + ws.mux.Unlock() + + go ws.read(connection) ws.OnConnected.Call(nil) return nil } @@ -125,15 +179,21 @@ func (m *WebSocketMessage) Map() (map[string]any, error) { return dataMapped, nil } -func (ws *WebSocket) read() { - ws.active = true - defer func() { - ws.active = false - }() +// read pumps a single connection. The connection is a parameter instead of a +// field read on every iteration, so that a reader can never drift onto a +// connection that replaced the one it was started for. +func (ws *WebSocket) read(conn *websocket.Conn) { for { - _, data, err := ws.conn.ReadMessage() + _, data, err := conn.ReadMessage() if err != nil { - _ = ws.conn.Close() + _ = conn.Close() + ws.mux.Lock() + // Only retire the connection if this reader still owns it. Clearing + // the flag here is what allows the handler below to reconnect. + if ws.conn == conn { + ws.active = false + } + ws.mux.Unlock() ws.OnDisconnected.Call(err) return } @@ -141,17 +201,29 @@ func (ws *WebSocket) read() { } } -// Disconnect stops the connection. +// Disconnect stops the connection. It is a no-op when nothing is connected. func (ws *WebSocket) Disconnect() error { + ws.mux.Lock() ws.DoReconnect = false - done := make(chan bool) - defer close(done) + conn := ws.conn + ws.mux.Unlock() + if conn == nil { + return nil + } + // Buffered, and sent without blocking: the handler runs on the reader + // goroutine, and it must neither block on a channel nobody reads after the + // wait below times out, nor send on one that has been closed. + done := make(chan struct{}, 1) + cb := ws.OnDisconnected.Recurring(func(e *callback.Event[error]) { + select { + case done <- struct{}{}: + default: + } + }) + defer ws.OnDisconnected.Deregister(cb) defer func() { - _ = ws.conn.Close() + _ = conn.Close() }() - callback := ws.OnDisconnected.Recurring(func(e *callback.Event[error]) { - done <- true - }) message := websocket.FormatCloseMessage(websocket.CloseNormalClosure, "") if err := ws.WriteMessage(websocket.CloseMessage, message); err != nil { return fmt.Errorf("write close failed: %s", err) @@ -160,12 +232,13 @@ func (ws *WebSocket) Disconnect() error { case <-done: case <-time.After(time.Second): } - ws.OnDisconnected.Deregister(callback) return nil } // IsActive returns the status of the connection. func (ws *WebSocket) IsActive() bool { + ws.mux.Lock() + defer ws.mux.Unlock() return ws.active } @@ -183,12 +256,15 @@ func (ws *WebSocket) WriteJSON(message any) error { // WriteMessage submits a raw message to the connection. func (ws *WebSocket) WriteMessage(messageType int, data []byte) error { - if ws.conn == nil { + ws.mux.Lock() + conn := ws.conn + ws.mux.Unlock() + if conn == nil { return fmt.Errorf("no connection") } ws.writeMux.Lock() defer ws.writeMux.Unlock() - if err := ws.conn.WriteMessage(messageType, data); err != nil { + if err := conn.WriteMessage(messageType, data); err != nil { return fmt.Errorf("write message failed: %s", err) } ws.OnSent.Call(NewWebSocketMessage(data)) diff --git a/pkg/kraken/websocket_test.go b/pkg/kraken/websocket_test.go new file mode 100644 index 0000000..3b72fff --- /dev/null +++ b/pkg/kraken/websocket_test.go @@ -0,0 +1,246 @@ +package kraken + +import ( + "errors" + "net/http" + "net/http/httptest" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/gorilla/websocket" + "github.com/krakenfx/api-go/v2/pkg/callback" +) + +// wsServer is a WebSocket server that counts the connections it accepts and can +// drop the live one on demand. +type wsServer struct { + server *httptest.Server + connections atomic.Int64 + mux sync.Mutex + live *websocket.Conn +} + +func newWSServer(t *testing.T) *wsServer { + t.Helper() + s := &wsServer{} + upgrader := websocket.Upgrader{} + s.server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + conn, err := upgrader.Upgrade(w, r, nil) + if err != nil { + return + } + s.connections.Add(1) + s.mux.Lock() + s.live = conn + s.mux.Unlock() + for { + if _, _, err := conn.ReadMessage(); err != nil { + _ = conn.Close() + return + } + } + })) + t.Cleanup(s.server.Close) + return s +} + +func (s *wsServer) url() string { + return "ws" + strings.TrimPrefix(s.server.URL, "http") +} + +// drop closes the connection the client is currently using. +func (s *wsServer) drop(t *testing.T) { + t.Helper() + deadline := time.After(2 * time.Second) + for { + s.mux.Lock() + conn := s.live + s.mux.Unlock() + if conn != nil { + _ = conn.Close() + return + } + select { + case <-deadline: + t.Fatal("no live connection to drop") + case <-time.After(5 * time.Millisecond): + } + } +} + +func (s *wsServer) waitForConnections(t *testing.T, want int64) { + t.Helper() + deadline := time.After(5 * time.Second) + for { + if s.connections.Load() >= want { + return + } + select { + case <-deadline: + t.Fatalf("server accepted %d connections, want %d", s.connections.Load(), want) + case <-time.After(10 * time.Millisecond): + } + } +} + +func newTestWebSocket(t *testing.T, s *wsServer) *WebSocket { + t.Helper() + ws := NewWebSocket() + ws.URL = s.url() + ws.ReconnectWait = 10 * time.Millisecond + ws.DoReconnect = true + t.Cleanup(func() { + ws.mux.Lock() + ws.DoReconnect = false + ws.mux.Unlock() + _ = ws.Disconnect() + }) + return ws +} + +func TestConnectRefusesASecondConnection(t *testing.T) { + server := newWSServer(t) + ws := newTestWebSocket(t, server) + + if err := ws.Connect(); err != nil { + t.Fatalf("first connect failed: %v", err) + } + server.waitForConnections(t, 1) + + if err := ws.Connect(); !errors.Is(err, ErrAlreadyConnected) { + t.Fatalf("second connect returned %v, want ErrAlreadyConnected", err) + } + time.Sleep(100 * time.Millisecond) + if got := server.connections.Load(); got != 1 { + t.Fatalf("server accepted %d connections, want 1", got) + } + if !ws.IsActive() { + t.Fatal("IsActive reports false while connected") + } +} + +// TestDuplicateReconnectHandlerIsHarmless covers the mistake this guard is for. +// NewWebSocket already registers an OnDisconnected handler that reconnects, so +// a caller that registers its own reconnecting handler used to open a second +// connection on every drop, leaving two readers on one connection. +func TestDuplicateReconnectHandlerIsHarmless(t *testing.T) { + server := newWSServer(t) + ws := newTestWebSocket(t, server) + ws.OnDisconnected.Recurring(func(e *callback.Event[error]) { + if ws.Reconnect != nil && !websocket.IsCloseError(e.Data, websocket.CloseNormalClosure) { + ws.Reconnect() + } + }) + + if err := ws.Connect(); err != nil { + t.Fatalf("connect failed: %v", err) + } + server.waitForConnections(t, 1) + + for i := 0; i < 3; i++ { + want := int64(i + 2) + server.drop(t) + server.waitForConnections(t, want) + // Allow a duplicate connection to appear if one is going to. + time.Sleep(200 * time.Millisecond) + if got := server.connections.Load(); got != want { + t.Fatalf("after %d drops the server accepted %d connections, want %d", i+1, got, want) + } + } +} + +func TestReconnectsAfterDrop(t *testing.T) { + server := newWSServer(t) + ws := newTestWebSocket(t, server) + var disconnects atomic.Int64 + ws.OnDisconnected.Recurring(func(e *callback.Event[error]) { + disconnects.Add(1) + }) + + if err := ws.Connect(); err != nil { + t.Fatalf("connect failed: %v", err) + } + server.waitForConnections(t, 1) + server.drop(t) + server.waitForConnections(t, 2) + + if got := disconnects.Load(); got != 1 { + t.Fatalf("%d disconnect events, want 1", got) + } +} + +// TestConcurrentConnectOpensOne covers the race between two dials in flight. +func TestConcurrentConnectOpensOne(t *testing.T) { + server := newWSServer(t) + ws := newTestWebSocket(t, server) + + var wg sync.WaitGroup + errs := make([]error, 8) + for i := range errs { + wg.Add(1) + go func() { + defer wg.Done() + errs[i] = ws.Connect() + }() + } + wg.Wait() + + var connected int + for _, err := range errs { + switch { + case err == nil: + connected++ + case errors.Is(err, ErrAlreadyConnected): + default: + t.Fatalf("unexpected connect error: %v", err) + } + } + if connected != 1 { + t.Fatalf("%d callers connected, want exactly 1", connected) + } + time.Sleep(100 * time.Millisecond) + // The losing callers must be turned away before they dial, not after. + if got := server.connections.Load(); got != 1 { + t.Fatalf("server accepted %d connections, want 1", got) + } +} + +func TestDisconnectDoesNotReconnect(t *testing.T) { + server := newWSServer(t) + ws := newTestWebSocket(t, server) + + if err := ws.Connect(); err != nil { + t.Fatalf("connect failed: %v", err) + } + server.waitForConnections(t, 1) + if err := ws.Disconnect(); err != nil { + t.Fatalf("disconnect failed: %v", err) + } + time.Sleep(300 * time.Millisecond) + if got := server.connections.Load(); got != 1 { + t.Fatalf("server accepted %d connections after disconnect, want 1", got) + } + if ws.IsActive() { + t.Fatal("IsActive reports true after disconnect") + } +} + +func TestDisconnectWithoutAConnection(t *testing.T) { + ws := NewWebSocket() + if err := ws.Disconnect(); err != nil { + t.Fatalf("Disconnect on an unconnected WebSocket returned %v, want nil", err) + } + if ws.IsActive() { + t.Fatal("IsActive reports true on an unconnected WebSocket") + } +} + +func TestWriteMessageWithoutAConnection(t *testing.T) { + ws := NewWebSocket() + if err := ws.WriteMessage(websocket.TextMessage, []byte("ping")); err == nil { + t.Fatal("WriteMessage without a connection succeeded, want an error") + } +}