From 6163012c5a211c11ddaca7ef52da0b427e7d2b4a Mon Sep 17 00:00:00 2001 From: Ralf Becher Date: Wed, 2 Sep 2026 14:31:38 +0200 Subject: [PATCH] fix: detect a peer that vanishes without FIN or RST MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The legacy SQL Server behind this proxy sits at the far end of a link that is cut nightly rather than closed. Nothing arrives, no error is raised, and a blocking read waits forever — so the proxy keeps the client leg open too, waiting for a response that will never come. On the Dremio side that strands the source plugin's state lock permanently: every subsequent health check times out against the lock, and only restarting the coordinator clears it. The teardown path is already correct — any exception in ProcessConnection runs Close(), which closes both legs. What is missing is the exception. Enable TCP keep-alive on both sockets so an unreachable peer faults the socket after ~90 seconds (60s idle, 3 probes 10s apart) instead of never. A live but idle peer answers the probes and is unaffected. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01Ld69q37Jq9eMvLLy6zgWwc --- src/TDSProxy/TDSConnection.cs | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/src/TDSProxy/TDSConnection.cs b/src/TDSProxy/TDSConnection.cs index 5a1b2b5..1aa3eca 100644 --- a/src/TDSProxy/TDSConnection.cs +++ b/src/TDSProxy/TDSConnection.cs @@ -482,6 +482,7 @@ public TDSConnection(TDSProxyService service, // forward from inside to outside, because it seems ODBC and OLEDB clients assume their SSL layer will deliver them // complete packets. Gross! outsideClient.NoDelay = false; + EnableKeepAlive(outsideClient.Client); _outsideEP = (IPEndPoint)outsideClient.Client.RemoteEndPoint; _outsideClient = outsideClient; @@ -492,6 +493,7 @@ public TDSConnection(TDSProxyService service, _insideEP = insideEndPoint; _insideClient = new TcpClient(_insideEP.AddressFamily) {NoDelay = false}; _insideClient.Connect(insideEndPoint); + EnableKeepAlive(_insideClient.Client); _insideStream = _insideClient.GetStream(); _insideActiveStream = _insideStream; // Start with plain stream, may upgrade to SSL later _serverTlsConfig = listener.ServerTlsConfig; @@ -499,6 +501,22 @@ public TDSConnection(TDSProxyService service, _processingTask = ProcessConnection(); } + /// + /// Turn on TCP keep-alive so a peer that disappears without FIN or RST is noticed. + /// A link that is cut rather than closed leaves a blocking read waiting forever: + /// nothing arrives, no error is raised, and the connection pair is stranded until the + /// process is restarted. With keep-alive the socket faults after roughly 90 seconds + /// (60s idle, then 3 probes 10s apart), the catch in ProcessConnection runs, and + /// Close() tears down BOTH legs — so the client is told as well. + /// + static void EnableKeepAlive(Socket socket) + { + socket.SetSocketOption(SocketOptionLevel.Socket, SocketOptionName.KeepAlive, true); + socket.SetSocketOption(SocketOptionLevel.Tcp, SocketOptionName.TcpKeepAliveTime, 60); + socket.SetSocketOption(SocketOptionLevel.Tcp, SocketOptionName.TcpKeepAliveInterval, 10); + socket.SetSocketOption(SocketOptionLevel.Tcp, SocketOptionName.TcpKeepAliveRetryCount, 3); + } + ~TDSConnection() { if (_state != StateEnum.Closed)