diff --git a/src/StackExchange.Redis/PhysicalBridge.cs b/src/StackExchange.Redis/PhysicalBridge.cs index c430cf5af..95b1d62e9 100644 --- a/src/StackExchange.Redis/PhysicalBridge.cs +++ b/src/StackExchange.Redis/PhysicalBridge.cs @@ -615,7 +615,7 @@ internal void OnHeartbeat(bool ifConnectedOnly) Interlocked.Exchange(ref connectTimeoutRetryCount, 0); tmp.BridgeCouldBeNull?.ServerEndPoint?.ClearUnselectable(UnselectableFlags.DidNotRespond); } - int timedOutThisHeartbeat = tmp.OnBridgeHeartbeat(); + tmp.OnBridgeHeartbeat(out int asyncTimeoutThisHeartbeat, out int syncTimeoutThisHeartbeat); int writeEverySeconds = ServerEndPoint.WriteEverySeconds; bool configCheckDue = ServerEndPoint.ConfigCheckSeconds > 0 && ServerEndPoint.LastInfoReplicationCheckSecondsAgo >= ServerEndPoint.ConfigCheckSeconds; @@ -664,14 +664,15 @@ internal void OnHeartbeat(bool ifConnectedOnly) } // This is an "always" check - we always want to evaluate a dead connection from a non-responsive sever regardless of the need to heartbeat above - if (timedOutThisHeartbeat > 0 + var totalTimeoutThisHeartbeat = asyncTimeoutThisHeartbeat + syncTimeoutThisHeartbeat; + if ((totalTimeoutThisHeartbeat > 0) && tmp.LastReadSecondsAgo * 1_000 > (tmp.BridgeCouldBeNull?.Multiplexer.AsyncTimeoutMilliseconds * 4)) { // If we've received *NOTHING* on the pipe in 4 timeouts worth of time and we're timing out commands, issue a connection failure so that we reconnect // This is meant to address the scenario we see often in Linux configs where TCP retries will happen for 15 minutes. // To us as a client, we'll see the socket as green/open/fine when writing but we'll bet getting nothing back. // Since we can't depend on the pipe to fail in that case, we want to error here based on the criteria above so we reconnect broken clients much faster. - tmp.BridgeCouldBeNull?.Multiplexer.Logger?.LogWarningDeadSocketDetected(tmp.LastReadSecondsAgo, timedOutThisHeartbeat); + tmp.BridgeCouldBeNull?.Multiplexer.Logger?.LogWarningDeadSocketDetected(tmp.LastReadSecondsAgo, totalTimeoutThisHeartbeat); OnDisconnected(ConnectionFailureType.SocketFailure, tmp, out _, out State oldState); tmp.Dispose(); // Cleanup the existing connection/socket if any, otherwise it will wait reading indefinitely } diff --git a/src/StackExchange.Redis/PhysicalConnection.cs b/src/StackExchange.Redis/PhysicalConnection.cs index 129fd9e07..4b86d7a51 100644 --- a/src/StackExchange.Redis/PhysicalConnection.cs +++ b/src/StackExchange.Redis/PhysicalConnection.cs @@ -746,10 +746,12 @@ internal void GetStormLog(StringBuilder sb) /// /// Runs on every heartbeat for a bridge, timing out any commands that are overdue and returning an integer of how many we timed out. /// - /// How many commands were overdue and threw timeout exceptions. - internal int OnBridgeHeartbeat() + /// How many async commands were overdue and threw timeout exceptions. + /// How many sync commands were overdue. No exception are thrown for these commands here. + internal void OnBridgeHeartbeat(out int asyncTimeoutDetected, out int syncTimeoutDetected) { - var result = 0; + asyncTimeoutDetected = 0; + syncTimeoutDetected = 0; var now = Environment.TickCount; Interlocked.Exchange(ref lastBeatTickCount, now); @@ -776,7 +778,13 @@ internal int OnBridgeHeartbeat() multiplexer.OnMessageFaulted(msg, timeoutEx); msg.SetExceptionAndComplete(timeoutEx, bridge); // tell the message that it is doomed multiplexer.OnAsyncTimeout(); - result++; + asyncTimeoutDetected++; + } + else + { + // Only count how many sync timeouts we detect here. + // The actual timeout is handled in ConnectionMultiplexer.ExecuteSyncImpl(). + syncTimeoutDetected++; } } else @@ -791,7 +799,6 @@ internal int OnBridgeHeartbeat() } } } - return result; } internal void OnInternalError(Exception exception, [CallerMemberName] string? origin = null)