agent: bound how long reconnection is delayed after the network recovers

The reconnection interval grows to 3 minutes, and nothing shortens it when the
network becomes usable again - the attempts made while it was down are what
pushed it there. After a network change that is the larger part of the delay:
measured with a SOCKS proxy that stalls the open streams and refuses new ones,
the client was dropped after 119 seconds, but reconnected 50 seconds after the
proxy started accepting again, with the interval still growing.

Detecting the dead client sooner makes this worse rather than better, as more of
the attempts are spent while the network cannot work yet.

src/Simplex/Messaging/Agent/Env/SQLite.hs

@@ -198,7 +198,9 @@ defaultReconnectInterval =
   RetryInterval
     { initialInterval = 2_000000,
       increaseAfter = 10_000000,
-      maxInterval = 180_000000
+      -- the network can become usable at any point in this interval, and nothing shortens it,
+      -- so it bounds how long reconnection is delayed after the network recovers
+      maxInterval = 30_000000
     }
 
 defaultMessageRetryInterval :: RetryInterval2

src/Simplex/Messaging/Client.hs

@@ -675,19 +675,22 @@ getProtocolClient g nm transportSession@(_, srv, _) cfg@ProtocolClientConfig {qS
       where
         loop :: Int64 -> IO ()
         loop delay = do
-          threadDelay' delay
+          -- the client is dropped here, so the loop cannot sleep past the next request timeout
+          threadDelay' $ if maxCnt > 0 then min delay requestTimeout else delay
           diff <- diffUTCTime <$> getCurrentTime <*> readTVarIO lastReceived
           let idle = diffToMicroseconds diff
               remaining = smpPingInterval - idle
-          if remaining > 1_000_000 -- delay pings only for significant time
-            then loop remaining
-            else do
-              whenM (readTVarIO sendPings) $ void . runExceptT $ sendProtocolCommand c NRMBackground Nothing NoEntity (protocolPing @v @err @msg)
-              -- sendProtocolCommand/getResponse updates counter for each command
-              cnt <- readTVarIO timeoutErrorCount
-              -- drop client when maxCnt of commands have timed out in sequence, but only after some time has passed after last received response
-              when (maxCnt == 0 || cnt < maxCnt || diff < recoverWindow) $ loop smpPingInterval
-        recoverWindow = 15 * 60 -- seconds
+              due = remaining <= 1_000_000 -- delay pings only for significant time
+          when due $ whenM (readTVarIO sendPings) $ void . runExceptT $ sendProtocolCommand c NRMBackground Nothing NoEntity (protocolPing @v @err @msg)
+          -- sendProtocolCommand/getResponse updates counter for each command
+          cnt <- readTVarIO timeoutErrorCount
+          -- drop client when maxCnt of commands have timed out in sequence, but only after some time has passed after last received response
+          when (maxCnt == 0 || cnt < maxCnt || idle < recoverWindow) $ loop $ if due then smpPingInterval else remaining
+        -- the counter only changes when a request times out, so nothing is lost by checking it that often
+        requestTimeout = fromIntegral $ netTimeoutInt tcpTimeout NRMBackground
+        -- the counter requires maxCnt requests to time out, so the window only has to cover their timeouts,
+        -- with a floor, so that a low request timeout cannot drop a connection that is only slow
+        recoverWindow = max 90_000000 $ fromIntegral maxCnt * requestTimeout
         maxCnt = smpPingCount networkConfig
 
     process :: ProtocolClient v err msg -> IO ()