Add a 60-second client reconnect (#80)

* Add a 60-second client reconnect

This improves reliability in most cases, such as when network errors are not reported immediately to the streaming request client. It's at the expense of a brief blip during every reconnection. But this should hardly be noticeable, assuming real-time RTL, it would affect ~0.2% of interactions.

* Remove dbg!

* Fix interval

Author: ekzhang

crates/sshx/src/controller.rs

@@ -1,6 +1,7 @@
 //! Network gRPC client allowing server control of terminals.
 
 use std::collections::HashMap;
+use std::pin::pin;
 
 use anyhow::{Context, Result};
 use sshx_core::proto::{
@@ -21,6 +22,9 @@ use crate::runner::{Runner, ShellData};
 /// Interval for sending empty heartbeat messages to the server.
 const HEARTBEAT_INTERVAL: Duration = Duration::from_secs(2);
 
+/// Interval to automatically reestablish connections.
+const RECONNECT_INTERVAL: Duration = Duration::from_secs(60);
+
 /// Handles a single session's communication with the remote server.
 pub struct Controller {
     origin: String,
@@ -129,6 +133,7 @@ impl Controller {
 
         let mut interval = time::interval(HEARTBEAT_INTERVAL);
         interval.set_missed_tick_behavior(MissedTickBehavior::Delay);
+        let mut reconnect = pin!(time::sleep(RECONNECT_INTERVAL));
         loop {
             let message = tokio::select! {
                 _ = interval.tick() => {
@@ -145,6 +150,9 @@ impl Controller {
                         .server_message
                         .context("server message is missing")?
                 }
+                _ = &mut reconnect => {
+                    return Ok(()); // Reconnect to the server.
+                }
             };
 
             match message {

fly.toml

@@ -1,7 +1,7 @@
 app = "sshx"
 primary_region = "ewr"
 kill_signal = "SIGINT"
-kill_timeout = 5
+kill_timeout = 90
 
 [experimental]
   auto_rollback = true