Skip to content

Commit d50545c

Browse files
authored
fix(core): send ws close frames before shutting down the pubsub server (solana-foundation#863)
1 parent e312963 commit d50545c

2 files changed

Lines changed: 140 additions & 6 deletions

File tree

‎crates/core/src/runloops/mod.rs‎

Lines changed: 57 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -4,9 +4,9 @@ use std::{
44
collections::{HashMap, HashSet},
55
net::SocketAddr,
66
path::PathBuf,
7-
sync::{Arc, RwLock},
7+
sync::{Arc, Mutex, RwLock},
88
thread::{JoinHandle, sleep},
9-
time::Duration,
9+
time::{Duration, Instant},
1010
};
1111

1212
use agave_geyser_plugin_interface::geyser_plugin_interface::{
@@ -19,7 +19,9 @@ use itertools::Itertools;
1919
use jsonrpc_core::MetaIoHandler;
2020
use jsonrpc_http_server::{DomainsValidation, ServerBuilder};
2121
use jsonrpc_pubsub::{PubSubHandler, Session};
22-
use jsonrpc_ws_server::{RequestContext, ServerBuilder as WsServerBuilder};
22+
use jsonrpc_ws_server::{
23+
RequestContext, ServerBuilder as WsServerBuilder, SessionId, SessionStats, ws::CloseCode,
24+
};
2325
use libloading::{Library, Symbol};
2426
use serde::Serialize;
2527
use solana_commitment_config::CommitmentConfig;
@@ -998,19 +1000,57 @@ impl RpcServer for jsonrpc_http_server::Server {
9981000
/// contexts, where they panic.
9991001
struct WsServerWithRuntime {
10001002
server: jsonrpc_ws_server::Server,
1003+
sessions: WsSessions,
10011004
_runtime: tokio::runtime::Runtime,
10021005
}
10031006

10041007
impl RpcServer for WsServerWithRuntime {
1005-
type CloseHandle = jsonrpc_ws_server::CloseHandle;
1008+
type CloseHandle = WsCloseHandle;
10061009
fn close_handle(&self) -> Self::CloseHandle {
1007-
self.server.close_handle()
1010+
WsCloseHandle {
1011+
server: self.server.close_handle(),
1012+
sessions: self.sessions.clone(),
1013+
}
10081014
}
10091015
fn wait(self) -> Result<(), String> {
10101016
self.server.wait().map_err(|e| format!("{:?}", e))
10111017
}
10121018
}
10131019

1020+
const WS_CLOSE_GRACE: Duration = Duration::from_secs(1);
1021+
1022+
type WsSessionCloser = Arc<dyn Fn() + Send + Sync>;
1023+
1024+
#[derive(Clone, Default)]
1025+
struct WsSessions(Arc<Mutex<HashMap<SessionId, WsSessionCloser>>>);
1026+
1027+
impl SessionStats for WsSessions {
1028+
fn open_session(&self, _id: SessionId) {}
1029+
fn close_session(&self, id: SessionId) {
1030+
self.0.lock().unwrap().remove(&id);
1031+
}
1032+
}
1033+
1034+
/// parity-ws drops its queued close frames on shutdown, so each session is closed first.
1035+
struct WsCloseHandle {
1036+
server: jsonrpc_ws_server::CloseHandle,
1037+
sessions: WsSessions,
1038+
}
1039+
1040+
impl WsCloseHandle {
1041+
fn close(self) {
1042+
let closers: Vec<_> = self.sessions.0.lock().unwrap().values().cloned().collect();
1043+
for close_session in closers {
1044+
close_session();
1045+
}
1046+
let deadline = Instant::now() + WS_CLOSE_GRACE;
1047+
while !self.sessions.0.lock().unwrap().is_empty() && Instant::now() < deadline {
1048+
sleep(Duration::from_millis(10));
1049+
}
1050+
self.server.close();
1051+
}
1052+
}
1053+
10141054
/// Runs one RPC server on a dedicated thread and reports how startup
10151055
/// went: the close handle on success, the error on failure. On failure
10161056
/// the error is published on the handshake channel first and the aborted
@@ -1159,7 +1199,7 @@ fn start_ws_rpc_server_runloop(
11591199
config: &SurfpoolConfig,
11601200
middleware: SurfpoolMiddleware,
11611201
simnet_events_tx: SimnetEventsTx,
1162-
) -> Result<(JoinHandle<()>, jsonrpc_ws_server::CloseHandle), String> {
1202+
) -> Result<(JoinHandle<()>, WsCloseHandle), String> {
11631203
let ws_server_bind: SocketAddr = config
11641204
.rpc
11651205
.get_ws_base_url()
@@ -1192,8 +1232,17 @@ fn start_ws_rpc_server_runloop(
11921232
.to_delegate(),
11931233
);
11941234

1235+
let sessions = WsSessions::default();
1236+
let tracked_sessions = sessions.clone();
11951237
let server = WsServerBuilder::new(rpc_io)
11961238
.session_meta_extractor(move |ctx: &RequestContext| {
1239+
let out = ctx.out.clone();
1240+
tracked_sessions.0.lock().unwrap().insert(
1241+
ctx.session_id,
1242+
Arc::new(move || {
1243+
let _ = out.close(CloseCode::Normal);
1244+
}),
1245+
);
11971246
let runloop_context = RunloopContext {
11981247
id: None,
11991248
svm_locker: middleware.surfnet_svm.clone(),
@@ -1215,11 +1264,13 @@ fn start_ws_rpc_server_runloop(
12151264
// shared handle removes that runtime (and its worker threads)
12161265
// entirely.
12171266
.event_loop_executor(tokio_handle)
1267+
.session_stats(sessions.clone())
12181268
.start(&ws_server_bind)
12191269
.map_err(|e| format!("{:?}", e))?;
12201270

12211271
Ok(WsServerWithRuntime {
12221272
server,
1273+
sessions,
12231274
_runtime: runtime,
12241275
})
12251276
})

‎crates/sdk/tests/ws_shutdown.rs‎

Lines changed: 83 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,83 @@
1+
use std::{
2+
io::{Read, Write},
3+
net::TcpStream,
4+
time::Duration,
5+
};
6+
7+
use surfpool_sdk::Surfnet;
8+
9+
const SERVER_NORMAL_CLOSE: [u8; 4] = [0x88, 0x02, 0x03, 0xE8];
10+
const CLIENT_NORMAL_CLOSE_MASKED: [u8; 8] = [0x88, 0x82, 0, 0, 0, 0, 0x03, 0xE8];
11+
12+
fn open_websocket(ws_url: &str) -> TcpStream {
13+
let address = ws_url.trim_start_matches("ws://");
14+
let mut stream = TcpStream::connect(address).expect("ws port should accept connections");
15+
stream
16+
.set_read_timeout(Some(Duration::from_secs(10)))
17+
.unwrap();
18+
let handshake = format!(
19+
"GET / HTTP/1.1\r\nHost: {address}\r\nUpgrade: websocket\r\nConnection: Upgrade\r\nSec-WebSocket-Key: dGhlIHNhbXBsZSBub25jZQ==\r\nSec-WebSocket-Version: 13\r\n\r\n"
20+
);
21+
stream.write_all(handshake.as_bytes()).unwrap();
22+
23+
let mut response = Vec::new();
24+
let mut byte = [0u8; 1];
25+
while !response.ends_with(b"\r\n\r\n") {
26+
stream.read_exact(&mut byte).expect("handshake response");
27+
response.push(byte[0]);
28+
}
29+
let response = String::from_utf8_lossy(&response);
30+
assert!(
31+
response.starts_with("HTTP/1.1 101"),
32+
"the upgrade should be accepted, got: {response}"
33+
);
34+
stream
35+
}
36+
37+
#[tokio::test(flavor = "multi_thread")]
38+
async fn stop_closes_websockets_with_a_normal_close_frame() {
39+
let mut surfnet = Surfnet::start().await.expect("surfnet should start");
40+
let mut stream = open_websocket(surfnet.ws_url());
41+
42+
let stop = tokio::task::spawn_blocking(move || surfnet.stop());
43+
44+
let mut frame = [0u8; 4];
45+
stream
46+
.read_exact(&mut frame)
47+
.expect("the server should send a close frame before dropping the socket");
48+
assert_eq!(frame, SERVER_NORMAL_CLOSE);
49+
50+
stream.write_all(&CLIENT_NORMAL_CLOSE_MASKED).unwrap();
51+
let mut rest = Vec::new();
52+
stream.read_to_end(&mut rest).unwrap();
53+
assert!(rest.is_empty(), "nothing should follow the close frame");
54+
55+
stop.await.unwrap().expect("surfnet should stop cleanly");
56+
}
57+
58+
#[tokio::test(flavor = "multi_thread")]
59+
async fn stop_does_not_wait_forever_on_unanswered_close_frames() {
60+
let mut surfnet = Surfnet::start().await.expect("surfnet should start");
61+
let mut stream = open_websocket(surfnet.ws_url());
62+
63+
let stop = tokio::task::spawn_blocking(move || surfnet.stop());
64+
65+
let mut frame = [0u8; 4];
66+
stream
67+
.read_exact(&mut frame)
68+
.expect("the server should send a close frame before dropping the socket");
69+
assert_eq!(frame, SERVER_NORMAL_CLOSE);
70+
71+
tokio::time::timeout(Duration::from_secs(4), stop)
72+
.await
73+
.expect("stop should give up on the close handshake after the grace period")
74+
.unwrap()
75+
.expect("surfnet should stop cleanly");
76+
77+
let mut rest = Vec::new();
78+
stream.read_to_end(&mut rest).unwrap();
79+
assert!(
80+
rest.is_empty(),
81+
"the server should drop the socket after the grace period"
82+
);
83+
}

0 commit comments

Comments
 (0)