diff --git a/.cargo/config.toml b/.cargo/config.toml new file mode 100644 index 000000000..bff29e6e1 --- /dev/null +++ b/.cargo/config.toml @@ -0,0 +1,2 @@ +[build] +rustflags = ["--cfg", "tokio_unstable"] diff --git a/Cargo.lock b/Cargo.lock index 6b0db91bc..a4242195b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -164,6 +164,17 @@ dependencies = [ "tokio", ] +[[package]] +name = "async-trait" +version = "0.1.89" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "atomic-waker" version = "1.1.2" @@ -206,7 +217,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5b098575ebe77cb6d14fc7f32749631a6e44edbef6b796f89b020e99ba20d425" dependencies = [ "axum-core", - "base64", + "base64 0.22.1", "bytes", "form_urlencoded", "futures-util", @@ -297,6 +308,12 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4c7f02d4ea65f2c1853089ffd8d2787bdbc63de2f0d29dedbcf8ccdfa0ccd4cf" +[[package]] +name = "base64" +version = "0.21.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d297deb1925b89f2ccc13d7635fa0714f12c87adce1c75356b39ca9b7178567" + [[package]] name = "base64" version = "0.22.1" @@ -508,6 +525,46 @@ dependencies = [ "crossbeam-utils", ] +[[package]] +name = "console-api" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8599749b6667e2f0c910c1d0dff6901163ff698a52d5a39720f61b5be4b20d3" +dependencies = [ + "futures-core", + "prost", + "prost-types", + "tonic", + "tonic-prost", + "tracing-core", +] + +[[package]] +name = "console-subscriber" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb4915b7d8dd960457a1b6c380114c2944f728e7c65294ab247ae6b6f1f37592" +dependencies = [ + "console-api", + "crossbeam-channel", + "crossbeam-utils", + "futures-task", + "hdrhistogram", + "humantime", + "hyper-util", + "prost", + "prost-types", + "serde", + "serde_json", + "thread_local", + "tokio", + "tokio-stream", + "tonic", + "tracing", + "tracing-core", + "tracing-subscriber", +] + [[package]] name = "const-oid" version = "0.9.6" @@ -557,6 +614,15 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "crossbeam-channel" +version = "0.5.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "82b8f8f868b36967f9606790d1903570de9ceaf870a7bf9fbbd3016d636a2cb2" +dependencies = [ + "crossbeam-utils", +] + [[package]] name = "crossbeam-utils" version = "0.8.21" @@ -831,6 +897,12 @@ version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3a3076410a55c90011c298b04d0cfa770b00fa04e1e3c97d3f6c9de105a03844" +[[package]] +name = "fixedbitset" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ce7134b9999ecaf8bcd65542e436736ef32ddca1b3e06094cb6ec5755203b80" + [[package]] name = "flate2" version = "1.1.5" @@ -1100,6 +1172,19 @@ version = "0.16.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" +[[package]] +name = "hdrhistogram" +version = "7.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "765c9198f173dd59ce26ff9f95ef0aafd0a0fe01fb9d72841bc5066a4c06511d" +dependencies = [ + "base64 0.21.7", + "byteorder", + "flate2", + "nom", + "num-traits", +] + [[package]] name = "heck" version = "0.5.0" @@ -1181,6 +1266,12 @@ version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" +[[package]] +name = "humantime" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "135b12329e5e3ce057a9f972339ea52bc954fe1e9358ef27f95e89716fbc5424" + [[package]] name = "hyper" version = "1.8.1" @@ -1221,13 +1312,26 @@ dependencies = [ "webpki-roots", ] +[[package]] +name = "hyper-timeout" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b90d566bffbce6a75bd8b09a05aa8c2cb1fabb6cb348f8840c9e4c90a0d83b0" +dependencies = [ + "hyper", + "hyper-util", + "pin-project-lite", + "tokio", + "tower-service", +] + [[package]] name = "hyper-util" version = "0.1.19" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "727805d60e7938b76b826a6ef209eb70eaa1812794f9424d4a4e2d740662df5f" dependencies = [ - "base64", + "base64 0.22.1", "bytes", "futures-channel", "futures-core", @@ -1422,6 +1526,15 @@ version = "1.70.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" +[[package]] +name = "itertools" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b192c782037fadd9cfa75548310488aabdbf3d2da73885b31bd0abd03351285" +dependencies = [ + "either", +] + [[package]] name = "itoa" version = "1.0.15" @@ -1477,7 +1590,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c76e1c7d7df3e34443b3621b459b066a7b79644f059fc8b2db7070c825fd417e" dependencies = [ "aws-lc-rs", - "base64", + "base64 0.22.1", "getrandom 0.2.16", "js-sys", "pem", @@ -1669,9 +1782,11 @@ dependencies = [ "anyhow", "bytes", "clap", + "console-subscriber", "futures", "hex", "moq-lite", + "parking_lot", "quinn", "rcgen", "reqwest", @@ -1723,7 +1838,7 @@ version = "0.5.5" dependencies = [ "anyhow", "aws-lc-rs", - "base64", + "base64 0.22.1", "elliptic-curve", "jsonwebtoken", "p256", @@ -1978,8 +2093,10 @@ version = "0.9.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1" dependencies = [ + "backtrace", "cfg-if", "libc", + "petgraph", "redox_syscall", "smallvec", "windows-link", @@ -1997,7 +2114,7 @@ version = "3.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1d30c53c26bc5b31a98cd02d20f25a7c8567146caf63ed593a9d87b2775291be" dependencies = [ - "base64", + "base64 0.22.1", "serde_core", ] @@ -2016,6 +2133,36 @@ version = "2.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" +[[package]] +name = "petgraph" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4c5cc86750666a3ed20bdaf5ca2a0344f9c67674cae0515bec2da16fbaa47db" +dependencies = [ + "fixedbitset", + "indexmap 2.12.1", +] + +[[package]] +name = "pin-project" +version = "1.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "677f1add503faace112b9f1373e43e9e054bfdd22ff1a63c1bc485eaec6a6a8a" +dependencies = [ + "pin-project-internal", +] + +[[package]] +name = "pin-project-internal" +version = "1.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e918e4ff8c4549eb882f14b3a4bc8c8bc93de829416eacf579f1207a8fbf861" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "pin-project-lite" version = "0.2.16" @@ -2100,6 +2247,38 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "prost" +version = "0.14.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7231bd9b3d3d33c86b58adbac74b5ec0ad9f496b19d22801d773636feaa95f3d" +dependencies = [ + "bytes", + "prost-derive", +] + +[[package]] +name = "prost-derive" +version = "0.14.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9120690fafc389a67ba3803df527d0ec9cbbc9cc45e4cc20b332996dfb672425" +dependencies = [ + "anyhow", + "itertools", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "prost-types" +version = "0.14.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9b4db3d6da204ed77bb26ba83b6122a73aeb2e87e25fbf7ad2e84c4ccbf8f72" +dependencies = [ + "prost", +] + [[package]] name = "quinn" version = "0.11.9" @@ -2310,7 +2489,7 @@ version = "0.12.26" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3b4c14b2d9afca6a60277086b0cc6a6ae0b568f6f7916c943a8cdc79f8be240f" dependencies = [ - "base64", + "base64 0.22.1", "bytes", "futures-core", "http", @@ -2708,7 +2887,7 @@ version = "3.16.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4fa237f2807440d238e0364a218270b98f767a00d3dada77b1c53ae88940e2e7" dependencies = [ - "base64", + "base64 0.22.1", "chrono", "hex", "indexmap 1.9.3", @@ -3032,6 +3211,7 @@ dependencies = [ "signal-hook-registry", "socket2", "tokio-macros", + "tracing", "windows-sys 0.61.2", ] @@ -3056,6 +3236,17 @@ dependencies = [ "tokio", ] +[[package]] +name = "tokio-stream" +version = "0.1.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eca58d7bba4a75707817a2c44174253f9236b2d5fbd055602e9d5c07c139a047" +dependencies = [ + "futures-core", + "pin-project-lite", + "tokio", +] + [[package]] name = "tokio-tungstenite" version = "0.24.0" @@ -3144,6 +3335,46 @@ version = "1.0.6+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ab16f14aed21ee8bfd8ec22513f7287cd4a91aa92e44edfe2c17ddd004e92607" +[[package]] +name = "tonic" +version = "0.14.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eb7613188ce9f7df5bfe185db26c5814347d110db17920415cf2fbcad85e7203" +dependencies = [ + "async-trait", + "axum", + "base64 0.22.1", + "bytes", + "h2", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-timeout", + "hyper-util", + "percent-encoding", + "pin-project", + "socket2", + "sync_wrapper", + "tokio", + "tokio-stream", + "tower", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "tonic-prost" +version = "0.14.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "66bd50ad6ce1252d87ef024b3d64fe4c3cf54a86fb9ef4c631fdd0ded7aeaa67" +dependencies = [ + "bytes", + "prost", + "tonic", +] + [[package]] name = "tower" version = "0.5.2" @@ -3152,9 +3383,12 @@ checksum = "d039ad9159c98b70ecfd540b2573b97f7f52c3e8d9f8ad57a24b916a536975f9" dependencies = [ "futures-core", "futures-util", + "indexmap 2.12.1", "pin-project-lite", + "slab", "sync_wrapper", "tokio", + "tokio-util", "tower-layer", "tower-service", "tracing", diff --git a/justfile b/justfile index 3907fc39b..f23bc544d 100644 --- a/justfile +++ b/justfile @@ -34,7 +34,7 @@ dev: # Run a localhost relay server without authentication. relay: # Run the relay server overriding the provided configuration file. - cargo run --bin moq-relay -- dev/relay.toml + TOKIO_CONSOLE_BIND=127.0.0.1:6680 cargo run --bin moq-relay -- dev/relay.toml # Run a cluster of relay servers cluster: @@ -144,7 +144,7 @@ pub name url="http://localhost:4443/anon" *args: -i "dev/{{name}}.fmp4" \ -c copy \ -f mp4 -movflags cmaf+separate_moof+delay_moov+skip_trailer+frag_every_frame \ - - | cargo run --bin hang -- publish --url "{{url}}" --name "{{name}}" fmp4 {{args}} + - | TOKIO_CONSOLE_BIND=127.0.0.1:6681 cargo run --bin hang -- publish --url "{{url}}" --name "{{name}}" fmp4 {{args}} # Generate and ingest an HLS stream from a video file. pub-hls name relay="http://localhost:4443/anon": @@ -258,7 +258,7 @@ serve name: # Run the web server web url='http://localhost:4443/anon': - VITE_RELAY_URL="{{url}}" bun run --filter='*' dev + cd js/hang-demo && VITE_RELAY_URL="{{url}}" bun run dev # Publish the clock broadcast # `action` is either `publish` or `subscribe` @@ -396,6 +396,14 @@ serve-hls name port="8000": echo ">>> HTTP server: http://localhost:{{port}}/" cd "$OUT_DIR" && python3 -m http.server {{port}} +# Connect tokio-console to the relay server (port 6680) +relay-console: + tokio-console http://127.0.0.1:6680 + +# Connect tokio-console to the publisher (port 6681) +pub-console: + tokio-console http://127.0.0.1:6681 + # Serve the documentation locally. doc: cd doc && bun run dev diff --git a/rs/moq-native/Cargo.toml b/rs/moq-native/Cargo.toml index d38df66ff..39d940272 100644 --- a/rs/moq-native/Cargo.toml +++ b/rs/moq-native/Cargo.toml @@ -15,14 +15,17 @@ categories = ["multimedia", "network-programming", "web-programming"] default = ["aws-lc-rs"] aws-lc-rs = ["rustls/aws-lc-rs", "rcgen/aws_lc_rs", "quinn/rustls-aws-lc-rs"] ring = ["rustls/ring", "rcgen/ring", "quinn/rustls-ring"] +tokio-console = ["dep:console-subscriber"] [dependencies] anyhow = { version = "1", features = ["backtrace"] } clap = { version = "4", features = ["derive", "env"] } +console-subscriber = { version = "0.5", optional = true } futures = "0.3" hex = "0.4" moq-lite = { workspace = true } +parking_lot = { version = "0.12", features = ["deadlock_detection"] } quinn = { version = "0.11", default-features = false, features = [ "platform-verifier", "runtime-tokio", diff --git a/rs/moq-native/src/log.rs b/rs/moq-native/src/log.rs index 745f0fa47..e18a488e2 100644 --- a/rs/moq-native/src/log.rs +++ b/rs/moq-native/src/log.rs @@ -2,7 +2,10 @@ use serde::{Deserialize, Serialize}; use serde_with::DisplayFromStr; use tracing::level_filters::LevelFilter; use tracing::Level; +use tracing_subscriber::layer::SubscriberExt; +use tracing_subscriber::util::SubscriberInitExt; use tracing_subscriber::EnvFilter; +use tracing_subscriber::Layer; #[serde_with::serde_as] #[derive(Clone, clap::Parser, Serialize, Deserialize, Debug)] @@ -32,13 +35,54 @@ impl Log { .add_directive("h2=warn".parse().unwrap()) .add_directive("quinn=info".parse().unwrap()) .add_directive("tracing::span=off".parse().unwrap()) - .add_directive("tracing::span::active=off".parse().unwrap()); + .add_directive("tracing::span::active=off".parse().unwrap()) + .add_directive("tokio=info".parse().unwrap()) + .add_directive("runtime=info".parse().unwrap()); - let logger = tracing_subscriber::FmtSubscriber::builder() + let fmt_layer = tracing_subscriber::fmt::layer() .with_writer(std::io::stderr) - .with_env_filter(filter) - .finish(); + .with_filter(filter); - tracing::subscriber::set_global_default(logger).unwrap(); + #[cfg(feature = "tokio-console")] + { + let console_layer = console_subscriber::spawn(); + tracing_subscriber::registry() + .with(fmt_layer) + .with(console_layer) + .init(); + } + + #[cfg(not(feature = "tokio-console"))] + { + tracing_subscriber::registry().with(fmt_layer).init(); + } + + // Start deadlock detection thread (only in debug builds) + #[cfg(debug_assertions)] + std::thread::spawn(Self::deadlock_detector); + } + + #[cfg(debug_assertions)] + fn deadlock_detector() { + loop { + std::thread::sleep(std::time::Duration::from_secs(1)); + + let deadlocks = parking_lot::deadlock::check_deadlock(); + if deadlocks.is_empty() { + continue; + } + + tracing::error!("DEADLOCK DETECTED"); + + for (i, threads) in deadlocks.iter().enumerate() { + tracing::error!("Deadlock #{}", i); + for t in threads { + tracing::error!("Thread Id {:#?}", t.thread_id()); + tracing::error!("{:#?}", t.backtrace()); + } + } + + // Optionally: std::process::abort() to get a core dump + } } }