Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
70 commits
Select commit Hold shift + click to select a range
20e6243
add opentelemetry for debugging and testing behind `otel` feature flag
mmsinclair Aug 20, 2025
7d7acc2
wip: tracing in nym-node
mmsinclair Aug 20, 2025
5888102
wip: sdk surb-reply example add otel tracing
mmsinclair Aug 26, 2025
2447520
revert changes from previous commit
IRClichtR Aug 27, 2025
fa603be
setup otel from run to forward_sphinx_packet
IRClichtR Aug 27, 2025
a11de8e
fix otel panic
IRClichtR Aug 27, 2025
acc11e8
setup otel for surb_reply
IRClichtR Aug 28, 2025
dacd354
configure format for sending logs to signoz
IRClichtR Aug 28, 2025
f56edf6
fix signoz ingestion
IRClichtR Aug 28, 2025
fee2ae5
change surb_reply to test relation with nym-node + basic instrumentation
IRClichtR Aug 29, 2025
89329ab
add context propagation
IRClichtR Aug 29, 2025
6c03ab6
try debug difference trace_id otel by use async for gateway and attac…
IRClichtR Aug 29, 2025
5c9d58b
fix trace_id propagation from sdk to mix-node
IRClichtR Aug 30, 2025
e3569cc
fixed trace_id distibuted from sdk to gateway
IRClichtR Sep 1, 2025
bb9f054
instrumentation of nym-node to nym-gateway workflow
IRClichtR Sep 1, 2025
bda45b4
try keep context accross async call
IRClichtR Sep 1, 2025
d55fce1
try debug client gets stuck
IRClichtR Sep 1, 2025
c2b620d
debug surb_reply client
IRClichtR Sep 2, 2025
953f930
try debug fresh handler context
IRClichtR Sep 2, 2025
9f3061a
broken: context won't satisfy spawn async move
IRClichtR Sep 2, 2025
1fa6159
Revert "broken: context won't satisfy spawn async move"
IRClichtR Sep 4, 2025
c4543c8
context propagation helpers
IRClichtR Sep 4, 2025
9d8c328
establishing initial authentication common trace_id
IRClichtR Sep 4, 2025
cfe8c08
print debug trace
IRClichtR Sep 4, 2025
7d7beba
try debug with otel specific tools
IRClichtR Sep 4, 2025
9d77486
experiment do I preserve trace_id with with method
IRClichtR Sep 4, 2025
76770fa
wip
IRClichtR Sep 5, 2025
8492c71
manual propagators for context from client to node
IRClichtR Sep 6, 2025
b9a2efe
use authenticatev2 and not authenticate
IRClichtR Sep 6, 2025
26da5da
verify existence of context
IRClichtR Sep 8, 2025
15f56bc
forget to enter span
IRClichtR Sep 8, 2025
f1bf986
use attach() instead of creating new span
IRClichtR Sep 8, 2025
4af1835
add method to extract trace_id from ContextCarrier
IRClichtR Sep 8, 2025
651d0bb
wip
IRClichtR Sep 8, 2025
b937798
propagation to parent function
IRClichtR Sep 8, 2025
2ae73b7
rm clone because panic
IRClichtR Sep 8, 2025
1fef24a
try debug panic when using parent span
IRClichtR Sep 8, 2025
74c0641
parent/chil propagation explicitation
IRClichtR Sep 8, 2025
6ba20e7
context extraction
IRClichtR Sep 8, 2025
32d7328
wip
IRClichtR Sep 8, 2025
faaafbe
wip
IRClichtR Sep 8, 2025
c5e6668
wip
IRClichtR Sep 8, 2025
7d08799
wip
IRClichtR Sep 8, 2025
254a414
change context propagation method to trace_id sharing to go accros as…
IRClichtR Sep 8, 2025
2955e50
add context to message handling
IRClichtR Sep 9, 2025
ca9b1d6
use otel span to parent tracing span
IRClichtR Sep 9, 2025
2805894
cleanup
IRClichtR Sep 10, 2025
682feac
add manual spans to handle_request functions
IRClichtR Sep 10, 2025
8275916
debug node
IRClichtR Sep 11, 2025
36b07b1
wip
IRClichtR Sep 15, 2025
e9cbd0a
add instrument to sdk send message
IRClichtR Sep 15, 2025
f05c5d2
wip
IRClichtR Sep 15, 2025
be37715
wip
IRClichtR Sep 15, 2025
60cde66
websocket otel trace handling correction
IRClichtR Sep 29, 2025
4116bb3
add traceparent to http headers
IRClichtR Sep 30, 2025
47e1e91
traceparent injection into headers
IRClichtR Oct 1, 2025
8e771c3
otel adaptation for sphinx instrumentation
IRClichtR Oct 2, 2025
9a96be3
add trace_id to sphinx packet
IRClichtR Oct 7, 2025
b292c4a
continue trace)id propagation into sphinx
IRClichtR Oct 8, 2025
ad431e8
cleanup and add guard to keep the tracer provider alive
IRClichtR Oct 14, 2025
9c13fe5
featurized otel
IRClichtR Oct 16, 2025
6eab87f
refactor init tracer and use of tracing_opentelemetry span for async …
IRClichtR Oct 17, 2025
ec09c7e
cleanup and PR ready
IRClichtR Oct 20, 2025
8427b09
cleanup final
IRClichtR Oct 21, 2025
2344a43
rebase reconciliation
IRClichtR Oct 21, 2025
2831f4a
wip
IRClichtR Oct 22, 2025
c2a0629
fmt
IRClichtR Oct 22, 2025
69bba68
wip
IRClichtR Oct 22, 2025
3bda29a
featurized InputMessage transformation
IRClichtR Oct 24, 2025
8aa5e61
wip
IRClichtR Oct 27, 2025
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2,292 changes: 1,105 additions & 1,187 deletions Cargo.lock

Large diffs are not rendered by default.

11 changes: 8 additions & 3 deletions Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -300,8 +300,11 @@ nix = "0.27.1"
notify = "5.1.0"
okapi = "0.7.0"
once_cell = "1.21.3"
opentelemetry = "0.19.0"
opentelemetry-jaeger = "0.18.0"
opentelemetry = "0.30.0"
opentelemetry-otlp = "0.30.0"
opentelemetry-semantic-conventions = "0.30.0"
opentelemetry_sdk = "0.30.0"
opentelemetry-stdout = "0.30.0"
parking_lot = "0.12.3"
pem = "0.8"
petgraph = "0.6.5"
Expand Down Expand Up @@ -360,8 +363,10 @@ toml = "0.8.22"
tower = "0.5.2"
tower-http = "0.5.2"
tracing = "0.1.41"
tracing-core = "0.1.33"
tracing-log = "0.2"
tracing-opentelemetry = "0.19.0"
tracing-opentelemetry = "0.31.0"
tracing-serde = "0.2.0"
tracing-subscriber = "0.3.19"
tracing-tree = "0.2.2"
tracing-indicatif = "0.3.9"
Expand Down
9 changes: 8 additions & 1 deletion clients/native/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,6 @@ nym-bandwidth-controller = { path = "../../common/bandwidth-controller" }
nym-bin-common = { path = "../../common/bin-common", features = [
"output_format",
"clap",
"basic_tracing",
] }
nym-client-core = { path = "../../common/client-core", features = [
"fs-credentials-storage",
Expand All @@ -71,3 +70,11 @@ nym-client-websocket-requests = { path = "websocket-requests" }
nym-id = { path = "../../common/nym-id" }

[dev-dependencies]

[features]
default = []
otel = [
"nym-client-core/otel",
"nym-bin-common/otel",
"nym-gateway-requests/otel",
]
4 changes: 2 additions & 2 deletions clients/native/src/main.rs
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
use std::error::Error;

use clap::{crate_name, crate_version, Parser};
use nym_bin_common::logging::{maybe_print_banner, setup_tracing_logger};
use nym_bin_common::logging::{maybe_print_banner, setup_no_otel_logger};
use nym_network_defaults::setup_env;

pub mod client;
Expand All @@ -20,7 +20,7 @@ async fn main() -> Result<(), Box<dyn Error + Send + Sync>> {
if !args.no_banner {
maybe_print_banner(crate_name!(), crate_version!());
}
setup_tracing_logger();
setup_no_otel_logger().expect("failed to initialize logging");

if let Err(err) = commands::execute(args).await {
log::error!("{err}");
Expand Down
20 changes: 17 additions & 3 deletions clients/native/src/websocket/handler.rs
Original file line number Diff line number Diff line change
Expand Up @@ -184,7 +184,14 @@ impl Handler {
});

// the ack control is now responsible for chunking, etc.
let input_msg = InputMessage::new_regular(recipient, message, lane, self.packet_type);
let input_msg = InputMessage::new_regular(
recipient,
message,
lane,
self.packet_type,
#[cfg(feature = "otel")]
None,
);
if let Err(err) = self.msg_input.send(input_msg).await {
if !self.shutdown_token.is_cancelled() {
error!("Failed to send message to the input buffer: {err}");
Expand Down Expand Up @@ -216,8 +223,15 @@ impl Handler {
TransmissionLane::ConnectionId(id)
});

let input_msg =
InputMessage::new_anonymous(recipient, message, reply_surbs, lane, self.packet_type);
let input_msg = InputMessage::new_anonymous(
recipient,
message,
reply_surbs,
lane,
self.packet_type,
#[cfg(feature = "otel")]
None,
);
if let Err(err) = self.msg_input.send(input_msg).await {
if !self.shutdown_token.is_cancelled() {
error!("Failed to send anonymous message to the input buffer: {err}");
Expand Down
5 changes: 4 additions & 1 deletion clients/socks5/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,6 @@ zeroize = { workspace = true }
nym-bin-common = { path = "../../common/bin-common", features = [
"output_format",
"clap",
"basic_tracing",
] }
nym-client-core = { path = "../../common/client-core", features = [
"fs-credentials-storage",
Expand All @@ -54,3 +53,7 @@ nym-validator-client = { path = "../../common/client-libs/validator-client", fea
[features]
default = []
eth = []
otel = [
"nym-socks5-client-core/otel",
"nym-bin-common/otel",
]
4 changes: 2 additions & 2 deletions clients/socks5/src/main.rs
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
use std::error::Error;

use clap::{crate_name, crate_version, Parser};
use nym_bin_common::logging::{maybe_print_banner, setup_tracing_logger};
use nym_bin_common::logging::{maybe_print_banner, setup_no_otel_logger};
use nym_network_defaults::setup_env;

mod commands;
Expand All @@ -19,7 +19,7 @@ async fn main() -> Result<(), Box<dyn Error + Send + Sync>> {
if !args.no_banner {
maybe_print_banner(crate_name!(), crate_version!());
}
setup_tracing_logger();
setup_no_otel_logger().expect("failed to initialize logging");

if let Err(err) = commands::execute(args).await {
log::error!("{err}");
Expand Down
38 changes: 24 additions & 14 deletions common/bin-common/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -8,24 +8,30 @@ license = { workspace = true }
repository = { workspace = true }

[dependencies]
chrono = { workspace = true, optional = true }
cfg-if = { workspace = true }
clap = { workspace = true, features = ["derive"], optional = true }
clap_complete = { workspace = true, optional = true }
clap_complete_fig = { workspace = true, optional = true }
const-str = { workspace = true }
log = { workspace = true }
opentelemetry = { workspace = true, optional = true }
opentelemetry-otlp = { workspace = true,features=["metrics", "grpc-tonic", "tls",
"tls-webpki-roots"], optional = true }
opentelemetry-semantic-conventions = { workspace = true, features = ["semconv_experimental"], optional = true }
opentelemetry-stdout = { workspace = true, features = ["trace", "metrics"], optional = true }
opentelemetry_sdk = { workspace = true, optional = true }
rand = { workspace = true, optional = true }
schemars = { workspace = true, features = ["preserve_order"], optional = true }
serde = { workspace = true, features = ["derive"] }
serde_json = { workspace = true, optional = true }

## tracing
tracing-subscriber = { workspace = true, features = ["env-filter"], optional = true }
tracing-tree = { workspace = true, optional = true }
tracing = { workspace = true, optional = true }
opentelemetry-jaeger = { workspace = true, features = ["rt-tokio", "collector_client", "isahc_collector_client"], optional = true }
thiserror = { workspace = true }
tracing = { workspace = true }
tracing-core = { workspace = true }
tracing-opentelemetry = { workspace = true, optional = true }
tracing-serde = { workspace = true }
tracing-subscriber = { workspace = true, features = ["env-filter", "json"] }
tracing-tree = { workspace = true }
utoipa = { workspace = true, optional = true }
opentelemetry = { workspace = true, features = ["rt-tokio"], optional = true }


[build-dependencies]
vergen = { workspace = true, features = ["build", "git", "gitcl", "rustc", "cargo"] }
Expand All @@ -35,13 +41,17 @@ default = []
openapi = ["utoipa"]
output_format = ["serde_json", "dep:clap"]
bin_info_schema = ["schemars"]
basic_tracing = ["dep:tracing", "tracing-subscriber"]
tracing = [
"basic_tracing",
"tracing-tree",
"opentelemetry-jaeger",
tokio-console = ["otel"]
otel = [
"chrono",
"tracing-opentelemetry",
"opentelemetry",
"opentelemetry-otlp",
"opentelemetry-semantic-conventions",
"opentelemetry-stdout",
"opentelemetry_sdk",
"serde_json",
"rand",
]
clap = ["dep:clap", "dep:clap_complete", "dep:clap_complete_fig"]
models = []
3 changes: 3 additions & 0 deletions common/bin-common/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,9 @@
pub mod build_information;
pub mod logging;

#[cfg(feature = "otel")]
pub mod opentelemetry;

#[cfg(feature = "clap")]
pub mod completions;

Expand Down
20 changes: 20 additions & 0 deletions common/bin-common/src/logging/error.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
// Copyright 2025 - Nym Technologies SA <[email protected]>
// SPDX-License-Identifier: Apache-2.0
#[cfg(feature = "otel")]
use opentelemetry_otlp::ExporterBuildError;

#[derive(thiserror::Error, Debug)]
pub enum TracingError {
#[error("tracing logger already initialised")]
TracingLoggerAlreadyInitialised,

#[error("Logging error: {0}")]
TracingTryInitError(tracing_subscriber::util::TryInitError),

#[cfg(feature = "otel")]
#[error("{0}")]
TracingExporterBuildError(#[from] ExporterBuildError),

#[error("{0}")]
TracingFilterParseError(#[from] tracing_subscriber::filter::ParseError),
}
86 changes: 39 additions & 47 deletions common/bin-common/src/logging/mod.rs
Original file line number Diff line number Diff line change
@@ -1,19 +1,12 @@
// Copyright 2022-2023 - Nym Technologies SA <[email protected]>
// SPDX-License-Identifier: Apache-2.0

pub mod error;

use error::TracingError;
use serde::{Deserialize, Serialize};
use std::io::IsTerminal;

#[cfg(feature = "tracing")]
pub use opentelemetry;
#[cfg(feature = "tracing")]
pub use opentelemetry_jaeger;
#[cfg(feature = "tracing")]
pub use tracing_opentelemetry;
#[cfg(feature = "tracing")]
pub use tracing_subscriber;
#[cfg(feature = "tracing")]
pub use tracing_tree;
use tracing_subscriber::{filter::Directive, layer::SubscriberExt, util::SubscriberInitExt};

#[derive(Debug, Default, Copy, Clone, Deserialize, PartialEq, Eq, Serialize)]
#[serde(deny_unknown_fields)]
Expand All @@ -22,7 +15,6 @@ pub struct LoggingSettings {
}

// don't call init so that we could attach additional layers
#[cfg(feature = "basic_tracing")]
pub fn build_tracing_logger() -> impl tracing_subscriber::layer::SubscriberExt {
use tracing_subscriber::prelude::*;

Expand All @@ -31,7 +23,6 @@ pub fn build_tracing_logger() -> impl tracing_subscriber::layer::SubscriberExt {
.with(default_tracing_env_filter())
}

#[cfg(feature = "basic_tracing")]
pub fn default_tracing_env_filter() -> tracing_subscriber::filter::EnvFilter {
if ::std::env::var("RUST_LOG").is_ok() {
tracing_subscriber::filter::EnvFilter::from_default_env()
Expand All @@ -43,7 +34,6 @@ pub fn default_tracing_env_filter() -> tracing_subscriber::filter::EnvFilter {
}
}

#[cfg(feature = "basic_tracing")]
pub fn default_tracing_fmt_layer<S, W>(
writer: W,
) -> impl tracing_subscriber::Layer<S> + Sync + Send + 'static
Expand All @@ -63,45 +53,47 @@ where
.with_target(false)
}

#[cfg(feature = "basic_tracing")]
pub fn setup_tracing_logger() {
use tracing_subscriber::util::SubscriberInitExt;
build_tracing_logger().init()
/// Creates a tracing filter that sets more granular log levels for specific crates.
/// This allows for finer control over logging verbosity.
pub(crate) fn granual_filtered_env() -> Result<tracing_subscriber::filter::EnvFilter, TracingError>
{
fn directive_checked(directive: impl Into<String>) -> Result<Directive, TracingError> {
directive.into().parse().map_err(From::from)
}

let mut filter = default_tracing_env_filter();

// these crates are more granularly filtered
let filter_crates = ["defguard_wireguard_rs"];
for crate_name in filter_crates {
filter = filter.add_directive(directive_checked(format!("{crate_name}=warn"))?);
}
Ok(filter)
}

pub fn setup_no_otel_logger() -> Result<(), TracingError> {
// Only set up if not already initialized
if tracing::dispatcher::has_been_set() {
// It shouldn't be - this is really checking that it is torn down between async command executions
return Err(TracingError::TracingLoggerAlreadyInitialised);
}

let registry = tracing_subscriber::registry()
.with(default_tracing_fmt_layer(std::io::stderr))
.with(granual_filtered_env()?);

registry
.try_init()
.map_err(|e| TracingError::TracingTryInitError(e))?;

Ok(())
}

// TODO: This has to be a macro, running it as a function does not work for the file_appender for some reason
#[cfg(feature = "tracing")]
#[macro_export]
macro_rules! setup_tracing {
($service_name: expr) => {
use nym_bin_common::logging::tracing_subscriber::layer::SubscriberExt;
use nym_bin_common::logging::tracing_subscriber::util::SubscriberInitExt;

let registry = nym_bin_common::logging::tracing_subscriber::Registry::default()
.with(nym_bin_common::logging::tracing_subscriber::EnvFilter::from_default_env())
.with(
nym_bin_common::logging::tracing_tree::HierarchicalLayer::new(4)
.with_targets(true)
.with_bracketed_fields(true),
);

let tracer = nym_bin_common::logging::opentelemetry_jaeger::new_collector_pipeline()
.with_endpoint("http://44.199.230.10:14268/api/traces")
.with_service_name($service_name)
.with_isahc()
.with_trace_config(
nym_bin_common::logging::opentelemetry::sdk::trace::config().with_sampler(
nym_bin_common::logging::opentelemetry::sdk::trace::Sampler::TraceIdRatioBased(
0.1,
),
),
)
.install_batch(nym_bin_common::logging::opentelemetry::runtime::Tokio)
.expect("Could not init tracer");

let telemetry = nym_bin_common::logging::tracing_opentelemetry::layer().with_tracer(tracer);

registry.with(telemetry).init();
setup_no_otel_logger()
};
}

Expand Down
Loading
Loading