Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

10 changes: 1 addition & 9 deletions crates/burn-central-fleet/src/inference.rs
Original file line number Diff line number Diff line change
Expand Up @@ -82,14 +82,6 @@ where
}

fn maybe_sync_and_rollout(&self) -> Result<(), FleetManagedInferenceError> {
let fleet_key = self.current_fleet_key();
let reconcile_span = tracing::info_span!(
"fleet.inference.reconcile",
fleet_key = fleet_key.as_str(),
inference_name = self.inference_name.as_str(),
);
let _reconcile_guard = reconcile_span.enter();

if self.active().is_some() && !self.should_sync_now() {
return Ok(());
}
Expand Down Expand Up @@ -219,7 +211,7 @@ where
fn infer(&self, input: Self::Input, writer: InferenceWriter<Self::Output>) {
let fleet_key = self.current_fleet_key();
let request_span = tracing::info_span!(
"fleet.inference.request",
"fleet.inference",
fleet_key = fleet_key.as_str(),
inference_name = self.inference_name.as_str(),
);
Expand Down
6 changes: 1 addition & 5 deletions crates/burn-central-fleet/src/model.rs
Original file line number Diff line number Diff line change
Expand Up @@ -184,7 +184,6 @@ pub fn load_cached_model_source(
}

tracing::info!(
root = %models_root.display(),
version = model_version_id,
"reading model source metadata for model version"
);
Expand All @@ -193,10 +192,7 @@ pub fn load_cached_model_source(
let manifest_path = model_root.join("manifest.json");

if !manifest_path.exists() {
tracing::error!(
path = %manifest_path.display(),
"cached model manifest not found for active model version"
);
tracing::error!("cached model manifest not found for active model version");
return Err(ModelCacheError::MissingCachedFile(
manifest_path.display().to_string(),
));
Expand Down
65 changes: 42 additions & 23 deletions crates/burn-central-fleet/src/telemetry/logs.rs
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
use serde::{Deserialize, Serialize};
use tracing::Dispatch;
use tracing::field::{Field, Visit};
use tracing_subscriber::registry::LookupSpan;

Expand Down Expand Up @@ -60,9 +61,9 @@ impl EventFieldVisitor {
self.message = Some(value.clone());
} else if key == "fleet_key" {
self.fleet_key = Some(value.clone());
} else {
self.fields.push(LogField { key, value });
}

self.fields.push(LogField { key, value });
}
}

Expand Down Expand Up @@ -117,12 +118,49 @@ impl SpanFields {
}

#[derive(Debug, Default)]
pub struct TelemetryLogLayer;
pub struct TelemetryLogLayer {
with_current_fleet_key: Option<fn(&Dispatch, &tracing::span::Id) -> Option<String>>,
}

impl TelemetryLogLayer {
pub(crate) fn current_fleet_key(&self, dispatch: &Dispatch) -> Option<String> {
let current = dispatch.current_span();
let id = current.id()?;
(self.with_current_fleet_key?)(dispatch, id)
}
}

/// Retrieves the fleet key from the current span context, if any. Returns None if there is no current span or if no fleet key is associated with the current span.
pub(crate) fn current_fleet_key() -> Option<String> {
tracing::dispatcher::get_default(|dispatch| {
dispatch
.downcast_ref::<TelemetryLogLayer>()?
.current_fleet_key(dispatch)
})
}

impl<S> tracing_subscriber::Layer<S> for TelemetryLogLayer
where
S: tracing::Subscriber + for<'a> LookupSpan<'a>,
{
fn on_layer(&mut self, _: &mut S) {
self.with_current_fleet_key = Some(|dispatch, id| {
let subscriber = dispatch.downcast_ref::<S>()?;
let span = subscriber.span(id)?;
let mut fleet_key = None;

for scope_span in span.scope().from_root() {
if let Some(span_fields) = scope_span.extensions().get::<SpanFields>() {
if let Some(candidate) = span_fields.fleet_key.as_ref() {
fleet_key = Some(candidate.clone());
}
}
}

fleet_key
});
}

fn on_new_span(
&self,
attrs: &tracing::span::Attributes<'_>,
Expand Down Expand Up @@ -222,7 +260,7 @@ mod tests {
.expect("test serial lock should not be poisoned");
super::super::clear_dispatched_log_records_for_test();

let subscriber = tracing_subscriber::registry().with(TelemetryLogLayer);
let subscriber = tracing_subscriber::registry().with(TelemetryLogLayer::default());
tracing::subscriber::with_default(subscriber, test_fn);

super::super::take_dispatched_log_records_for_test()
Expand Down Expand Up @@ -285,23 +323,4 @@ mod tests {
"events without fleet key should not be dispatched",
);
}

#[test]
fn log_layer_prefers_event_fleet_key_over_span_fleet_key() {
let records = run_with_log_layer(|| {
let span = tracing::info_span!("request", fleet_key = "span-fleet");
let _guard = span.enter();

tracing::info!(fleet_key = "event-fleet", "override fleet key");
});

assert_eq!(records.len(), 1);
let record = &records[0];
assert_eq!(record.fleet_key, "event-fleet");
assert_eq!(field_value(record, "fleet_key"), Some("event-fleet"));
assert_eq!(
field_value(record, "span.request.fleet_key"),
Some("span-fleet")
);
}
}
Loading
Loading