Add migration to include Provider fields in RequestLogs and create PostgreSQL dashboard for Grafana

- Added ProviderId and ProviderType columns to RequestLogs table in the migration file 20251201072144_AddProviderFieldsToRequestLogs.cs.
- Created a comprehensive Grafana dashboard configuration for monitoring PostgreSQL metrics, including status, connections, transactions, performance, and block I/O.

Author: nickna

Services/ConduitLLM.Gateway/Middleware/UsageTrackingMiddleware.cs

@@ -280,6 +280,16 @@ await ProcessVideoResponseAsync(context, responseBody, costCalculationService, b
 
                 UsageMetrics.UsageTrackingCosts.WithLabels(model, providerType, endpointType).Inc(Convert.ToDouble(totalCost));
 
+                // Record business metrics for Grafana dashboards (real-time counters)
+                var requestStatus = context.Response.StatusCode >= 200 && context.Response.StatusCode < 300 ? "success" : "error";
+                BusinessMetricsService.RecordModelRequest(model, providerType, requestStatus);
+                BusinessMetricsService.RecordTokens(model, providerType, usage.PromptTokens ?? 0, usage.CompletionTokens ?? 0);
+                BusinessMetricsService.RecordResponseTime(model, providerType, UsageExtractor.GetResponseTime(context) / 1000.0);
+                if (totalCost > 0)
+                {
+                    BusinessMetricsService.RecordCost(providerType, model, endpointType, Convert.ToDouble(totalCost));
+                }
+
                 // Update spend using batch service only if there's a cost
                 if (totalCost > 0)
                 {
@@ -441,6 +451,16 @@ private async Task TrackStreamingUsageAsync(
 
             UsageMetrics.UsageTrackingCosts.WithLabels(model, providerType, endpointType + "_stream").Inc(Convert.ToDouble(cost));
 
+            // Record business metrics for Grafana dashboards (real-time counters)
+            var requestStatus = context.Response.StatusCode >= 200 && context.Response.StatusCode < 300 ? "success" : "error";
+            BusinessMetricsService.RecordModelRequest(model, providerType, requestStatus);
+            BusinessMetricsService.RecordTokens(model, providerType, usage.PromptTokens ?? 0, usage.CompletionTokens ?? 0);
+            BusinessMetricsService.RecordResponseTime(model, providerType, UsageExtractor.GetResponseTime(context) / 1000.0);
+            if (cost > 0)
+            {
+                BusinessMetricsService.RecordCost(providerType, model, endpointType, Convert.ToDouble(cost));
+            }
+
             // Update spend only if there's a cost
             if (cost > 0)
             {
@@ -473,10 +493,20 @@ private async Task LogRequestAsync(
             {
                 var requestType = UsageExtractor.DetermineRequestType(context.Request.Path);
 
+                // Extract provider info from HttpContext.Items (set by controllers)
+                int? providerId = context.Items.TryGetValue("ProviderId", out var providerIdObj) && providerIdObj is int pid
+                    ? pid
+                    : null;
+                var providerType = context.Items.TryGetValue("ProviderType", out var providerTypeObj)
+                    ? providerTypeObj?.ToString()
+                    : null;
+
                 var logRequest = new LogRequestDto
                 {
                     VirtualKeyId = virtualKeyId,
                     ModelName = model,
+                    ProviderId = providerId,
+                    ProviderType = providerType,
                     RequestType = requestType,
                     InputTokens = usage.PromptTokens ?? 0,
                     OutputTokens = usage.CompletionTokens ?? 0,
@@ -580,6 +610,15 @@ private async Task ProcessFunctionResponseAsync(
                 UsageMetrics.UsageTrackingRequests.WithLabels("function", "success").Inc();
                 UsageMetrics.UsageTrackingCosts.WithLabels(functionName, providerType, "function").Inc(Convert.ToDouble(cost));
 
+                // Record business metrics for Grafana dashboards (real-time counters)
+                var requestStatus = context.Response.StatusCode >= 200 && context.Response.StatusCode < 300 ? "success" : "error";
+                BusinessMetricsService.RecordModelRequest(functionName, providerType, requestStatus);
+                BusinessMetricsService.RecordResponseTime(functionName, providerType, UsageExtractor.GetResponseTime(context) / 1000.0);
+                if (cost > 0)
+                {
+                    BusinessMetricsService.RecordCost(providerType, functionName, "function", Convert.ToDouble(cost));
+                }
+
                 // Update spend if there's a cost
                 if (cost > 0)
                 {
@@ -726,6 +765,15 @@ private async Task ProcessImageResponseAsync(
                 UsageMetrics.UsageTrackingRequests.WithLabels("image", "success").Inc();
                 UsageMetrics.UsageTrackingCosts.WithLabels(model, providerType, "image").Inc(Convert.ToDouble(cost));
 
+                // Record business metrics for Grafana dashboards (real-time counters)
+                var requestStatus = context.Response.StatusCode >= 200 && context.Response.StatusCode < 300 ? "success" : "error";
+                BusinessMetricsService.RecordModelRequest(model, providerType, requestStatus);
+                BusinessMetricsService.RecordResponseTime(model, providerType, UsageExtractor.GetResponseTime(context) / 1000.0);
+                if (cost > 0)
+                {
+                    BusinessMetricsService.RecordCost(providerType, model, "image", Convert.ToDouble(cost));
+                }
+
                 // Update spend if there's a cost
                 if (cost > 0)
                 {
@@ -912,6 +960,15 @@ private async Task ProcessVideoResponseAsync(
                 UsageMetrics.UsageTrackingRequests.WithLabels("video", "success").Inc();
                 UsageMetrics.UsageTrackingCosts.WithLabels(model, providerType, "video").Inc(Convert.ToDouble(cost));
 
+                // Record business metrics for Grafana dashboards (real-time counters)
+                var requestStatus = context.Response.StatusCode >= 200 && context.Response.StatusCode < 300 ? "success" : "error";
+                BusinessMetricsService.RecordModelRequest(model, providerType, requestStatus);
+                BusinessMetricsService.RecordResponseTime(model, providerType, UsageExtractor.GetResponseTime(context) / 1000.0);
+                if (cost > 0)
+                {
+                    BusinessMetricsService.RecordCost(providerType, model, "video", Convert.ToDouble(cost));
+                }
+
                 // Update spend if there's a cost
                 if (cost > 0)
                 {

Services/ConduitLLM.Gateway/Program.Monitoring.cs

@@ -103,5 +103,16 @@ public static void ConfigureMonitoringServices(WebApplicationBuilder builder)
 
         // Add cache statistics registration service
         builder.Services.AddHostedService<ConduitLLM.Gateway.Services.CacheStatisticsRegistrationService>();
+
+        // Add business metrics service for Prometheus/Grafana dashboards
+        // Uses leader election to avoid duplicate metrics collection in scaled-out deployments
+        builder.Services.AddLeaderElectedHostedService<ConduitLLM.Gateway.Services.BusinessMetricsService>(
+            serviceProvider =>
+            {
+                var scopeFactory = serviceProvider.GetRequiredService<IServiceScopeFactory>();
+                var logger = serviceProvider.GetRequiredService<ILogger<ConduitLLM.Gateway.Services.BusinessMetricsService>>();
+                return new ConduitLLM.Gateway.Services.BusinessMetricsService(scopeFactory, logger);
+            },
+            "BusinessMetricsService");
     }
 }

Services/ConduitLLM.Gateway/Services/BusinessMetricsService.cs

@@ -132,9 +132,6 @@ public class BusinessMetricsService : BackgroundService
                     LabelNames = new[] { "sla_type", "model" } // sla_type: latency, availability, error_rate
                 });
 
-        private readonly Dictionary<string, DateTime> _lastCostUpdate = new();
-        private readonly Dictionary<string, decimal> _lastCostValue = new();
-
         public BusinessMetricsService(
             IServiceScopeFactory serviceScopeFactory,
             ILogger<BusinessMetricsService> logger)
@@ -201,21 +198,28 @@ private async Task CollectVirtualKeyMetrics(IServiceScope scope)
 
         private async Task CollectModelUsageMetrics(IServiceScope scope)
         {
+            // NOTE: Model/provider counters (conduit_model_requests_total, conduit_model_tokens_total)
+            // are updated in REAL-TIME via static methods called from UsageTrackingMiddleware.
+            // This background method only collects supplementary gauge metrics.
+            //
+            // DO NOT increment counters here - it would cause double-counting since the middleware
+            // already records each request as it happens.
+
             try
             {
                 var dbContextFactory = scope.ServiceProvider.GetRequiredService<IDbContextFactory<ConduitLLM.Configuration.ConduitDbContext>>();
                 await using var context = await dbContextFactory.CreateDbContextAsync();
 
-                // Get model usage statistics for the last hour
-                var oneHourAgo = DateTime.UtcNow.AddHours(-1);
+                // Get model usage statistics for the last 5 minutes to calculate current rates
+                var fiveMinutesAgo = DateTime.UtcNow.AddMinutes(-5);
 
-                // First get the data, then process in memory to avoid expression tree issues
                 var requestLogs = await context.RequestLogs
-                    .Where(r => r.Timestamp >= oneHourAgo)
+                    .Where(r => r.Timestamp >= fiveMinutesAgo)
                     .ToListAsync();
 
+                // Use the new ProviderType field directly instead of parsing model names
                 var modelStats = requestLogs
-                    .GroupBy(r => new { Model = r.ModelName, Provider = r.ModelName.Contains("/") ? r.ModelName.Split('/')[0] : "unknown" })
+                    .GroupBy(r => new { Model = r.ModelName, Provider = r.ProviderType ?? "unknown" })
                     .Select(g => new
                     {
                         g.Key.Model,
@@ -227,23 +231,15 @@ private async Task CollectModelUsageMetrics(IServiceScope scope)
                     })
                     .ToList();
 
+                _logger.LogDebug("Collected model usage metrics: {Count} model/provider combinations in last 5 minutes",
+                    modelStats.Count);
+
+                // Observe average response times (histograms are safe to update periodically)
                 foreach (var stat in modelStats)
                 {
-                    if (stat.TotalPromptTokens > 0)
-                    {
-                        ModelTokensProcessed.WithLabels(stat.Model ?? "unknown", stat.Provider ?? "unknown", "prompt")
-                            .Inc(stat.TotalPromptTokens);
-                    }
-
-                    if (stat.TotalCompletionTokens > 0)
-                    {
-                        ModelTokensProcessed.WithLabels(stat.Model ?? "unknown", stat.Provider ?? "unknown", "completion")
-                            .Inc(stat.TotalCompletionTokens);
-                    }
-
                     if (stat.AvgResponseTime > 0)
                     {
-                        ModelResponseTime.WithLabels(stat.Model ?? "unknown", stat.Provider ?? "unknown")
+                        ModelResponseTime.WithLabels(stat.Model ?? "unknown", stat.Provider)
                             .Observe(stat.AvgResponseTime / 1000.0); // Convert ms to seconds
                     }
                 }
@@ -256,21 +252,25 @@ private async Task CollectModelUsageMetrics(IServiceScope scope)
 
         private async Task CollectCostMetrics(IServiceScope scope)
         {
+            // NOTE: Cost counters (conduit_cost_total_dollars) are updated in REAL-TIME via
+            // static methods called from UsageTrackingMiddleware.
+            // This background method only updates the CostRate gauge for rate calculations.
+
             try
             {
                 var dbContextFactory = scope.ServiceProvider.GetRequiredService<IDbContextFactory<ConduitLLM.Configuration.ConduitDbContext>>();
                 await using var context = await dbContextFactory.CreateDbContextAsync();
 
-                // Calculate cost rate per provider
+                // Calculate cost rate per provider using the ProviderType field
                 var fiveMinutesAgo = DateTime.UtcNow.AddMinutes(-5);
 
-                // First get the data, then process in memory to avoid expression tree issues
                 var costLogs = await context.RequestLogs
                     .Where(r => r.Timestamp >= fiveMinutesAgo && r.Cost > 0)
                     .ToListAsync();
 
+                // Use the new ProviderType field directly
                 var costByProvider = costLogs
-                    .GroupBy(r => r.ModelName.Contains("/") ? r.ModelName.Split('/')[0] : "unknown")
+                    .GroupBy(r => r.ProviderType ?? "unknown")
                     .Select(g => new
                     {
                         Provider = g.Key,
@@ -280,28 +280,15 @@ private async Task CollectCostMetrics(IServiceScope scope)
 
                 foreach (var providerCost in costByProvider)
                 {
-                    var provider = providerCost.Provider ?? "unknown";
+                    var provider = providerCost.Provider;
                     var costPerMinute = (double)(providerCost.TotalCost / 5); // 5-minute window
 
+                    // Update the rate gauge (this is safe to update periodically)
                     CostRate.WithLabels(provider).Set(costPerMinute);
-
-                    // Track cost changes
-                    if (_lastCostUpdate.TryGetValue(provider, out var lastUpdate))
-                    {
-                        var timeDiff = (DateTime.UtcNow - lastUpdate).TotalMinutes;
-                        if (timeDiff > 0 && _lastCostValue.TryGetValue(provider, out var lastCost))
-                        {
-                            var costDiff = providerCost.TotalCost - lastCost;
-                            if (costDiff > 0)
-                            {
-                                CostTotal.WithLabels(provider, "all", "inference").Inc((double)costDiff);
-                            }
-                        }
-                    }
-
-                    _lastCostUpdate[provider] = DateTime.UtcNow;
-                    _lastCostValue[provider] = providerCost.TotalCost;
                 }
+
+                _logger.LogDebug("Collected cost metrics: {Count} providers with costs in last 5 minutes",
+                    costByProvider.Count);
             }
             catch (Exception ex)
             {
@@ -368,6 +355,26 @@ public static void RecordCost(string provider, string model, string operationTyp
             CostPerRequest.WithLabels(model, provider).Observe(costDollars);
         }
 
+        public static void RecordTokens(string model, string provider, int promptTokens, int completionTokens)
+        {
+            if (promptTokens > 0)
+            {
+                ModelTokensProcessed.WithLabels(model, provider, "prompt").Inc(promptTokens);
+            }
+            if (completionTokens > 0)
+            {
+                ModelTokensProcessed.WithLabels(model, provider, "completion").Inc(completionTokens);
+            }
+        }
+
+        public static void RecordResponseTime(string model, string provider, double responseTimeSeconds)
+        {
+            if (responseTimeSeconds > 0)
+            {
+                ModelResponseTime.WithLabels(model, provider).Observe(responseTimeSeconds);
+            }
+        }
+
         public static void RecordSLAViolation(string slaType, string model)
         {
             SLAViolations.WithLabels(slaType, model).Inc();

Shared/ConduitLLM.Configuration/DTOs/LogRequestDto.cs

@@ -20,6 +20,16 @@ public class LogRequestDto
         /// </summary>
         public string ModelName { get; set; } = string.Empty;
 
+        /// <summary>
+        /// ID of the provider that processed the request.
+        /// </summary>
+        public int? ProviderId { get; set; }
+
+        /// <summary>
+        /// Type of the provider that processed the request (e.g., "OpenAI", "Anthropic").
+        /// </summary>
+        public string? ProviderType { get; set; }
+
         /// <summary>
         /// Type of the request (chat, completion, embedding, etc.)
         /// </summary>

Shared/ConduitLLM.Configuration/Entities/RequestLog.cs

@@ -32,6 +32,19 @@ public class RequestLog
     [MaxLength(100)]
     public string ModelName { get; set; } = string.Empty;
 
+    /// <summary>
+    /// ID of the provider that processed the request.
+    /// References the Provider entity for accurate provider tracking.
+    /// </summary>
+    public int? ProviderId { get; set; }
+
+    /// <summary>
+    /// Type of the provider that processed the request.
+    /// Stored as string for flexibility and query performance.
+    /// </summary>
+    [MaxLength(50)]
+    public string? ProviderType { get; set; }
+
     /// <summary>
     /// Type of the request (chat, completion, embedding, etc.)
     /// </summary>