Skip to content
Draft
Show file tree
Hide file tree
Changes from 2 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
93 changes: 75 additions & 18 deletions src/app/src/renderer/LogsWindow.tsx
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
import React, { useEffect, useRef, useState } from 'react';
import React, { useCallback, useEffect, useRef, useState } from 'react';
Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Image

What's up with all the WebSocketServer::handleConnection() HTTP status: 400 error: Missing Sec-WebSocket-Key value? Never seen that before.

import { getAPIKey, getServerBaseUrl, onServerUrlChange, serverConfig } from './utils/serverConfig';
import {EventSource} from 'eventsource';
Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Doesn't seem to work for Kokoro, SD, Whisper, or RyzenAI-Server?

Image Image

The KokoroServer logs are ending up under Lemonade Router not Kokoro Server.


Expand All @@ -7,6 +7,11 @@ interface LogsWindowProps {
height?: number;
}
Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Image

Two instances of llama-server (under --max-loaded-models 2) share the same log view. Should it be clearly separated and labeled into llama-server[0] and [1]? Or better yet, filter by model name instead of process name?


interface LogSource {
name: string;
label: string;
}

const BOTTOM_FOLLOW_THRESHOLD_PX = 60;

const LogsWindow: React.FC<LogsWindowProps> = ({ isVisible, height }) => {
Expand All @@ -23,6 +28,10 @@ const LogsWindow: React.FC<LogsWindowProps> = ({ isVisible, height }) => {
const [apiKey, setAPIKey] = useState<string>('');
const [isInitialized, setIsInitialized] = useState(false);

const [selectedSource, setSelectedSource] = useState<string>('all');
const [availableSources, setAvailableSources] = useState<LogSource[]>([]);
const sourcePollRef = useRef<NodeJS.Timeout | null>(null);

const isNearBottom = () => {
const logsContent = logsContentRef.current;
if (!logsContent) return true;
Expand All @@ -38,13 +47,11 @@ const LogsWindow: React.FC<LogsWindowProps> = ({ isVisible, height }) => {
isProgrammaticScrollRef.current = true;
logsEndRef.current.scrollIntoView({ behavior: 'auto', block: 'end' });

// Keep programmatic-scroll guard through the next paint.
requestAnimationFrame(() => {
isProgrammaticScrollRef.current = false;
});
};

// Wait for serverConfig to initialize and get the correct URL
useEffect(() => {
serverConfig.waitForInit().then(() => {
setServerUrl(getServerBaseUrl());
Expand All @@ -53,7 +60,6 @@ const LogsWindow: React.FC<LogsWindowProps> = ({ isVisible, height }) => {
});
}, []);

// Listen for URL changes (covers both port changes and explicit URL updates)
useEffect(() => {
const unsubscribe = onServerUrlChange((newUrl: string, newAPIKey: string) => {
console.log('Server URL changed, updating logs URL:', newUrl);
Expand All @@ -66,6 +72,45 @@ const LogsWindow: React.FC<LogsWindowProps> = ({ isVisible, height }) => {
};
}, []);

// Poll for available log sources
const fetchSources = useCallback(async () => {
if (!serverUrl || !isInitialized) return;

try {
const headers: Record<string, string> = {};
if (apiKey) {
headers['Authorization'] = `Bearer ${apiKey}`;
}
const resp = await fetch(`${serverUrl}/api/v1/logs/sources`, { headers });
if (resp.ok) {
const data = await resp.json();
setAvailableSources(data.sources || []);
}
} catch {
// Silently ignore — the sources list is a convenience, not critical
}
}, [serverUrl, apiKey, isInitialized]);

useEffect(() => {
if (!isVisible || !isInitialized || !serverUrl) {
if (sourcePollRef.current) {
clearInterval(sourcePollRef.current);
sourcePollRef.current = null;
}
return;
}

fetchSources();
sourcePollRef.current = setInterval(fetchSources, 10000);

return () => {
if (sourcePollRef.current) {
clearInterval(sourcePollRef.current);
sourcePollRef.current = null;
}
};
}, [isVisible, isInitialized, serverUrl, apiKey, fetchSources]);

// Auto-scroll to bottom when new logs arrive (if auto-scroll is enabled)
useEffect(() => {
if (autoScroll) {
Expand Down Expand Up @@ -95,11 +140,9 @@ const LogsWindow: React.FC<LogsWindowProps> = ({ isVisible, height }) => {
return () => logsContent.removeEventListener('scroll', handleScroll);
}, []);

// Connect to SSE log stream
// Connect to SSE log stream — reconnects when selectedSource changes
useEffect(() => {
// Don't connect until we have the correct URL from initialization
if (!isVisible || !isInitialized || !serverUrl) {
// Clean up connection when logs window is hidden or not ready
if (eventSourceRef.current) {
eventSourceRef.current.close();
eventSourceRef.current = null;
Expand All @@ -115,7 +158,6 @@ const LogsWindow: React.FC<LogsWindowProps> = ({ isVisible, height }) => {
try {
setConnectionStatus('connecting');

// Close existing connection if any
if (eventSourceRef.current) {
eventSourceRef.current.close();
}
Expand All @@ -130,31 +172,31 @@ const LogsWindow: React.FC<LogsWindowProps> = ({ isVisible, height }) => {
},
})} : {};

const eventSource = new EventSource(`${serverUrl}/api/v1/logs/stream`, options)
const streamUrl = selectedSource && selectedSource !== 'all'
? `${serverUrl}/api/v1/logs/stream?source=${encodeURIComponent(selectedSource)}`
: `${serverUrl}/api/v1/logs/stream`;

const eventSource = new EventSource(streamUrl, options);
eventSourceRef.current = eventSource;

eventSource.onopen = () => {
console.log('Log stream connected to:', serverUrl);
console.log('Log stream connected to:', streamUrl);
setConnectionStatus('connected');
};

eventSource.onmessage = (event) => {
// SSE sends data as "data: <log line>"
const logLine = event.data;

// Skip heartbeat messages
if (logLine.trim() === '' || logLine === 'heartbeat') {
return;
}

// Keep follow mode sticky when user is effectively at bottom.
const shouldFollowNextLine = autoScrollRef.current || isNearBottom();
if (shouldFollowNextLine && !autoScrollRef.current) {
setAutoScroll(true);
}

setLogs((prevLogs) => {
// Keep last 1000 lines to prevent memory issues
const newLogs = [...prevLogs, logLine];
return newLogs.length > 1000 ? newLogs.slice(-1000) : newLogs;
});
Expand All @@ -165,7 +207,6 @@ const LogsWindow: React.FC<LogsWindowProps> = ({ isVisible, height }) => {
setConnectionStatus('error');
eventSource.close();

// Reconnect after 5 seconds
reconnectTimeoutRef.current = setTimeout(() => {
console.log('Attempting to reconnect to log stream...');
connectToLogStream();
Expand All @@ -181,10 +222,8 @@ const LogsWindow: React.FC<LogsWindowProps> = ({ isVisible, height }) => {
}
};

// Initial connection
connectToLogStream();

// Cleanup on unmount or when visibility changes
return () => {
if (eventSourceRef.current) {
eventSourceRef.current.close();
Expand All @@ -195,7 +234,7 @@ const LogsWindow: React.FC<LogsWindowProps> = ({ isVisible, height }) => {
reconnectTimeoutRef.current = null;
}
};
}, [isVisible, serverUrl, apiKey, isInitialized]);
}, [isVisible, serverUrl, apiKey, isInitialized, selectedSource]);

const handleClearLogs = () => {
setLogs([]);
Expand All @@ -206,13 +245,31 @@ const LogsWindow: React.FC<LogsWindowProps> = ({ isVisible, height }) => {
scrollToBottom();
};

const handleSourceChange = (e: React.ChangeEvent<HTMLSelectElement>) => {
setLogs([]);
setSelectedSource(e.target.value);
};

if (!isVisible) return null;

return (
<div className="logs-window" style={height ? { height: `${height}px`, flex: 'none' } : undefined}>
<div className="logs-header">
<h3>Server Logs</h3>
<div className="logs-controls">
<select
className="logs-source-select"
value={selectedSource}
onChange={handleSourceChange}
title="Filter logs by source"
>
<option value="all">All Sources</option>
{availableSources.map((src) => (
<option key={src.name} value={src.name}>
{src.label}
</option>
))}
</select>
<span className={`connection-status status-${connectionStatus}`}>
{connectionStatus === 'connecting' && '⟳ Connecting...'}
{connectionStatus === 'connected' && '● Connected'}
Expand Down
21 changes: 21 additions & 0 deletions src/app/styles.css
Original file line number Diff line number Diff line change
Expand Up @@ -4049,6 +4049,27 @@ footer {
background: rgba(136, 136, 136, 0.1);
}

.logs-source-select {
font-size: 0.7rem;
padding: 4px 8px;
background: #1a1a1a;
color: #fff;
border: 1px solid #333;
border-radius: 4px;
cursor: pointer;
transition: all 0.2s ease;
max-width: 160px;
}

.logs-source-select:hover {
border-color: #555;
}

.logs-source-select:focus {
outline: none;
border-color: #666;
}

.logs-control-btn {
font-size: 0.7rem;
padding: 4px 10px;
Expand Down
3 changes: 3 additions & 0 deletions src/cpp/include/lemon/router.h
Original file line number Diff line number Diff line change
Expand Up @@ -78,6 +78,9 @@ class Router {
void completion_stream(const std::string& request_body, httplib::DataSink& sink);
void responses_stream(const std::string& request_body, httplib::DataSink& sink);

// Get distinct server names of currently loaded backends
std::vector<std::string> get_loaded_server_names() const;

// Get telemetry data
json get_stats() const;

Expand Down
1 change: 1 addition & 0 deletions src/cpp/include/lemon/server.h
Original file line number Diff line number Diff line change
Expand Up @@ -73,6 +73,7 @@ class Server {
void handle_log_level(const httplib::Request& req, httplib::Response& res);
void handle_shutdown(const httplib::Request& req, httplib::Response& res);
void handle_logs_stream(const httplib::Request& req, httplib::Response& res);
void handle_log_sources(const httplib::Request& req, httplib::Response& res);
#ifdef HAVE_SYSTEMD
void handle_logs_stream_journald(const httplib::Request& req, httplib::Response& res);
#endif
Expand Down
5 changes: 4 additions & 1 deletion src/cpp/include/lemon/utils/process_manager.h
Original file line number Diff line number Diff line change
Expand Up @@ -20,13 +20,16 @@ using OutputLineCallback = std::function<bool(const std::string& line)>;
class ProcessManager {
public:
// Start a process with arguments
// source_name: when non-empty, backend output is tagged with this name in logs
// and pipe-based capture is forced (even when filter_health_logs is false)
static ProcessHandle start_process(
const std::string& executable,
const std::vector<std::string>& args,
const std::string& working_dir = "",
bool inherit_output = false,
bool filter_health_logs = false,
const std::vector<std::pair<std::string, std::string>>& env_vars = {});
const std::vector<std::pair<std::string, std::string>>& env_vars = {},
const std::string& source_name = "");

// Run a process and capture its output line by line
// Blocks until process exits or callback returns false (which kills the process)
Expand Down
3 changes: 2 additions & 1 deletion src/cpp/include/lemon/wrapped_server.h
Original file line number Diff line number Diff line change
Expand Up @@ -60,7 +60,8 @@ class WrappedServer : public ICompletionServer {
virtual ~WrappedServer() = default;


// Set log level
const std::string& get_server_name() const { return server_name_; }

void set_log_level(const std::string& log_level) { log_level_ = log_level; }

// Check if debug logging is enabled
Expand Down
2 changes: 1 addition & 1 deletion src/cpp/server/backends/fastflowlm_server.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -262,7 +262,7 @@ void FastFlowLMServer::load(const std::string& model_name,
}
LOG(INFO, "ProcessManager") << std::endl;

process_handle_ = utils::ProcessManager::start_process(flm_path, args, "", is_debug(), true);
process_handle_ = utils::ProcessManager::start_process(flm_path, args, "", is_debug(), true, {}, server_name_);
LOG(INFO, "ProcessManager") << "Process started successfully" << std::endl;

// Wait for flm-server to be ready
Expand Down
3 changes: 2 additions & 1 deletion src/cpp/server/backends/kokoro_server.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -112,7 +112,8 @@ void KokoroServer::load(const std::string& model_name, const ModelInfo& model_in
"", // working_dir (empty = current)
is_debug(), // inherit_output
false,
env_vars
env_vars,
server_name_
);

if (process_handle_.pid == 0) {
Expand Down
2 changes: 1 addition & 1 deletion src/cpp/server/backends/llamacpp_server.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -369,7 +369,7 @@ void LlamaCppServer::load(const std::string& model_name,
// Start process (inherit output if debug logging enabled, filter health check spam)
// Keep llama-server output visible at info log level.
bool inherit_llama_output = (log_level_ == "info") || is_debug();
process_handle_ = ProcessManager::start_process(executable, args, "", inherit_llama_output, true, env_vars);
process_handle_ = ProcessManager::start_process(executable, args, "", inherit_llama_output, true, env_vars, server_name_);

// Wait for server to be ready
if (!wait_for_ready("/health")) {
Expand Down
4 changes: 3 additions & 1 deletion src/cpp/server/backends/ryzenaiserver.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -102,7 +102,9 @@ void RyzenAIServer::load(const std::string& model_name,
args,
"",
is_debug(),
true
true,
{},
server_name_
);

if (!utils::ProcessManager::is_running(process_handle_)) {
Expand Down
3 changes: 2 additions & 1 deletion src/cpp/server/backends/sd_server.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -173,7 +173,8 @@ void SDServer::load(const std::string& model_name,
"", // working_dir (empty = current)
is_debug(), // inherit_output
false, // filter_health_logs
env_vars
env_vars,
server_name_
);

if (process_handle_.pid == 0) {
Expand Down
3 changes: 2 additions & 1 deletion src/cpp/server/backends/whisper_server.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -237,7 +237,8 @@ void WhisperServer::load(const std::string& model_name,
"", // working_dir (empty = current)
is_debug(), // inherit_output
false, // filter_health_logs
env_vars
env_vars,
server_name_
);

if (process_handle_.pid == 0) {
Expand Down
12 changes: 12 additions & 0 deletions src/cpp/server/router.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -701,6 +701,18 @@ json Router::image_variations(const json& request) {
});
}

std::vector<std::string> Router::get_loaded_server_names() const {
std::lock_guard<std::mutex> lock(load_mutex_);
std::vector<std::string> names;
for (const auto& server : loaded_servers_) {
const auto& name = server->get_server_name();
if (std::find(names.begin(), names.end(), name) == names.end()) {
names.push_back(name);
}
}
return names;
}

json Router::get_stats() const {
std::lock_guard<std::mutex> lock(load_mutex_);
WrappedServer* server = get_most_recent_server();
Expand Down
Loading