Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
26 changes: 26 additions & 0 deletions packages/gateway/gateway-server.ts
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@ import { listKnownWorkspaces } from "../../src/core/event-store/workspace.js";
import { prefetchLoginShellPath } from "../../src/utils/login-shell-path.js";
import { normalizeCwd, scheduledCwdsOnDisk } from "./scheduler-guard.js";
import { acquireGatewayLock, readGatewayOwner, type GatewayLockInfo } from "./gateway-lock.js";
import { GatewayWatchdog, type GatewayWatchdogOptions, type WatchdogAlert } from "./watchdog.js";
import { getMainLockPath, isPidLockOwnerAlive, readPidLock, type MainAgentLock } from "../../src/core/agent/main-agent.js";
import { attachJsonlLineReader, serializeJsonLine } from "./jsonl.js";
import {
Expand Down Expand Up @@ -117,6 +118,12 @@ export interface GatewayServerOptions {
* scheduled-workspace discovery — it is NOT excluded from tell destination
* name lookups (the main workspace is a valid tell target). */
mainDir?: string;
/**
* Deterministic gateway watchdog: observes each pooled agent's event stream
* and emits alerts for idle model streams, idle tools, and repeated tool
* calls. Pass `false` to disable. Alerts never abort a turn by themselves.
*/
watchdog?: GatewayWatchdogOptions | false;
/** Pizza version (from package.json), reported in `status` so clients can detect an outdated gateway after an upgrade. */
version?: string;
/**
Expand Down Expand Up @@ -205,6 +212,8 @@ export interface GatewayServerEvents {
tell: (to: string, ok: boolean, durationMs: number) => void;
/** An agent process was spawned. */
agentSpawned: (cwd: string) => void;
/** The gateway watchdog emitted an alert (alert only; no action is taken). */
watchdog: (alert: WatchdogAlert) => void;
/** An agent process was torn down (idle eviction or shutdown). */
agentClosed: (cwd: string) => void;
/** Server started listening. */
Expand Down Expand Up @@ -253,6 +262,19 @@ export function createGatewayServer(options: GatewayServerOptions): GatewayServe
const schedulerGuardInterval = options.schedulerGuardInterval ?? 60_000;
const unresponsiveOwnerTimeout = options.unresponsiveOwnerTimeout ?? 10_000;
const emitter = new EventEmitter();
const watchdog = new GatewayWatchdog({
...(options.watchdog === false ? { enabled: false } : options.watchdog),
onAlert: (alert) => {
console.error(`[watchdog] ${alert.kind} ${alert.workspace}: ${alert.message}`);
emitter.emit("watchdog", alert);
// Also expose the alert on the same event stream channels already
// subscribe to, so a panel/sidecar can render it without another RPC.
broadcastEvent(alert.workspace, {
type: "WATCHDOG_ALERT",
payload: alert,
} as GatewayRpcFrame);
},
});

const pool = new Map<string, PoolEntry>();
let server: Server | undefined;
Expand Down Expand Up @@ -307,6 +329,7 @@ export function createGatewayServer(options: GatewayServerOptions): GatewayServe
// `rpc` path (e.g. a desktop prompt) which never touches the
// `tell` path the timer was originally wired to.
entry.lastActivity = Date.now();
watchdog.observe(cwd, event);
broadcastEvent(cwd, event as unknown as GatewayRpcFrame);
});
eventForwarders.set(cwd, unsubscribe);
Expand Down Expand Up @@ -619,6 +642,7 @@ export function createGatewayServer(options: GatewayServerOptions): GatewayServe
forwarder();
eventForwarders.delete(cwd);
}
watchdog.forget(cwd);
await entry.client.stop().catch(() => {});
emitter.emit("agentClosed", cwd as never);
}
Expand Down Expand Up @@ -1515,6 +1539,7 @@ function nextMessageId(): string {
startHealthCheck();
startSchedulerGuard();
startOwnershipWatch();
watchdog.start();
// After the socket is listening: adapters the supervisor spawns connect
// back to this gateway immediately, so start it only once we can serve.
channelSupervisor.start();
Expand All @@ -1526,6 +1551,7 @@ function nextMessageId(): string {
if (ownershipTimer) { clearInterval(ownershipTimer); ownershipTimer = undefined; }
stopHealthCheck();
stopSchedulerGuard();
watchdog.stop();
await channelSupervisor.shutdown();
// Tear down all pooled agents.
const cwds = Array.from(pool.keys());
Expand Down
6 changes: 6 additions & 0 deletions packages/gateway/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,12 @@ export {

export { ensureGateway, resolveGatewaySocket } from "./gateway-lifecycle.js";
export { readGatewayOwner, type GatewayLockInfo } from "./gateway-lock.js";
export {
GatewayWatchdog,
type GatewayWatchdogOptions,
type WatchdogAlert,
type WatchdogAlertKind,
} from "./watchdog.js";

export {
type GatewayRequest,
Expand Down
Loading
Loading