migration: migrate to Pino (logger) (#1269)

Summary

  Migrates the HyperDX API and related services from Winston to Pino for standardized,
  faster, and more structured logging with improved OpenTelemetry integration.

Changes

  Core Migration

  - Replaced Winston with Pino across all logging infrastructure
  - Upgraded @hyperdx/node-opentelemetry from v0.8.2 to v0.9.0 to support Pino
  transport
  - Removed deprecated dependencies:
    - winston and express-winston
    - @opentelemetry/host-metrics and @opentelemetry/sdk-metrics (consolidated into
  newer OTel SDK)
  - Added new dependencies:
    - pino and pino-http for core logging
    - pino-pretty for development console output

  Logger Configuration (packages/api/src/utils/logger.ts)

  Production:
  - Outputs stringified JSON to stdout via pino/file transport
  - Maintains HyperDX transport integration when API key is configured
  - Includes full OpenTelemetry trace context (trace_id, span_id, trace_flags)
 
<img width="830" height="184" alt="Screenshot 2025-10-14 at 4 31 36 PM" src="https://github.com/user-attachments/assets/82e60919-5c4d-4688-a6f5-d54632aef749" />


  Development:
  - Uses pino-pretty for human-readable, colorized console output
  - Hides verbose fields from console: pid, hostname, trace_id, span_id, trace_flags
  - HTTP request/response objects excluded from logs via custom serializers

<img width="825" height="350" alt="image" src="https://github.com/user-attachments/assets/64b293d8-bc95-4715-931a-dbf73483d247" />

  HTTP Logging:
  - Replaced express-winston with pino-http
  - Custom log levels based on HTTP status codes (warn for 4xx, error for 5xx+)
  - Simplified log messages: HTTP {method} {url}

  Error Logging Updates

  Updated error logging patterns throughout the codebase to follow Pino's structured
  logging conventions:

  // Before (Winston)
  logger.error('Error message:', error);

  // After (Pino)
  logger.error({ err: error }, 'Error message');


Ref: HDX-2588

This PR should also address issue: #1035

Author: wrn14897

.changeset/wet-stingrays-behave.md

@@ -0,0 +1,6 @@
+---
+"@hyperdx/api": minor
+"@hyperdx/app": minor
+---
+
+migration: migrate to Pino for standardized and faster logging

packages/api/package.json

@@ -10,7 +10,7 @@
     "@ai-sdk/anthropic": "^2.0.23",
     "@esm2cjs/p-queue": "^7.3.0",
     "@hyperdx/common-utils": "^0.7.0",
-    "@hyperdx/node-opentelemetry": "^0.8.2",
+    "@hyperdx/node-opentelemetry": "^0.9.0",
     "@hyperdx/passport-local-mongoose": "^9.0.1",
     "@opentelemetry/api": "^1.8.0",
     "@opentelemetry/host-metrics": "^0.35.5",
@@ -27,7 +27,6 @@
     "express": "^4.19.2",
     "express-rate-limit": "^6.7.1",
     "express-session": "^1.17.3",
-    "express-winston": "^4.2.0",
     "handlebars": "^4.7.8",
     "http-graceful-shutdown": "^3.1.13",
     "http-proxy-middleware": "^3.0.5",
@@ -40,12 +39,13 @@
     "on-headers": "^1.1.0",
     "passport": "^0.6.0",
     "passport-local": "^1.0.0",
+    "pino": "^10.0.0",
+    "pino-http": "^11.0.0",
     "promised-handlebars": "^2.0.1",
     "protobufjs": "^7.5.2",
     "semver": "^7.5.2",
     "serialize-error": "^8.1.0",
     "uuid": "^8.3.2",
-    "winston": "^3.10.0",
     "zod": "3.25",
     "zod-express-middleware": "^1.4.0"
   },
@@ -67,6 +67,7 @@
     "jest": "^28.1.3",
     "migrate-mongo": "^11.0.0",
     "nodemon": "^2.0.20",
+    "pino-pretty": "^13.1.1",
     "rimraf": "^4.4.1",
     "supertest": "^6.3.1",
     "swagger-jsdoc": "^6.2.8",

packages/api/src/fixtures.ts

@@ -63,10 +63,7 @@ const healthCheck = async () => {
   const client = await getClickhouseClient();
   const result = await client.ping();
   if (!result.success) {
-    logger.error({
-      message: 'ClickHouse health check failed',
-      error: result.error,
-    });
+    logger.error({ error: result.error }, 'ClickHouse health check failed');
     throw result.error;
   }
 };

packages/api/src/index.ts

@@ -21,7 +21,7 @@ if (config.IS_DEV) {
 const server = new Server();
 
 process.on('uncaughtException', (err: Error) => {
-  logger.error(serializeError(err));
+  logger.error({ err: serializeError(err) }, 'Uncaught exception');
 
   // FIXME: disable server restart until
   // we make sure all expected exceptions are handled properly
@@ -32,7 +32,9 @@ process.on('uncaughtException', (err: Error) => {
 
 process.on('unhandledRejection', (err: any) => {
   // TODO: do we want to throw here ?
-  logger.error(serializeError(err));
+  logger.error({ err: serializeError(err) }, 'Unhandled rejection');
 });
 
-server.start().catch(e => logger.error(serializeError(e)));
+server
+  .start()
+  .catch(e => logger.error({ err: serializeError(e) }, 'Server start failed'));

packages/api/src/middleware/auth.ts

@@ -31,7 +31,8 @@ export function redirectToDashboard(req: Request, res: Response) {
     return res.redirect(`${config.FRONTEND_URL}/search`);
   } else {
     logger.error(
-      `Password login for user failed, user or team not found ${req?.user?._id}`,
+      { userId: req?.user?._id },
+      'Password login for user failed, user or team not found',
     );
     res.redirect(`${config.FRONTEND_URL}/login?err=unknown`);
   }
@@ -43,7 +44,7 @@ export function handleAuthError(
   res: Response,
   next: NextFunction,
 ) {
-  logger.debug({ message: 'Auth error', authErr: serializeError(err) });
+  logger.debug({ authErr: serializeError(err) }, 'Auth error');
   if (res.headersSent) {
     return next(err);
   }

packages/api/src/models/index.ts

@@ -22,12 +22,12 @@ mongoose.connection.on('disconnected', () => {
   logger.info('Lost connection to MongoDB server');
 });
 
-mongoose.connection.on('error', () => {
-  logger.error('Could not connect to MongoDB');
+mongoose.connection.on('error', err => {
+  logger.error({ err }, 'Could not connect to MongoDB');
 });
 
 mongoose.connection.on('reconnected', () => {
-  logger.error('Reconnected to MongoDB');
+  logger.warn('Reconnected to MongoDB');
 });
 
 mongoose.connection.on('reconnectFailed', () => {

packages/api/src/opamp/controllers/opampController.ts

@@ -303,7 +303,7 @@ export class OpampController {
 
       // Decode the AgentToServer message
       const agentToServer = decodeAgentToServer(req.body);
-      logger.debug('agentToServer', agentToServer);
+      logger.debug({ agentToServer }, 'agentToServer');
       logger.debug(
         // @ts-ignore
         `Received message from agent: ${agentToServer.instanceUid?.toString(
@@ -353,7 +353,7 @@ export class OpampController {
       res.setHeader('Content-Type', 'application/x-protobuf');
       res.send(encodedResponse);
     } catch (error) {
-      logger.error('Error handling OpAMP message:', error);
+      logger.error({ err: error }, 'Error handling OpAMP message');
       res.status(500).send('Internal Server Error');
     }
   }

packages/api/src/opamp/services/agentService.ts

@@ -63,7 +63,7 @@ export class AgentService {
 
       return agent;
     } catch (error) {
-      logger.error('Error processing agent status:', error);
+      logger.error({ err: error }, 'Error processing agent status');
       throw error;
     }
   }

packages/api/src/opamp/utils/protobuf.ts

@@ -21,7 +21,7 @@ try {
   root = protobuf.loadSync(PROTO_PATH);
   logger.debug('OpAMP proto definition loaded successfully');
 } catch (error) {
-  logger.error('Failed to load OpAMP proto definition:', error);
+  logger.error({ err: error }, 'Failed to load OpAMP proto definition');
   throw error;
 }
 
@@ -46,7 +46,7 @@ export function decodeAgentToServer(data: Buffer): protobuf.Message {
   try {
     return AgentToServer.decode(data);
   } catch (error) {
-    logger.error('Failed to decode AgentToServer message:', error);
+    logger.error({ err: error }, 'Failed to decode AgentToServer message');
     throw error;
   }
 }
@@ -68,7 +68,7 @@ export function encodeServerToAgent(message: any): Buffer {
     // Encode the message
     return Buffer.from(ServerToAgent.encode(serverToAgent).finish());
   } catch (error) {
-    logger.error('Failed to encode ServerToAgent message:', error);
+    logger.error({ err: error }, 'Failed to encode ServerToAgent message');
     throw error;
   }
 }
@@ -105,7 +105,7 @@ export function createRemoteConfig(
       configHash: configHash,
     };
   } catch (error) {
-    logger.error('Failed to create remote config message:', error);
+    logger.error({ err: error }, 'Failed to create remote config message');
     throw error;
   }
 }
@@ -130,7 +130,7 @@ function calculateConfigHash(configFiles: Map<string, Buffer>): Buffer {
 
     return hash.digest();
   } catch (error) {
-    logger.error('Failed to calculate config hash:', error);
+    logger.error({ err: error }, 'Failed to calculate config hash');
     throw error;
   }
 }

packages/api/src/routers/api/ai.ts

@@ -188,7 +188,7 @@ router.post(
       const source = await getSource(teamId.toString(), sourceId);
 
       if (source == null) {
-        logger.error({ message: 'invalid source id', sourceId, teamId });
+        logger.error({ sourceId, teamId }, 'invalid source id');
         return res.status(400).json({
           error: 'Invalid source',
         });