feat: add state field to AlertHistory collection (#157)

Author: wrn14897

.changeset/great-bags-smoke.md

@@ -0,0 +1,5 @@
+---
+'@hyperdx/api': minor
+---
+
+feat: add state field to AlertHistory collection

packages/api/src/models/alertHistory.ts

@@ -1,12 +1,15 @@
 import mongoose, { Schema } from 'mongoose';
 import ms from 'ms';
 
+import { AlertState } from '@/models/alert';
+
 import type { ObjectId } from '.';
 
 export interface IAlertHistory {
   alert: ObjectId;
   counts: number;
   createdAt: Date;
+  state: AlertState;
 }
 
 const AlertHistorySchema = new Schema<IAlertHistory>({
@@ -19,6 +22,11 @@ const AlertHistorySchema = new Schema<IAlertHistory>({
     required: true,
   },
   alert: { type: mongoose.Schema.Types.ObjectId, ref: 'Alert' },
+  state: {
+    type: String,
+    enum: Object.values(AlertState),
+    required: true,
+  },
 });
 
 AlertHistorySchema.index(

packages/api/src/tasks/__tests__/checkAlerts.test.ts

@@ -184,15 +184,6 @@ describe('checkAlerts', () => {
 
       // shoud fetch 5m of logs
       await processAlert(now, alert);
-      // check alert history
-      const alertHistories = await AlertHistory.find({
-        alertId: alert._id,
-      });
-      expect(alertHistories.length).toBe(1);
-      expect(alertHistories[0].counts).toBe(1);
-      expect(alertHistories[0].createdAt).toEqual(
-        new Date('2023-11-16T22:10:00.000Z'),
-      );
       expect(alert.state).toBe('ALERT');
 
       // skip since time diff is less than 1 window size
@@ -206,6 +197,24 @@ describe('checkAlerts', () => {
       // alert should be in ok state
       expect(alert.state).toBe('OK');
 
+      // check alert history
+      const alertHistories = await AlertHistory.find({
+        alertId: alert._id,
+      }).sort({
+        createdAt: 1,
+      });
+      expect(alertHistories.length).toBe(2);
+      expect(alertHistories[0].state).toBe('ALERT');
+      expect(alertHistories[0].counts).toBe(1);
+      expect(alertHistories[0].createdAt).toEqual(
+        new Date('2023-11-16T22:10:00.000Z'),
+      );
+      expect(alertHistories[1].state).toBe('OK');
+      expect(alertHistories[1].counts).toBe(0);
+      expect(alertHistories[1].createdAt).toEqual(
+        new Date('2023-11-16T22:15:00.000Z'),
+      );
+
       // check if checkAlert query + webhook were triggered
       expect(clickhouse.checkAlert).toHaveBeenNthCalledWith(1, {
         endTime: new Date('2023-11-16T22:10:00.000Z'),
@@ -331,15 +340,6 @@ describe('checkAlerts', () => {
 
       // shoud fetch 5m of logs
       await processAlert(now, alert);
-      // check alert history
-      const alertHistories = await AlertHistory.find({
-        alertId: alert._id,
-      });
-      expect(alertHistories.length).toBe(1);
-      expect(alertHistories[0].counts).toBe(1);
-      expect(alertHistories[0].createdAt).toEqual(
-        new Date('2023-11-16T22:10:00.000Z'),
-      );
       expect(alert.state).toBe('ALERT');
 
       // skip since time diff is less than 1 window size
@@ -353,6 +353,24 @@ describe('checkAlerts', () => {
       // alert should be in ok state
       expect(alert.state).toBe('OK');
 
+      // check alert history
+      const alertHistories = await AlertHistory.find({
+        alertId: alert._id,
+      }).sort({
+        createdAt: 1,
+      });
+      expect(alertHistories.length).toBe(2);
+      expect(alertHistories[0].state).toBe('ALERT');
+      expect(alertHistories[0].counts).toBe(1);
+      expect(alertHistories[0].createdAt).toEqual(
+        new Date('2023-11-16T22:10:00.000Z'),
+      );
+      expect(alertHistories[1].state).toBe('OK');
+      expect(alertHistories[1].counts).toBe(0);
+      expect(alertHistories[1].createdAt).toEqual(
+        new Date('2023-11-16T22:15:00.000Z'),
+      );
+
       // check if getLogsChart query + webhook were triggered
       expect(clickhouse.getLogsChart).toHaveBeenNthCalledWith(1, {
         aggFn: 'max',
@@ -477,15 +495,6 @@ describe('checkAlerts', () => {
 
       // shoud fetch 5m of logs
       await processAlert(now, alert);
-      // check alert history
-      const alertHistories = await AlertHistory.find({
-        alertId: alert._id,
-      });
-      expect(alertHistories.length).toBe(1);
-      expect(alertHistories[0].counts).toBe(1);
-      expect(alertHistories[0].createdAt).toEqual(
-        new Date('2023-11-16T22:10:00.000Z'),
-      );
       expect(alert.state).toBe('ALERT');
 
       // skip since time diff is less than 1 window size
@@ -499,6 +508,24 @@ describe('checkAlerts', () => {
       // alert should be in ok state
       expect(alert.state).toBe('OK');
 
+      // check alert history
+      const alertHistories = await AlertHistory.find({
+        alertId: alert._id,
+      }).sort({
+        createdAt: 1,
+      });
+      expect(alertHistories.length).toBe(2);
+      expect(alertHistories[0].state).toBe('ALERT');
+      expect(alertHistories[0].counts).toBe(1);
+      expect(alertHistories[0].createdAt).toEqual(
+        new Date('2023-11-16T22:10:00.000Z'),
+      );
+      expect(alertHistories[1].state).toBe('OK');
+      expect(alertHistories[1].counts).toBe(0);
+      expect(alertHistories[1].createdAt).toEqual(
+        new Date('2023-11-16T22:15:00.000Z'),
+      );
+
       // check if getLogsChart query + webhook were triggered
       expect(clickhouse.getMetricsChart).toHaveBeenNthCalledWith(1, {
         aggFn: 'max',

packages/api/src/tasks/checkAlerts.ts

@@ -464,12 +464,13 @@ export const processAlert = async (now: Date, alert: AlertDocument) => {
       return;
     }
 
+    // TODO: support INSUFFICIENT_DATA state
+    let alertState = AlertState.OK;
     const history = await new AlertHistory({
       alert: alert._id,
       createdAt: nowInMinsRoundDown,
+      state: alertState,
     }).save();
-    // TODO: support INSUFFICIENT_DATA state
-    let alertState = AlertState.OK;
     if (checksData?.rows && checksData?.rows > 0) {
       for (const checkData of checksData.data) {
         const totalCount = isString(checkData.data)
@@ -498,8 +499,11 @@ export const processAlert = async (now: Date, alert: AlertDocument) => {
           history.counts += 1;
         }
       }
+
+      history.state = alertState;
       await history.save();
     }
+
     alert.state = alertState;
     await alert.save();
   } catch (e) {