Alertmanager Implementation in CueObserve

.env.dev

@@ -11,7 +11,7 @@ NGINX_UI_URL=http://cueo-frontend:3030
 #POSTGRES_DB_HOST=localhost
 #POSTGRES_DB_USERNAME=postgres
 #POSTGRES_DB_PASSWORD=postgres
-#POSTGRES_DB_SCHEMA=cue_observe
+#POSTGRES_DB_SCHEMA=cueobserve_loc
 #POSTGRES_DB_PORT=5432
 
 ## SUPERUSER'S VARIABLE

api/anomaly/services/alertmanager.py

@@ -0,0 +1,53 @@
+
+import requests
+import os
+
+
+ALERTMANAGER_API_URL = os.environ.get("ALERTMANAGER_API_URL", "http://localhost:9093")
+
+class AlertManagers:
+
+    def cueObserveAlerts(name, message):
+
+        url = f'{ALERTMANAGER_API_URL}/api/v1/alerts'
+        data = [
+                    {
+                        "status": "firing",
+                        "labels": {
+                        "alertname": "CueObserve Alert",
+                        "service": name,
+                        "severity": "critical",
+                        "instance": "1"
+                        },
+                        "annotations": {
+                        "title":"CueObserve Alert",
+                        "summary": "Error",
+                        "description": message,
+                        },
+                        "generatorURL": "",
+                    }
+                ]
+        requests.request("POST", url ,json=data)
+
+    def anomalyAlert(name, message, details,subject):
+
+        url = f'{ALERTMANAGER_API_URL}/api/v1/alerts'
+        data = [
+                    {
+                        "status": "firing",
+                        "labels": {
+                        "alertname": "Anomaly Alert",
+                        "service": name,
+                        "severity": "critical",
+                        "instance": "1"
+                        },
+                        "annotations": {
+                        "title":"Anomaly Alert",
+                        "summary": subject,
+                        "description": message+" "+details
+                        },
+                        "generatorURL": "",
+                    }
+                ]
+
+        requests.request("POST", url ,json=data)

api/anomaly/services/alerts.py

@@ -1,6 +1,7 @@
 import logging
 import os
 from email.mime.image import MIMEImage
+import re
 import requests
 from anomaly.services.settings import ANOMALY_ALERT_SLACK_ID, APP_ALERTS_SLACK_ID, SLACK_BOT_TOKEN, SEND_EMAIL_TO, WEBHOOK_URL
 from anomaly.models import Setting
@@ -13,7 +14,6 @@
 ALERT_API_URL = os.environ.get("ALERT_API_URL", "http://localhost:8100")
 
 
-
 class SlackAlert:
     @staticmethod
     def slackAlertHelper(title, message, name, details="", anomalyId: int = None):
@@ -24,6 +24,7 @@ def slackAlertHelper(title, message, name, details="", anomalyId: int = None):
         anomalyAlertChannelId = ""
         appAlertChannelId = ""
         try:
+
             settings = Setting.objects.all()
             for setting in settings.values():
                 if setting["name"] == ANOMALY_ALERT_SLACK_ID:
@@ -44,16 +45,6 @@ def slackAlertHelper(title, message, name, details="", anomalyId: int = None):
                     "details": details,
                 }
                 requests.post(url, data=payload, files={'fileImg': fileImg})
-            # AppAlert
-            if name == "appAlert":
-                url = f'{ALERT_API_URL}/alerts/app-alert'
-                payload = {
-                    "token": token,
-                    "appAlertChannelId": appAlertChannelId,
-                    "title": title,
-                    "message": message
-                }
-                requests.request("POST", url, data=payload)
 
         except Exception as ex:
             logger.error("Slack URL not given or wrong URL given:%s", str(ex))

api/ops/tasks/anomalyDetectionTasks.py

@@ -11,7 +11,7 @@
 from celery import shared_task, group
 from celery.result import allow_join_result
 
-from anomaly.services.alerts import EmailAlert, WebHookAlert
+from anomaly.services.alertmanager import AlertManagers
 from anomaly.models import (
     Anomaly,
     AnomalyDefinition,
@@ -213,7 +213,7 @@ def anomalyDetectionJob(anomalyDef_id: int, manualRun: bool = False):
     runStatusObj.endTimestamp = dt.datetime.now()
     runStatusObj.save()
 
-    ################################################# Slack Alert ########################################################
+    ################################################# AlertManager ########################################################
     title = "CueObserve Alerts"
     if runStatusObj.status == ANOMALY_DETECTION_SUCCESS:
         event_logs(anomalyDef_id,runStatusObj.status,totalAnomalyPublished,totalAnomalyCount)
@@ -245,44 +245,18 @@ def anomalyDetectionJob(anomalyDef_id: int, manualRun: bool = False):
             data.update(data["data"]["anomalyLatest"])
 
             details = (
-                html2text.html2text(Template(cardTemplate.title).render(Context(data))).replace("**", "*")
+                html2text.html2text(Template(cardTemplate.title).render(Context(data))).replace("**", "")
                 + "\n"
             )
+            subject = details
             details = details + html2text.html2text(
                 Template(cardTemplate.bodyText).render(Context(data))
             )
 
             name = "anomalyAlert"
-            SlackAlert.slackAlertHelper(title, message, name, details=details, anomalyId=anomalyId)
+            message = message.replace("*","")
+            AlertManagers.anomalyAlert( name, message, details,subject )
 
-            ################################################## Email Alert ############################################################
-            numPublished = logs["numAnomaliesPulished"]
-            messageHtml = f"{numPublished} {'anomalies' if numPublished > 1 else 'anomaly'} published. <br>"
-            messageHtml = (
-                messageHtml
-                + f"Anomaly Definition: <b>{anomalyDefinition.metric}{dimText}{highLowText}{topNtext}</b> <br>"
-            )
-            messageHtml = (
-                messageHtml
-                + f"Dataset: {anomalyDefinition.dataset.name} <br>"
-            )
-            messageHtml = (
-                messageHtml +  f"Granularity: {anomalyDefinition.dataset.granularity} <br> <br>"
-            )
-            emailSubject = (
-                html2text.html2text(Template(cardTemplate.title).render(Context(data))).replace("**", "").replace("\n","")
-
-            )
-            detailsHtml = Template(cardTemplate.title).render(Context(data)) + "<br>"
-            subjectHtml = emailSubject
-
-            detailsHtml = detailsHtml + Template(cardTemplate.bodyText).render(Context(data)) +"<br>"
-            EmailAlert.sendEmail(messageHtml, detailsHtml, subjectHtml, anomalyId)
-
-            ############################################################### Webhook Alert #############################################################################
-
-            numPublished = logs["numAnomaliesPulished"]
-            webhookAlertMessageFormat(numPublished, anomalyDefinition)
 
     if runStatusObj.status == ANOMALY_DETECTION_ERROR:
         message = (
@@ -293,52 +267,7 @@ def anomalyDetectionJob(anomalyDef_id: int, manualRun: bool = False):
         message = message + str(logs["log"])
         name = "appAlert"
         event_logs(anomalyDef_id,runStatusObj.status, totalAnomalyPublished ,totalAnomalyCount )
-        SlackAlert.slackAlertHelper(title, message, name)
-
-        ############ Webhook Alert ############
-        WebHookAlert.webhookAlertHelper(name, title, message)
-
+        AlertManagers.cueObserveAlerts(name, message)
 
 
-def webhookAlertMessageFormat(numPublished, anomalyDefinition: AnomalyDefinition):
-    """ Format message for webhook URL alert"""
-    try:
-        textMessage = f"{numPublished} {'anomalies' if numPublished > 1 else 'anomaly'} published. "
-        topNtext = (
-            f" Top {anomalyDefinition.value}"
-            if int(float(anomalyDefinition.value)) > 0
-            else ""
-        )
-        dimText = f" {anomalyDefinition.dimension}" if anomalyDefinition.dimension else ""
-        highLowText = f" {anomalyDefinition.highOrLow}" if anomalyDefinition.highOrLow else ""
-        textMessage = (
-            textMessage
-            + f"Anomaly Definition: {anomalyDefinition.metric}{dimText}{highLowText}{topNtext}"+", "
-        )
-        textMessage = (
-            textMessage
-            + f"Dataset: {anomalyDefinition.dataset.name}" + ", "
-        )
-        textMessage = (
-            textMessage +  f"Granularity: {anomalyDefinition.dataset.granularity}" + ", "
-        )
-        highestContriAnomaly = anomalyDefinition.anomaly_set.order_by(
-            "data__contribution"
-        ).last()
-        anomalyId = highestContriAnomaly.id
-        data = AnomalySerializer(highestContriAnomaly).data
-        templateName = anomalyDefinition.getAnomalyTemplateName()
-        cardTemplate = AnomalyCardTemplate.objects.get(templateName=templateName)
-        data.update(data["data"]["anomalyLatest"])
-        textSubject = (
-            html2text.html2text(Template(cardTemplate.title).render(Context(data))).replace("**", "").replace("\n","")
-
-        )
-        textDetails = Template(cardTemplate.title).render(Context(data)) + " "
-        textDetails = textDetails + Template(cardTemplate.bodyText).render(Context(data)) + " "
-        textDetails = textDetails.replace("<b>", "").replace("</b>", "")
-        name = "anomalyAlert"
-        WebHookAlert.webhookAlertHelper(name, textSubject, textMessage, textDetails, anomalyDefinition.id, anomalyId)
-    except Exception as ex:
-        logger.error("Webhook alert failed ",str(ex))
 

docker-compose-dev.yml

@@ -20,7 +20,7 @@ services:
       context: api
       dockerfile: Dockerfile
     volumes:
-      - ./api:/code:rw
+      - ./api:/app:rw
     command:
       bash -c "chmod +x /code/start_server_new.sh && /code/start_server_new.sh"
     environment:
@@ -73,6 +73,24 @@ services:
     image: redis
     network_mode: "host"
 
+  # prometheus:
+  #   image: prom/prometheus:v2.28.1
+  #   container_name: prometheus
+  #   volumes:
+  #     - ./prometheus/server/prometheus.yml:/etc/prometheus/prometheus.yml
+  #     - ./prometheus/server/rules.yml:/etc/prometheus/rules.yml
+  #   network_mode: "host"
+
+  # node-exporter:
+  #   image: prom/node-exporter
+  #   network_mode: "host"
+
+  alertmanager:
+    image: prom/alertmanager:v0.22.2
+    container_name: alertmanager
+    volumes:
+      - ./prometheus/server/alertmanager.yml:/etc/alertmanager/alertmanager.yml
+    network_mode: "host"
 volumes:
   pgdata:
 

prometheus/server/alertmanager.yml

@@ -0,0 +1,60 @@
+global:
+  resolve_timeout: 2s 
+receivers:
+  - name: 'slack-notifications'
+    slack_configs:
+      - send_resolved: false
+        text: "{{ .CommonAnnotations.description }}"
+        title: "{{ .CommonAnnotations.title }} "
+        username: 'Prometheus'
+        api_url: ""
+        channel: '#test'
+
+  - name: 'slack-2'
+    slack_configs:
+      - send_resolved: false
+        text: "{{ .CommonAnnotations.description }}"
+        title: "{{ .CommonAnnotations.title }} "
+        username: 'Prometheus'
+        api_url: ""
+        channel: '#testapp'
+
+  - name: 'email-notifications'
+    email_configs:
+    - to: 
+      from: 
+      smarthost: smtp.gmail.com:587
+      auth_username: 
+      auth_identity: 
+      auth_password: 
+      send_resolved: false
+
+route:
+  group_wait: 1s
+  group_interval: 1s
+  repeat_interval: 24h
+  group_by: ['alertname']
+  receiver: 'slack-notifications'
+
+  routes:
+  - receiver: 'slack-notifications'
+    group_wait: 1s
+    match_re:
+      service: anomalyAlert
+    continue: true
+
+  - receiver: 'email-notifications'
+    group_wait: 1s
+    match_re:
+      service: anomalyAlert
+    continue: true
+
+  - receiver: 'slack-2'
+    group_wait: 1s
+    match_re:
+      service: appAlert
+    continue: true
+
+
+
+