diff --git a/.env.dev b/.env.dev index aa6dd94..e38d199 100644 --- a/.env.dev +++ b/.env.dev @@ -11,7 +11,7 @@ NGINX_UI_URL=http://cueo-frontend:3030 #POSTGRES_DB_HOST=localhost #POSTGRES_DB_USERNAME=postgres #POSTGRES_DB_PASSWORD=postgres -#POSTGRES_DB_SCHEMA=cue_observe +#POSTGRES_DB_SCHEMA=cueobserve_loc #POSTGRES_DB_PORT=5432 ## SUPERUSER'S VARIABLE diff --git a/api/anomaly/services/alertmanager.py b/api/anomaly/services/alertmanager.py new file mode 100644 index 0000000..f674d1f --- /dev/null +++ b/api/anomaly/services/alertmanager.py @@ -0,0 +1,53 @@ + +import requests +import os + + +ALERTMANAGER_API_URL = os.environ.get("ALERTMANAGER_API_URL", "http://localhost:9093") + +class AlertManagers: + + def cueObserveAlerts(name, message): + + url = f'{ALERTMANAGER_API_URL}/api/v1/alerts' + data = [ + { + "status": "firing", + "labels": { + "alertname": "CueObserve Alert", + "service": name, + "severity": "critical", + "instance": "1" + }, + "annotations": { + "title":"CueObserve Alert", + "summary": "Error", + "description": message, + }, + "generatorURL": "", + } + ] + requests.request("POST", url ,json=data) + + def anomalyAlert(name, message, details,subject): + + url = f'{ALERTMANAGER_API_URL}/api/v1/alerts' + data = [ + { + "status": "firing", + "labels": { + "alertname": "Anomaly Alert", + "service": name, + "severity": "critical", + "instance": "1" + }, + "annotations": { + "title":"Anomaly Alert", + "summary": subject, + "description": message+" "+details + }, + "generatorURL": "", + } + ] + + requests.request("POST", url ,json=data) \ No newline at end of file diff --git a/api/anomaly/services/alerts.py b/api/anomaly/services/alerts.py index a241fd3..ca56697 100644 --- a/api/anomaly/services/alerts.py +++ b/api/anomaly/services/alerts.py @@ -1,6 +1,7 @@ import logging import os from email.mime.image import MIMEImage +import re import requests from anomaly.services.settings import ANOMALY_ALERT_SLACK_ID, APP_ALERTS_SLACK_ID, SLACK_BOT_TOKEN, SEND_EMAIL_TO, WEBHOOK_URL from anomaly.models import Setting @@ -13,7 +14,6 @@ ALERT_API_URL = os.environ.get("ALERT_API_URL", "http://localhost:8100") - class SlackAlert: @staticmethod def slackAlertHelper(title, message, name, details="", anomalyId: int = None): @@ -24,6 +24,7 @@ def slackAlertHelper(title, message, name, details="", anomalyId: int = None): anomalyAlertChannelId = "" appAlertChannelId = "" try: + settings = Setting.objects.all() for setting in settings.values(): if setting["name"] == ANOMALY_ALERT_SLACK_ID: @@ -44,16 +45,6 @@ def slackAlertHelper(title, message, name, details="", anomalyId: int = None): "details": details, } requests.post(url, data=payload, files={'fileImg': fileImg}) - # AppAlert - if name == "appAlert": - url = f'{ALERT_API_URL}/alerts/app-alert' - payload = { - "token": token, - "appAlertChannelId": appAlertChannelId, - "title": title, - "message": message - } - requests.request("POST", url, data=payload) except Exception as ex: logger.error("Slack URL not given or wrong URL given:%s", str(ex)) diff --git a/api/ops/tasks/anomalyDetectionTasks.py b/api/ops/tasks/anomalyDetectionTasks.py index 012aff4..5c2b481 100644 --- a/api/ops/tasks/anomalyDetectionTasks.py +++ b/api/ops/tasks/anomalyDetectionTasks.py @@ -11,7 +11,7 @@ from celery import shared_task, group from celery.result import allow_join_result -from anomaly.services.alerts import EmailAlert, WebHookAlert +from anomaly.services.alertmanager import AlertManagers from anomaly.models import ( Anomaly, AnomalyDefinition, @@ -213,7 +213,7 @@ def anomalyDetectionJob(anomalyDef_id: int, manualRun: bool = False): runStatusObj.endTimestamp = dt.datetime.now() runStatusObj.save() - ################################################# Slack Alert ######################################################## + ################################################# AlertManager ######################################################## title = "CueObserve Alerts" if runStatusObj.status == ANOMALY_DETECTION_SUCCESS: event_logs(anomalyDef_id,runStatusObj.status,totalAnomalyPublished,totalAnomalyCount) @@ -245,44 +245,18 @@ def anomalyDetectionJob(anomalyDef_id: int, manualRun: bool = False): data.update(data["data"]["anomalyLatest"]) details = ( - html2text.html2text(Template(cardTemplate.title).render(Context(data))).replace("**", "*") + html2text.html2text(Template(cardTemplate.title).render(Context(data))).replace("**", "") + "\n" ) + subject = details details = details + html2text.html2text( Template(cardTemplate.bodyText).render(Context(data)) ) name = "anomalyAlert" - SlackAlert.slackAlertHelper(title, message, name, details=details, anomalyId=anomalyId) + message = message.replace("*","") + AlertManagers.anomalyAlert( name, message, details,subject ) - ################################################## Email Alert ############################################################ - numPublished = logs["numAnomaliesPulished"] - messageHtml = f"{numPublished} {'anomalies' if numPublished > 1 else 'anomaly'} published.
" - messageHtml = ( - messageHtml - + f"Anomaly Definition: {anomalyDefinition.metric}{dimText}{highLowText}{topNtext}
" - ) - messageHtml = ( - messageHtml - + f"Dataset: {anomalyDefinition.dataset.name}
" - ) - messageHtml = ( - messageHtml + f"Granularity: {anomalyDefinition.dataset.granularity}

" - ) - emailSubject = ( - html2text.html2text(Template(cardTemplate.title).render(Context(data))).replace("**", "").replace("\n","") - - ) - detailsHtml = Template(cardTemplate.title).render(Context(data)) + "
" - subjectHtml = emailSubject - - detailsHtml = detailsHtml + Template(cardTemplate.bodyText).render(Context(data)) +"
" - EmailAlert.sendEmail(messageHtml, detailsHtml, subjectHtml, anomalyId) - - ############################################################### Webhook Alert ############################################################################# - - numPublished = logs["numAnomaliesPulished"] - webhookAlertMessageFormat(numPublished, anomalyDefinition) if runStatusObj.status == ANOMALY_DETECTION_ERROR: message = ( @@ -293,52 +267,7 @@ def anomalyDetectionJob(anomalyDef_id: int, manualRun: bool = False): message = message + str(logs["log"]) name = "appAlert" event_logs(anomalyDef_id,runStatusObj.status, totalAnomalyPublished ,totalAnomalyCount ) - SlackAlert.slackAlertHelper(title, message, name) - - ############ Webhook Alert ############ - WebHookAlert.webhookAlertHelper(name, title, message) - + AlertManagers.cueObserveAlerts(name, message) -def webhookAlertMessageFormat(numPublished, anomalyDefinition: AnomalyDefinition): - """ Format message for webhook URL alert""" - try: - textMessage = f"{numPublished} {'anomalies' if numPublished > 1 else 'anomaly'} published. " - topNtext = ( - f" Top {anomalyDefinition.value}" - if int(float(anomalyDefinition.value)) > 0 - else "" - ) - dimText = f" {anomalyDefinition.dimension}" if anomalyDefinition.dimension else "" - highLowText = f" {anomalyDefinition.highOrLow}" if anomalyDefinition.highOrLow else "" - textMessage = ( - textMessage - + f"Anomaly Definition: {anomalyDefinition.metric}{dimText}{highLowText}{topNtext}"+", " - ) - textMessage = ( - textMessage - + f"Dataset: {anomalyDefinition.dataset.name}" + ", " - ) - textMessage = ( - textMessage + f"Granularity: {anomalyDefinition.dataset.granularity}" + ", " - ) - highestContriAnomaly = anomalyDefinition.anomaly_set.order_by( - "data__contribution" - ).last() - anomalyId = highestContriAnomaly.id - data = AnomalySerializer(highestContriAnomaly).data - templateName = anomalyDefinition.getAnomalyTemplateName() - cardTemplate = AnomalyCardTemplate.objects.get(templateName=templateName) - data.update(data["data"]["anomalyLatest"]) - textSubject = ( - html2text.html2text(Template(cardTemplate.title).render(Context(data))).replace("**", "").replace("\n","") - - ) - textDetails = Template(cardTemplate.title).render(Context(data)) + " " - textDetails = textDetails + Template(cardTemplate.bodyText).render(Context(data)) + " " - textDetails = textDetails.replace("", "").replace("", "") - name = "anomalyAlert" - WebHookAlert.webhookAlertHelper(name, textSubject, textMessage, textDetails, anomalyDefinition.id, anomalyId) - except Exception as ex: - logger.error("Webhook alert failed ",str(ex)) diff --git a/docker-compose-dev.yml b/docker-compose-dev.yml index 0d7ccee..cb77ac6 100644 --- a/docker-compose-dev.yml +++ b/docker-compose-dev.yml @@ -20,7 +20,7 @@ services: context: api dockerfile: Dockerfile volumes: - - ./api:/code:rw + - ./api:/app:rw command: bash -c "chmod +x /code/start_server_new.sh && /code/start_server_new.sh" environment: @@ -73,6 +73,24 @@ services: image: redis network_mode: "host" + # prometheus: + # image: prom/prometheus:v2.28.1 + # container_name: prometheus + # volumes: + # - ./prometheus/server/prometheus.yml:/etc/prometheus/prometheus.yml + # - ./prometheus/server/rules.yml:/etc/prometheus/rules.yml + # network_mode: "host" + + # node-exporter: + # image: prom/node-exporter + # network_mode: "host" + + alertmanager: + image: prom/alertmanager:v0.22.2 + container_name: alertmanager + volumes: + - ./prometheus/server/alertmanager.yml:/etc/alertmanager/alertmanager.yml + network_mode: "host" volumes: pgdata: diff --git a/prometheus/server/alertmanager.yml b/prometheus/server/alertmanager.yml new file mode 100644 index 0000000..9e2117b --- /dev/null +++ b/prometheus/server/alertmanager.yml @@ -0,0 +1,60 @@ +global: + resolve_timeout: 2s +receivers: + - name: 'slack-notifications' + slack_configs: + - send_resolved: false + text: "{{ .CommonAnnotations.description }}" + title: "{{ .CommonAnnotations.title }} " + username: 'Prometheus' + api_url: "" + channel: '#test' + + - name: 'slack-2' + slack_configs: + - send_resolved: false + text: "{{ .CommonAnnotations.description }}" + title: "{{ .CommonAnnotations.title }} " + username: 'Prometheus' + api_url: "" + channel: '#testapp' + + - name: 'email-notifications' + email_configs: + - to: + from: + smarthost: smtp.gmail.com:587 + auth_username: + auth_identity: + auth_password: + send_resolved: false + +route: + group_wait: 1s + group_interval: 1s + repeat_interval: 24h + group_by: ['alertname'] + receiver: 'slack-notifications' + + routes: + - receiver: 'slack-notifications' + group_wait: 1s + match_re: + service: anomalyAlert + continue: true + + - receiver: 'email-notifications' + group_wait: 1s + match_re: + service: anomalyAlert + continue: true + + - receiver: 'slack-2' + group_wait: 1s + match_re: + service: appAlert + continue: true + + + +