Merge pull request #268 from strongdm/feat/add-prometheus-monitoring

Add prometheus monitoring

Author: camposer

README.md

@@ -85,6 +85,7 @@ docker logs accessbot_accessbot_1
 #### Without Docker
 
 If you want to install and execute the bot locally without Docker, please refer to: [Configure Local Environment](docs/CONFIGURE_LOCAL_ENV.md)
+If you want to expose a Prometheus endpoint with AccessBot Metrics, please refer to [Configure Monitoring](docs/configure_accessbot/CONFIGURE_MONITORING.md)
 
 ## Getting Started
 Once AccessBot is up and running, you can add it as an app or to a channel and start using it!

config.py

@@ -110,3 +110,5 @@ def get_bot_admins():
     "bot_id": None,  # will be initialized in SlackBoltBackend.resolve_access_form_bot_id method
     "nickname": os.getenv("SDM_ACCESS_FORM_BOT_NICKNAME")
 }
+
+EXPOSE_METRICS = os.getenv("SDM_EXPOSE_METRICS", "false").lower() == "true"

docker-compose-prometheus.yaml

@@ -0,0 +1,21 @@
+version: "3.9"
+services:
+  accessbot:
+    image: public.ecr.aws/strongdm/accessbot:latest
+    env_file:
+      # You could use env-file.example as a reference
+      - env-file
+    environment:
+      - SDM_EXPOSE_METRICS=true
+    ports:
+      - 3141:3141
+      - 3142:3142
+  prometheus:
+    build: tools/prometheus
+    ports:
+      - 9090:9090
+  grafana:
+    build: tools/grafana
+    ports:
+      - 3000:3000
+

docker-compose.yaml

@@ -7,3 +7,4 @@ services:
       - env-file
     ports:
       - 3141:3141
+      - 3142:3142

docs/configure_accessbot/CONFIGURE_MONITORING.md

@@ -0,0 +1,56 @@
+# CONFIGURE MONITORING
+
+To enable monitoring you need to set the variable `SDM_EXPOSE_METRICS=true`.
+
+After enabling it, a metrics endpoint will available at port `3142`. There you can see the following metrics:
+- `accessbot_total_received_messages` - total count of received messages
+- `accessbot_total_access_requests` - total count of received access requests messages
+- `accessbot_total_pending_access_requests` - total count of pending access requests
+- `accessbot_total_manual_approvals` - total count of manually approved access requests
+- `accessbot_total_auto_approvals` - total count of auto approved access requests
+- `accessbot_total_denied_access_requests` - total count of manually denied access requests
+- `accessbot_total_timed_out_access_requests` - total count of timed out access requests
+- `accessbot_total_consecutive_errors` - total count of consecutive errors
+
+To see an example, follow these steps:
+1. Download the file [docker-compose-prometheus.yaml](../../docker-compose-prometheus.yaml);
+  - Make sure that your `env-file` is properly configured following the `env-file.example` template
+2. Run with your preferred container orchestrator (with docker, you can simply run `docker-compose -f docker-compose-prometheus.yaml up`)
+
+Now you can go to the **AccessBot Metrics** Grafana Dashboard in `http://localhost:3000/d/982GyKX7z/accessbot-metrics` and see the following charts:
+
+1 - Received Messages Count (`accessbot_total_received_messages` metric):
+
+![image](https://user-images.githubusercontent.com/49597325/168816013-b71ff2b5-be8b-45ea-9a58-4cec4e51cb53.png)
+
+2 - Access Requests Count (`accessbot_total_access_requests` metric):
+
+![image](https://user-images.githubusercontent.com/49597325/168816036-f51baf75-67ed-4735-be77-51e2f9ce379a.png)
+
+3 - Pending Access Requests Count (`accessbot_total_pending_access_requests` metric):
+
+![image](https://user-images.githubusercontent.com/49597325/168816111-8b330af8-110c-4dc4-96f2-6ff554e6703b.png)
+
+4 - Manually Approved Access Requests Count (`accessbot_total_manual_approvals` metric):
+
+![image](https://user-images.githubusercontent.com/49597325/168816119-344c8c2c-ddad-4008-a5c6-4ee8c02fb66f.png)
+
+5 - Auto Approved Access Requests Count (`accessbot_total_auto_approvals` metric):
+
+![image](https://user-images.githubusercontent.com/49597325/168816132-755b62f2-da7c-49ab-9c7f-bf20b5b0162b.png)
+
+6 - Manually Denied Access Requests Count (`accessbot_total_denied_access_requests` metric):
+
+![image](https://user-images.githubusercontent.com/49597325/168816152-1ffbffe7-128e-475a-9ba4-2971431c380d.png)
+
+7 - Timed Out Access Requests Count (`accessbot_total_timed_out_access_requests` metric):
+
+![image](https://user-images.githubusercontent.com/49597325/168816166-8deb901e-ccfd-4101-9ae7-8e290f429ea6.png)
+
+8 - Total Consecutive Errors Count (`accessbot_total_consecutive_errors` metric):
+
+![image](https://user-images.githubusercontent.com/49597325/168816189-a559a694-2790-49be-87a0-1d06f4a73cb4.png)
+
+9 - Last Execution Status (`accessbot_total_consecutive_errors` metric - 0 means that everything is fine, 1 means that the last execution(s) failed):
+
+![image](https://user-images.githubusercontent.com/49597325/168816198-c76173b2-cb18-4799-9590-d7405bf5496f.png)

e2e/test_common.py

@@ -22,7 +22,8 @@ class ErrBotExtraTestSettings:
                 'allowprivate': True,
                 'allowmuc': False,
             }
-        }
+        },
+        'EXPOSE_METRICS': False,
     }
     extra_plugin_dir = "plugins/sdm"
 
@@ -42,7 +43,8 @@ class MSTeamsErrBotExtraTestSettings:
                 'allowprivate': True,
                 'allowmuc': False,
             }
-        }
+        },
+        'EXPOSE_METRICS': False,
     }
     extra_plugin_dir = "plugins/sdm"
 

plugins/sdm/accessbot.py

@@ -1,19 +1,16 @@
 import os
 import re
-import time
-import json
-import copy
 from itertools import chain
-from errbot import BotPlugin, re_botcmd, Message, botcmd
+
+from errbot import BotPlugin, re_botcmd, Message
 from errbot.core import ErrBot
 from slack_sdk.errors import SlackApiError
-from collections import namedtuple
 
 import config_template
 from lib import ApproveHelper, create_sdm_service, MSTeamsPlatform, PollerHelper, \
     ShowResourcesHelper, ShowRolesHelper, SlackBoltPlatform, SlackRTMPlatform, \
     ResourceGrantHelper, RoleGrantHelper, DenyHelper, CommandAliasHelper, ArgumentsHelper, \
-    GrantRequestHelper, WhoamiHelper
+    GrantRequestHelper, WhoamiHelper, MetricsHelper
 from lib.util import normalize_utf8
 from grant_request_type import GrantRequestType
 
@@ -25,6 +22,7 @@
 SHOW_ROLES_REGEX = r"show available roles"
 FIVE_SECONDS = 5
 ONE_MINUTE = 60
+MSG_ERROR_OCCURRED = "An error occurred, please contact your SDM admin"
 
 def get_callback_message_fn(bot):
     def callback_message(msg):
@@ -38,6 +36,14 @@ def callback_message(msg):
         ErrBot.callback_message(bot, msg)
     return callback_message
 
+def get_send_simple_reply(bot):
+    def send_simple_reply(msg, text, private=False, threaded=False):
+        if text.startswith(MSG_ERROR_OCCURRED):
+            accessbot = bot.plugin_manager.plugins['AccessBot']
+            accessbot.get_metrics_helper().increment_consecutive_errors()
+        return ErrBot.send_simple_reply(bot, msg, text, private=private, threaded=threaded)
+    return send_simple_reply
+
 def get_platform(bot):
     platform = bot.bot_config.BOT_PLATFORM if hasattr(bot.bot_config, 'BOT_PLATFORM') else None
     if platform == 'ms-teams':
@@ -50,13 +56,15 @@ def get_platform(bot):
 # pylint: disable=too-many-ancestors
 class AccessBot(BotPlugin):
     __grant_requests_helper = None
+    __metrics_helper = None
     _platform = None
 
     def activate(self):
         super().activate()
         self._platform = get_platform(self)
-        self._bot.MSG_ERROR_OCCURRED = 'An error occurred, please contact your SDM admin'
+        self._bot.MSG_ERROR_OCCURRED = MSG_ERROR_OCCURRED
         self._bot.callback_message = get_callback_message_fn(self._bot)
+        self._bot.send_simple_reply = get_send_simple_reply(self._bot)
         self.init_access_form_bot()
         self.update_access_control_admins()
         self['auto_approve_uses'] = {}
@@ -68,6 +76,8 @@ def activate(self):
         # If something doesn't need to be "instantiated" again we shouldn't be doing it
         if self.__grant_requests_helper is None:
             self.__grant_requests_helper = GrantRequestHelper(self)
+        if self.__metrics_helper is None:
+            self.__metrics_helper = MetricsHelper(self)
         self._hide_utils_whoami_command()
 
     def _hide_utils_whoami_command(self):
@@ -156,6 +166,7 @@ def access_resource(self, message, match):
         """
         Grant access to a resource (using the requester's email address)
         """
+        self.__metrics_helper.increment_access_requests()
         arguments = re.sub(ACCESS_REGEX, "\\1", match.string.replace("*", ""), flags=re.IGNORECASE)
         if re.match("^role (.*)", arguments, flags=re.IGNORECASE):
             self.log.debug("##SDM## AccessBot.access better match for assign_role")
@@ -172,6 +183,7 @@ def access_resource(self, message, match):
             yield str(e)
             return
         yield from self.get_resource_grant_helper().request_access(message, resource_name, flags=flags)
+        self.__metrics_helper.reset_consecutive_errors()
 
     @re_botcmd(pattern=ASSIGN_ROLE_REGEX, flags=re.IGNORECASE, prefixed=False, re_cmd_name_help="access to role role-name")
     def assign_role(self, message, match):
@@ -180,48 +192,58 @@ def assign_role(self, message, match):
         """
         if not self._platform.can_assign_role(message):
             return
+        self.__metrics_helper.increment_access_requests()
         role_name = re.sub(ASSIGN_ROLE_REGEX, "\\1", match.string.replace("*", ""), flags=re.IGNORECASE)
         yield from self.get_role_grant_helper().request_access(message, role_name)
+        self.__metrics_helper.reset_consecutive_errors()
 
     @re_botcmd(pattern=APPROVE_REGEX, flags=re.IGNORECASE, prefixed=False, hidden=True)
     def approve(self, message, match):
         """
         Approve a grant (resource or role)
         """
+        self.__metrics_helper.increment_received_messages()
         access_request_id = re.sub(APPROVE_REGEX, r"\1", match.string.replace("*", ""), flags=re.IGNORECASE).upper()
         approver = message.frm
         yield from self.get_approve_helper().execute(approver, access_request_id)
+        self.__metrics_helper.reset_consecutive_errors()
 
     @re_botcmd(pattern=DENY_REGEX, flags=re.IGNORECASE, prefixed=False, hidden=True)
     def deny(self, message, match):
         """
         Deny a grant request (resource or role)
         """
+        self.__metrics_helper.increment_received_messages()
         access_request_id = re.sub(DENY_REGEX, r"\1", match.string.replace("*", ""), flags=re.IGNORECASE).upper()
         denial_reason = re.sub(DENY_REGEX, r"\2", match.string.replace("*", ""), flags=re.IGNORECASE)
         admin = message.frm
         yield from self.get_deny_helper().execute(admin, access_request_id, denial_reason)
+        self.__metrics_helper.reset_consecutive_errors()
 
     #pylint: disable=unused-argument
     @re_botcmd(pattern=SHOW_RESOURCES_REGEX, flags=re.IGNORECASE, prefixed=False, re_cmd_name_help="show available resources [--filter expression]")
     def show_resources(self, message, match):
         """
         Show all available resources
         """
+        self.__metrics_helper.increment_received_messages()
         if not self._platform.can_show_resources(message):
             return
         flags = self.get_arguments_helper().extract_flags(message.body)
         yield from self.get_show_resources_helper().execute(message, flags=flags)
+        self.__metrics_helper.reset_consecutive_errors()
 
     #pylint: disable=unused-argument
     @re_botcmd(pattern=SHOW_ROLES_REGEX, flags=re.IGNORECASE, prefixed=False, re_cmd_name_help="show available roles")
     def show_roles(self, message, match):
         """
         Show all available roles
         """
+        self.__metrics_helper.increment_received_messages()
         if not self._platform.can_show_roles(message):
             return
         yield from self.get_show_roles_helper().execute(message)
+        self.__metrics_helper.reset_consecutive_errors()
 
     @re_botcmd(pattern=r"whoami", flags=re.IGNORECASE, prefixed=False, name="accessbot-whoami")
     def whoami(self, message, _):
@@ -279,17 +301,22 @@ def get_arguments_helper(self):
     def get_whoami_helper(self):
         return WhoamiHelper(self)
 
+    def get_metrics_helper(self):
+        return self.__metrics_helper
+
     def get_admin_ids(self):
         return self._platform.get_admin_ids()
 
     def enter_grant_request(self, request_id: str, message, sdm_object, sdm_account, grant_request_type: GrantRequestType, flags: dict = None):
         self.__grant_requests_helper.add(request_id, message, sdm_object, sdm_account, grant_request_type, flags)
+        self.__metrics_helper.increment_pending_requests()
 
     def grant_requests_exists(self, request_id: str):
         return self.__grant_requests_helper.exists(request_id)
 
     def remove_grant_request(self, request_id):
         self.__grant_requests_helper.remove(request_id)
+        self.__metrics_helper.decrement_pending_requests()
 
     def get_grant_request(self, request_id):
         return self.__grant_requests_helper.get(request_id)

plugins/sdm/lib/helper/__init__.py

@@ -11,3 +11,4 @@
 from .arguments_helper import *
 from .grant_request_helper import *
 from .whoami_helper import *
+from .metrics_helper import *

plugins/sdm/lib/helper/approve_helper.py

@@ -3,6 +3,7 @@
 from grant_request_type import GrantRequestType
 from .base_evaluate_request_helper import BaseEvaluateRequestHelper
 from ..util import convert_duration_flag_to_timedelta, get_formatted_duration_string
+from metric_type import MetricGaugeType
 
 
 class ApproveHelper(BaseEvaluateRequestHelper):
@@ -24,6 +25,7 @@ def __approve_assign_role(self, grant_request):
         self._bot.add_thumbsup_reaction(grant_request['message'])
         self._bot.remove_grant_request(grant_request['id'])
         yield from self.__notify_assign_role_request_granted(grant_request['message'], grant_request['sdm_object'].name)
+        self._bot.get_metrics_helper().increment_manual_approvals()
 
     def __approve_access_resource(self, grant_request):
         duration = grant_request['flags'].get('duration')
@@ -37,6 +39,7 @@ def __approve_access_resource(self, grant_request):
         self._bot.add_thumbsup_reaction(grant_request['message'])
         self._bot.remove_grant_request(grant_request['id'])
         yield from self.__notify_access_request_granted(grant_request['message'], resource, duration, needs_renewal)
+        self._bot.get_metrics_helper().increment_manual_approvals()
 
     def __grant_temporal_access_by_role(self, role_name, account_id):
         grant_start_from = datetime.datetime.now(datetime.timezone.utc)

plugins/sdm/lib/helper/base_grant_helper.py

@@ -6,6 +6,8 @@
     convert_duration_flag_to_timedelta, get_approvers_channel
 from grant_request_type import GrantRequestType
 
+from metric_type import MetricGaugeType
+
 
 class BaseGrantHelper(ABC):
     def __init__(self, bot, sdm_service, admin_ids, grant_type, auto_approve_tag_key, auto_approve_all_key):
@@ -91,6 +93,7 @@ def __auto_approve_access_request(self, message, sdm_object, sdm_account, execut
         self.__enter_grant_request(message, sdm_object, sdm_account, self.__grant_type, request_id, flags=flags)
         self.__bot.log.info("##SDM## %s GrantHelper.__grant_%s granting access", execution_id, self.__grant_type)
         yield from self.__bot.get_approve_helper().evaluate(request_id, is_auto_approve=True)
+        self.__bot.get_metrics_helper().increment_auto_approvals()
 
     def __request_manual_approval(self, message, sdm_object, sdm_account, execution_id, request_id, sender_nick, flags: dict):
         approvers_channel_name = sdm_object.tags.get(self.__bot.config['APPROVERS_CHANNEL_TAG']) if sdm_object.tags else None