Label alerts destined for PagerDuty

Author: adamgibbins

lib/stackbuilder/stacks/services/app_service.rb

@@ -33,6 +33,7 @@ module Stacks::Services::AppService
   attr_accessor :alerts_channel
   attr_accessor :startup_alert_threshold
   attr_accessor :monitor_readiness_probe
+  attr_accessor :page_on_critical
 
   alias_method :database_application_name, :application
 
@@ -60,6 +61,7 @@ def configure
     @cpu_request = false
     @cpu_limit = false
     @monitor_readiness_probe = true
+    @page_on_critical = false
   end
 
   def enable_ehcache
@@ -597,6 +599,8 @@ def generate_k8s_pod_disruption_budget(app_service_labels)
   def generate_k8s_alerting(site, app_service_labels)
     fail("app_service '#{name}' in '#{@environment.name}' requires alerts_channel (set self.alerts_channel)") if @alerts_channel.nil?
 
+    pagerduty = page_on_critical ? { 'pagerduty' => 'true' } : {}
+
     rules = []
 
     rules << {
@@ -606,7 +610,7 @@ def generate_k8s_alerting(site, app_service_labels)
         'severity' => 'critical',
         'alertname' => "#{k8s_app_resources_name} CRITICAL",
         'alert_owner_channel' => alerts_channel
-      },
+      }.merge(pagerduty),
       'annotations' => {
         'message' => '{{ $value }} components are critical on {{ $labels.namespace }}/{{ $labels.pod }}',
         'status_page_url' => "https://go.timgroup.com/insight/#{site}/proxy/{{ $labels.namespace }}/{{ $labels.pod }}/info/status"
@@ -637,7 +641,7 @@ def generate_k8s_alerting(site, app_service_labels)
         'severity' => 'critical',
         'alertname' => "#{k8s_app_resources_name} is stuck in a crash loop",
         'alert_owner_channel' => alerts_channel
-      },
+      }.merge(pagerduty),
       'annotations' => {
         'message' => 'Pod {{ $labels.namespace }}/{{ $labels.pod }} ({{ $labels.container }}) is restarting ' \
           '{{ printf "%.2f" $value }} times / 5 minutes.'

spec/app_service_spec.rb

@@ -1343,6 +1343,60 @@ def k8s_resource(set, kind)
                                             })
       end
 
+      it 'creates a paging alert rule for critical status components' do
+        factory = eval_stacks do
+          stack "mystack" do
+            app_service "x", :kubernetes => true do
+              self.maintainers = [person('Testers')]
+              self.description = 'Testing'
+              self.alerts_channel = 'test'
+              self.page_on_critical = true
+
+              self.application = 'MyApplication'
+              self.startup_alert_threshold = '1h'
+            end
+          end
+          env "e1", :primary_site => 'space' do
+            instantiate_stack "mystack"
+          end
+        end
+        set = factory.inventory.find_environment('e1').definitions['mystack'].k8s_machinesets['x']
+        prometheus_rule = k8s_resource(set, 'PrometheusRule')
+
+        expect(prometheus_rule['apiVersion']).to eql('monitoring.coreos.com/v1')
+        expect(prometheus_rule['metadata']).to eql('labels' => {
+                                                     'prometheus' => 'main',
+                                                     'role' => 'alert-rules',
+                                                     'app.kubernetes.io/managed-by' => 'stacks',
+                                                     'stack' => 'mystack',
+                                                     'machineset' => 'x',
+                                                     'group' => 'blue',
+                                                     'app.kubernetes.io/instance' => 'blue',
+                                                     'app.kubernetes.io/part-of' => 'x',
+                                                     'app.kubernetes.io/component' => 'app_service'
+                                                   },
+                                                   'name' => 'x-blue-app',
+                                                   'namespace' => 'e1')
+
+        expect(prometheus_rule['spec']['groups'].first['name']).to eql('stacks-alerts')
+        status_critical_rule = prometheus_rule['spec']['groups'].first['rules'].find do |r|
+          r['alert'] == 'StatusCritical'
+        end
+        expected_status_page_url = "https://go.timgroup.com/insight/space/proxy/{{ $labels.namespace }}/{{ $labels.pod }}/info/status"
+        expect(status_critical_rule).to eql('alert' => 'StatusCritical',
+                                            'expr' => 'sum(tucker_component_status{job="x-blue-app",status="critical"}) by (pod, namespace) > 0',
+                                            'labels' => {
+                                              'severity' => 'critical',
+                                              'alertname' => 'x-blue-app CRITICAL',
+                                              'alert_owner_channel' => 'test',
+                                              'pagerduty' => 'true'
+                                            },
+                                            'annotations' => {
+                                              'message' => '{{ $value }} components are critical on {{ $labels.namespace }}/{{ $labels.pod }}',
+                                              'status_page_url' => expected_status_page_url
+                                            })
+      end
+
       it 'creates an alert rule for pods stuck in a crash loop' do
         factory = eval_stacks do
           stack "mystack" do