[DEV-13237] Allow Alert Tuning. Fix Standalone EKS Cluster (#410)

Author: NathanOkolita

monitoring.tf

@@ -99,7 +99,16 @@ alerting:
     enabled: ${var.alerting_pagerduty_enabled}
     integrationKey: ${var.alerting_pagerduty_integration_key}
     integrationUrl: "https://events.pagerduty.com/generic/2010-04-15/create_event.json"
+${local.standard_rules}
 EOT
+  )
+  standard_rules = var.alerting_standard_rules != "" ? (<<EOT
+  standardRules:
+    ${indent(4, base64decode(var.alerting_standard_rules))}
+EOT
+  ) : (<<EOT
+  noExtraConfigs: true
+  EOT
   )
   alertmanager_tls = var.acm_arn == "" ? (<<EOT
       tls:

nodes.tf

@@ -18,6 +18,7 @@ locals {
       spot             = false
       desired_capacity = "0"
       taints           = "--register-with-taints=indico.io/celery=true:NoSchedule"
+      additional_node_labels = ""
     },
     static-workers = {
       min_size         = 1
@@ -26,6 +27,8 @@ locals {
       type             = "cpu"
       spot             = false
       desired_capacity = "0"
+      additional_node_labels = ""
+      taints           = ""
     },
     pdf-workers = {
       min_size         = 0
@@ -35,6 +38,7 @@ locals {
       spot             = false
       desired_capacity = "1"
       taints           = "--register-with-taints=indico.io/pdfextraction=true:NoSchedule"
+      additional_node_labels = ""
     },
     highmem-workers = {
       min_size         = 0
@@ -44,6 +48,7 @@ locals {
       spot             = false
       desired_capacity = "0"
       taints           = "--register-with-taints=indico.io/highmem=true:NoSchedule"
+      additional_node_labels = ""    
     },
     monitoring-workers = {
       min_size         = 1
@@ -53,6 +58,7 @@ locals {
       spot             = false
       desired_capacity = "1"
       taints           = "--register-with-taints=indico.io/monitoring=true:NoSchedule"
+      additional_node_labels = ""
     },
     pgo-workers = {
       min_size         = 1
@@ -62,6 +68,7 @@ locals {
       spot             = false
       desired_capacity = "1"
       taints           = "--register-with-taints=indico.io/crunchy=true:NoSchedule"
+      additional_node_labels = ""
     },
     readapi-servers = {
       min_size         = 0
@@ -71,6 +78,7 @@ locals {
       spot             = false
       desired_capacity = "0"
       taints           = "--register-with-taints=indico.io/readapi-server=true:NoSchedule"
+      additional_node_labels = ""
     },
     readapi-azurite = {
       min_size         = 0
@@ -80,6 +88,7 @@ locals {
       spot             = false
       desired_capacity = "1"
       taints           = "--register-with-taints=indico.io/azurite=true:NoSchedule"
+      additional_node_labels = ""
     }
   }
 
@@ -91,6 +100,8 @@ locals {
       min_size         = 1
       max_size         = 5
       desired_capacity = "3"
+      additional_node_labels = ""
+      taints           = ""
     },
     pgo-workers = {
       type             = "cpu"
@@ -100,6 +111,7 @@ locals {
       max_size         = 2
       desired_capacity = "2"
       taints           = "--register-with-taints=indico.io/crunchy=true:NoSchedule"
+      additional_node_labels = ""
     },
     celery-workers = {
       type             = "cpu"
@@ -109,6 +121,7 @@ locals {
       max_size         = 3
       desired_capacity = "1"
       taints           = "--register-with-taints=indico.io/celery-workers=true:NoSchedule"
+      additional_node_labels = ""
     },
     minio = {
       type             = "cpu"
@@ -118,6 +131,7 @@ locals {
       max_size         = 4
       desired_capacity = "4"
       taints           = "--register-with-taints=indico.io/minio=true:NoSchedule"
+      additional_node_labels = ""
     },
     monitoring-workers = {
       min_size         = 1
@@ -127,6 +141,7 @@ locals {
       spot             = false
       desired_capacity = "1"
       taints           = "--register-with-taints=indico.io/monitoring=true:NoSchedule"
+      additional_node_labels = ""
     },
     weaviate = {
       type             = "cpu"
@@ -136,6 +151,7 @@ locals {
       max_size         = 3
       desired_capacity = "3"
       taints           = "--register-with-taints=indico.io/weaviate=true:NoSchedule"
+      additional_node_labels = ""
     },
     weaviate-workers = {
       type             = "cpu"
@@ -145,10 +161,31 @@ locals {
       max_size         = 4
       desired_capacity = "2"
       taints           = "--register-with-taints=indico.io/weaviate-workers=true:NoSchedule"
+      additional_node_labels = ""
     }
   }
 
-  default_node_groups = merge((var.insights_enabled ? local.insights_default_node_groups : tomap(null)), (var.ipa_enabled ? local.intake_default_node_groups : tomap(null)))
+  standalone_node_groups = {
+    default-workers = {
+      min_size         = 1
+      max_size         = 3
+      instance_types   = ["m5.xlarge"]
+      type             = "cpu"
+      spot             = false
+      desired_capacity = "0"
+      additional_node_labels = ""
+      taints           = ""
+    }
+  }
+
+  default_node_groups = (
+    var.ipa_enabled == false && var.insights_enabled == false
+    ? local.standalone_node_groups
+    : merge(
+      var.insights_enabled ? local.insights_default_node_groups : tomap(null),
+      var.ipa_enabled ? local.intake_default_node_groups : tomap(null)
+    )
+  )
 
   # This is to avoid terraform errors when the node groups variable is set,
   # as different keys make the objects incompatible for a ternary function. 

tf-smoketest-variables.tf

@@ -119,6 +119,7 @@ resource "kubernetes_config_map" "terraform-variables" {
     alerting_email_host = "${jsonencode(var.alerting_email_host)}"
     alerting_email_username = "${jsonencode(var.alerting_email_username)}"
     alerting_email_password = "${jsonencode(var.alerting_email_password)}"
+    alerting_standard_rules = "${jsonencode(var.alerting_standard_rules)}"
     eks_addon_version_guardduty = "${jsonencode(var.eks_addon_version_guardduty)}"
     use_static_ssl_certificates = "${jsonencode(var.use_static_ssl_certificates)}"
     ssl_static_secret_name = "${jsonencode(var.ssl_static_secret_name)}"

variables.tf

@@ -711,6 +711,12 @@ variable "alerting_email_password" {
   description = "alerting_email_password"
 }
 
+variable "alerting_standard_rules" {
+  type        = string
+  default     = ""
+  description = "alerting_standard_rules"
+}
+
 variable "eks_addon_version_guardduty" {
   type        = bool
   default     = true