Add alerts for unsynced and unready crossplane resources

class/defaults.yml

@@ -136,7 +136,7 @@ parameters:
     appuioManaged: true
     quotasEnabled: false
     billingEnabled: false
-    monitoringEnabled: false
+    monitoringEnabled: true
     grpcEndpoint: host.docker.internal:9443
     proxyFunction: false
 

component/appcat_controller.jsonnet

@@ -175,6 +175,55 @@ local servicemonitor = loadManifest('servicemonitor.yaml') {
   },
 };
 
+local prometheusrule = kube._Object('monitoring.coreos.com/v1', 'PrometheusRule', 'appcat-crossplane-resources') {
+  metadata+: {
+    namespace: controllersParams.namespace,
+    labels: {
+      'app.kubernetes.io/name': 'appcat',
+      'app.kubernetes.io/managed-by': 'commodore',
+    },
+  },
+  spec: {
+    groups: [
+      {
+        name: 'crossplane-resources.rules',
+        rules: [
+          {
+            alert: 'CrossplaneResourceUnsynced',
+            expr: 'max by (api_version, kind, name, claim_name, claim_namespace, instance_name, status_synced, status_ready) (crossplane_resource_info{status_synced!="ReconcileSuccess"}) == 1',
+            'for': '20m',
+            labels: {
+              severity: 'warning',
+              syn: 'true',
+              syn_team: 'schedar',
+              syn_component: 'appcat',
+            },
+            annotations: {
+              summary: 'Crossplane resource {{ $labels.kind }} is not synced',
+              description: 'Crossplane resource {{ $labels.name }} ({{ $labels.kind }}) in namespace {{ $labels.claim_namespace }} has status_synced={{ $labels.status_synced }} for more than 20 minutes',
+            },
+          },
+          {
+            alert: 'CrossplaneResourceNotReady',
+            expr: 'max by (api_version, kind, name, claim_name, claim_namespace, instance_name, status_synced, status_ready) (crossplane_resource_info{status_ready!="Available"}) == 1',
+            'for': '15m',
+            labels: {
+              severity: 'warning',
+              syn: 'true',
+              syn_team: 'schedar',
+              syn_component: 'appcat',
+            },
+            annotations: {
+              summary: 'Crossplane resource {{ $labels.kind }} is not ready',
+              description: 'Crossplane resource {{ $labels.name }} ({{ $labels.kind }}) in namespace {{ $labels.claim_namespace }} has status_ready={{ $labels.status_ready }} for more than 15 minutes',
+            },
+          },
+        ],
+      },
+    ],
+  },
+};
+
 local webhookService = loadManifest('webhook-service.yaml') {
   metadata+: {
     name: 'webhook-service',
@@ -310,4 +359,5 @@ if controllersParams.enabled then {
   [if controllersParams.controlPlaneKubeconfig != '' then 'controllers/appcat/10_controlplane_credentials']: controlKubeConfig,
   [if controllersParams.monitoringEnabled then 'controllers/appcat/40_service']: service,
   [if controllersParams.monitoringEnabled then 'controllers/appcat/40_servicemonitor']: servicemonitor,
+  [if controllersParams.monitoringEnabled then 'controllers/appcat/40_prometheusrule']: prometheusrule,
 } else {}

tests/golden/control-plane/appcat/appcat/controllers/appcat/30_deployment.yaml

@@ -25,10 +25,19 @@ spec:
             - --leader-elect
             - --quotas=false
             - --billing=false
-            - --crossplane-metrics=false
+            - --crossplane-metrics=true
           env:
             - name: PLANS_NAMESPACE
               value: syn-appcat
+            - name: CROSSPLANE_LABEL_MAPPING
+              value: |
+                {
+                  "crossplane.io/claim-name": "claim_name",
+                  "crossplane.io/claim-namespace": "claim_namespace",
+                  "crossplane.io/composite": "instance_name"
+                }
+            - name: CROSSPLANE_EXTRA_RESOURCES
+              value: ''
           image: ghcr.io/vshn/appcat:8af486767f62151ab7290018fb289f92d4d5e95a
           imagePullPolicy: IfNotPresent
           livenessProbe:

tests/golden/control-plane/appcat/appcat/controllers/appcat/40_prometheusrule.yaml

@@ -0,0 +1,43 @@
+apiVersion: monitoring.coreos.com/v1
+kind: PrometheusRule
+metadata:
+  annotations: {}
+  labels:
+    app.kubernetes.io/managed-by: commodore
+    app.kubernetes.io/name: appcat
+  name: appcat-crossplane-resources
+  namespace: syn-appcat
+spec:
+  groups:
+    - name: crossplane-resources.rules
+      rules:
+        - alert: CrossplaneResourceUnsynced
+          annotations:
+            description: Crossplane resource {{ $labels.name }} ({{ $labels.kind }})
+              in namespace {{ $labels.claim_namespace }} has status_synced={{ $labels.status_synced
+              }} for more than 20 minutes
+            summary: Crossplane resource {{ $labels.kind }} is not synced
+          expr: max by (api_version, kind, name, claim_name, claim_namespace, instance_name,
+            status_synced, status_ready) (crossplane_resource_info{status_synced!="ReconcileSuccess"})
+            == 1
+          for: 20m
+          labels:
+            severity: warning
+            syn: 'true'
+            syn_component: appcat
+            syn_team: schedar
+        - alert: CrossplaneResourceNotReady
+          annotations:
+            description: Crossplane resource {{ $labels.name }} ({{ $labels.kind }})
+              in namespace {{ $labels.claim_namespace }} has status_ready={{ $labels.status_ready
+              }} for more than 15 minutes
+            summary: Crossplane resource {{ $labels.kind }} is not ready
+          expr: max by (api_version, kind, name, claim_name, claim_namespace, instance_name,
+            status_synced, status_ready) (crossplane_resource_info{status_ready!="Available"})
+            == 1
+          for: 15m
+          labels:
+            severity: warning
+            syn: 'true'
+            syn_component: appcat
+            syn_team: schedar

tests/golden/control-plane/appcat/appcat/controllers/appcat/40_service.yaml

@@ -0,0 +1,15 @@
+apiVersion: v1
+kind: Service
+metadata:
+  labels:
+    appcat-controller: appcat-controller
+  name: appcat-controller
+  namespace: syn-appcat
+spec:
+  ports:
+    - name: metrics
+      port: 8080
+      protocol: TCP
+      targetPort: 8080
+  selector:
+    appcat-controller: appcat-controller

tests/golden/control-plane/appcat/appcat/controllers/appcat/40_servicemonitor.yaml

@@ -0,0 +1,12 @@
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  name: appcat
+  namespace: syn-appcat
+spec:
+  endpoints:
+    - path: /metrics
+      port: metrics
+  selector:
+    matchLabels:
+      appcat-controller: appcat-controller

tests/golden/dev/appcat/appcat/controllers/appcat/40_prometheusrule.yaml

@@ -0,0 +1,43 @@
+apiVersion: monitoring.coreos.com/v1
+kind: PrometheusRule
+metadata:
+  annotations: {}
+  labels:
+    app.kubernetes.io/managed-by: commodore
+    app.kubernetes.io/name: appcat
+  name: appcat-crossplane-resources
+  namespace: syn-appcat
+spec:
+  groups:
+    - name: crossplane-resources.rules
+      rules:
+        - alert: CrossplaneResourceUnsynced
+          annotations:
+            description: Crossplane resource {{ $labels.name }} ({{ $labels.kind }})
+              in namespace {{ $labels.claim_namespace }} has status_synced={{ $labels.status_synced
+              }} for more than 20 minutes
+            summary: Crossplane resource {{ $labels.kind }} is not synced
+          expr: max by (api_version, kind, name, claim_name, claim_namespace, instance_name,
+            status_synced, status_ready) (crossplane_resource_info{status_synced!="ReconcileSuccess"})
+            == 1
+          for: 20m
+          labels:
+            severity: warning
+            syn: 'true'
+            syn_component: appcat
+            syn_team: schedar
+        - alert: CrossplaneResourceNotReady
+          annotations:
+            description: Crossplane resource {{ $labels.name }} ({{ $labels.kind }})
+              in namespace {{ $labels.claim_namespace }} has status_ready={{ $labels.status_ready
+              }} for more than 15 minutes
+            summary: Crossplane resource {{ $labels.kind }} is not ready
+          expr: max by (api_version, kind, name, claim_name, claim_namespace, instance_name,
+            status_synced, status_ready) (crossplane_resource_info{status_ready!="Available"})
+            == 1
+          for: 15m
+          labels:
+            severity: warning
+            syn: 'true'
+            syn_component: appcat
+            syn_team: schedar

tests/golden/exodev/appcat/appcat/controllers/appcat/30_deployment.yaml

@@ -25,10 +25,19 @@ spec:
             - --leader-elect
             - --quotas=false
             - --billing=false
-            - --crossplane-metrics=false
+            - --crossplane-metrics=true
           env:
             - name: PLANS_NAMESPACE
               value: syn-appcat
+            - name: CROSSPLANE_LABEL_MAPPING
+              value: |
+                {
+                  "crossplane.io/claim-name": "claim_name",
+                  "crossplane.io/claim-namespace": "claim_namespace",
+                  "crossplane.io/composite": "instance_name"
+                }
+            - name: CROSSPLANE_EXTRA_RESOURCES
+              value: ''
           image: ghcr.io/vshn/appcat:8af486767f62151ab7290018fb289f92d4d5e95a
           imagePullPolicy: IfNotPresent
           livenessProbe:

tests/golden/exodev/appcat/appcat/controllers/appcat/40_prometheusrule.yaml

@@ -0,0 +1,43 @@
+apiVersion: monitoring.coreos.com/v1
+kind: PrometheusRule
+metadata:
+  annotations: {}
+  labels:
+    app.kubernetes.io/managed-by: commodore
+    app.kubernetes.io/name: appcat
+  name: appcat-crossplane-resources
+  namespace: syn-appcat
+spec:
+  groups:
+    - name: crossplane-resources.rules
+      rules:
+        - alert: CrossplaneResourceUnsynced
+          annotations:
+            description: Crossplane resource {{ $labels.name }} ({{ $labels.kind }})
+              in namespace {{ $labels.claim_namespace }} has status_synced={{ $labels.status_synced
+              }} for more than 20 minutes
+            summary: Crossplane resource {{ $labels.kind }} is not synced
+          expr: max by (api_version, kind, name, claim_name, claim_namespace, instance_name,
+            status_synced, status_ready) (crossplane_resource_info{status_synced!="ReconcileSuccess"})
+            == 1
+          for: 20m
+          labels:
+            severity: warning
+            syn: 'true'
+            syn_component: appcat
+            syn_team: schedar
+        - alert: CrossplaneResourceNotReady
+          annotations:
+            description: Crossplane resource {{ $labels.name }} ({{ $labels.kind }})
+              in namespace {{ $labels.claim_namespace }} has status_ready={{ $labels.status_ready
+              }} for more than 15 minutes
+            summary: Crossplane resource {{ $labels.kind }} is not ready
+          expr: max by (api_version, kind, name, claim_name, claim_namespace, instance_name,
+            status_synced, status_ready) (crossplane_resource_info{status_ready!="Available"})
+            == 1
+          for: 15m
+          labels:
+            severity: warning
+            syn: 'true'
+            syn_component: appcat
+            syn_team: schedar

tests/golden/exodev/appcat/appcat/controllers/appcat/40_service.yaml

@@ -0,0 +1,15 @@
+apiVersion: v1
+kind: Service
+metadata:
+  labels:
+    appcat-controller: appcat-controller
+  name: appcat-controller
+  namespace: syn-appcat
+spec:
+  ports:
+    - name: metrics
+      port: 8080
+      protocol: TCP
+      targetPort: 8080
+  selector:
+    appcat-controller: appcat-controller

tests/golden/exodev/appcat/appcat/controllers/appcat/40_servicemonitor.yaml

@@ -0,0 +1,12 @@
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  name: appcat
+  namespace: syn-appcat
+spec:
+  endpoints:
+    - path: /metrics
+      port: metrics
+  selector:
+    matchLabels:
+      appcat-controller: appcat-controller

tests/golden/service-cluster/appcat/appcat/controllers/appcat/30_deployment.yaml

@@ -28,12 +28,21 @@ spec:
             - --event-forwarding=false
             - --quotas=false
             - --billing=false
-            - --crossplane-metrics=false
+            - --crossplane-metrics=true
           env:
             - name: PLANS_NAMESPACE
               value: syn-appcat
             - name: CONTROL_PLANE_KUBECONFIG
               value: /config/config
+            - name: CROSSPLANE_LABEL_MAPPING
+              value: |
+                {
+                  "crossplane.io/claim-name": "claim_name",
+                  "crossplane.io/claim-namespace": "claim_namespace",
+                  "crossplane.io/composite": "instance_name"
+                }
+            - name: CROSSPLANE_EXTRA_RESOURCES
+              value: ''
           image: ghcr.io/vshn/appcat:8af486767f62151ab7290018fb289f92d4d5e95a
           imagePullPolicy: IfNotPresent
           livenessProbe:

tests/golden/service-cluster/appcat/appcat/controllers/appcat/40_prometheusrule.yaml

@@ -0,0 +1,43 @@
+apiVersion: monitoring.coreos.com/v1
+kind: PrometheusRule
+metadata:
+  annotations: {}
+  labels:
+    app.kubernetes.io/managed-by: commodore
+    app.kubernetes.io/name: appcat
+  name: appcat-crossplane-resources
+  namespace: syn-appcat
+spec:
+  groups:
+    - name: crossplane-resources.rules
+      rules:
+        - alert: CrossplaneResourceUnsynced
+          annotations:
+            description: Crossplane resource {{ $labels.name }} ({{ $labels.kind }})
+              in namespace {{ $labels.claim_namespace }} has status_synced={{ $labels.status_synced
+              }} for more than 20 minutes
+            summary: Crossplane resource {{ $labels.kind }} is not synced
+          expr: max by (api_version, kind, name, claim_name, claim_namespace, instance_name,
+            status_synced, status_ready) (crossplane_resource_info{status_synced!="ReconcileSuccess"})
+            == 1
+          for: 20m
+          labels:
+            severity: warning
+            syn: 'true'
+            syn_component: appcat
+            syn_team: schedar
+        - alert: CrossplaneResourceNotReady
+          annotations:
+            description: Crossplane resource {{ $labels.name }} ({{ $labels.kind }})
+              in namespace {{ $labels.claim_namespace }} has status_ready={{ $labels.status_ready
+              }} for more than 15 minutes
+            summary: Crossplane resource {{ $labels.kind }} is not ready
+          expr: max by (api_version, kind, name, claim_name, claim_namespace, instance_name,
+            status_synced, status_ready) (crossplane_resource_info{status_ready!="Available"})
+            == 1
+          for: 15m
+          labels:
+            severity: warning
+            syn: 'true'
+            syn_component: appcat
+            syn_team: schedar

tests/golden/service-cluster/appcat/appcat/controllers/appcat/40_service.yaml

@@ -0,0 +1,15 @@
+apiVersion: v1
+kind: Service
+metadata:
+  labels:
+    appcat-controller: appcat-controller
+  name: appcat-controller
+  namespace: syn-appcat
+spec:
+  ports:
+    - name: metrics
+      port: 8080
+      protocol: TCP
+      targetPort: 8080
+  selector:
+    appcat-controller: appcat-controller

tests/golden/service-cluster/appcat/appcat/controllers/appcat/40_servicemonitor.yaml

@@ -0,0 +1,12 @@
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  name: appcat
+  namespace: syn-appcat
+spec:
+  endpoints:
+    - path: /metrics
+      port: metrics
+  selector:
+    matchLabels:
+      appcat-controller: appcat-controller