BREAK-8: add promErr attribute to CircuitBreaker class

scality · Jun 19, 2024 · e4c98b6 · e4c98b6
1 parent cdfacd2
commit e4c98b6
Show file tree

Hide file tree

Showing 3 changed files with 19 additions and 8 deletions.
diff --git a/src/CircuitBreaker.ts b/src/CircuitBreaker.ts
@@ -24,12 +24,14 @@ export class CircuitBreaker extends EventEmitter {
     // a promise that unit tests can await on, as jest's fake timers
     // get confused with async flows
     _evaluatingPromiseHook: Promise<unknown> | null;
+    _promError: boolean;
 
     constructor(config: unknown) {
         super();
 
         this._config = validate(config);
         this._probes = (this._config.probes || []).map(buildProbe);
+        this._promError = false;
 
         this._aggregateState = BreakerState.Nominal;
         this._stabilizingCounter = 0;
@@ -49,6 +51,10 @@ export class CircuitBreaker extends EventEmitter {
         return this._aggregateState;
     }
 
+    get promErr(): boolean {
+        return this._promError;
+    }
+
     start() {
         this._aggregateState = BreakerState.Nominal;
         this._scheduleNextEvaluation();
@@ -62,12 +68,17 @@ export class CircuitBreaker extends EventEmitter {
     }
 
     async _evaluate() {
-        this._evaluatingPromiseHook = Promise.allSettled(this._probes.map(p => p.check()));
+        this._promError = false;
+        this._evaluatingPromiseHook = Promise.allSettled(this._probes.map(p => p.check().catch(error => {
+            if (error instanceof Error && error.message === 'Failed to query Prometheus') {
+                this._promError = true;
+            }
+            return 'failed';
+        })));
         await this._evaluatingPromiseHook;
         this._evaluatingPromiseHook = null;
 
         const allOk = this._probes.every(probe => probe.value);
-
         const initialState = this._aggregateState;
 
         if (allOk) {
@@ -84,14 +95,12 @@ export class CircuitBreaker extends EventEmitter {
                     this._aggregateState = BreakerState.Nominal;
                 }
                 break;
-
             default:
                 break;
             }
         } else {
             this._aggregateState = BreakerState.Tripped;
         }
-
         if (initialState !== this._aggregateState) {
             process.nextTick(() => {
                 this.emit('state-changed', this._aggregateState);

diff --git a/src/probes/PrometheusClient.ts b/src/probes/PrometheusClient.ts
@@ -37,7 +37,7 @@ export class PrometheusClient {
             `sum(${q})`;
     }
 
-    async instantQuery(): Promise<number | null> {
+    async instantQuery(): Promise<number | boolean | null> {
         try {
             const res = await this.prom.instantQuery(this.q);
 
@@ -47,11 +47,10 @@ export class PrometheusClient {
             if (series.length == 0) {
                 return null;
             }
-
             return series[0].value.value;
         } catch (error: unknown) {
             log.error('unable to query prometheus', { error });
-            return null;
+            return false;
         }
     }
 }
diff --git a/src/probes/PrometheusQueryProbe.ts b/src/probes/PrometheusQueryProbe.ts
@@ -36,13 +36,16 @@ export class PrometheusQueryProbe implements Probe {
 
     async check() {
         const v = await this.prometheusClient.instantQuery();
-        if (v != null) {
+        if (v != null && typeof(v) !== 'boolean') {
             if (Number.isNaN(v)) {
                 log.warn('warning: ignoring received NaN value (interval too long for retention?)');
                 return;
             }
             this.observed = v;
         }
+        if (v === false) {
+            throw new Error('Failed to query Prometheus');
+        }
     }
 
     get value() {