33
33
34
34
# Logging Message Constants
35
35
LOG_CIRCUIT_BREAKER_STATE_CHANGED = "Circuit breaker state changed from %s to %s for %s"
36
- LOG_CIRCUIT_BREAKER_OPENED = "Circuit breaker opened for %s - telemetry requests will be blocked"
37
- LOG_CIRCUIT_BREAKER_CLOSED = "Circuit breaker closed for %s - telemetry requests will be allowed"
38
- LOG_CIRCUIT_BREAKER_HALF_OPEN = "Circuit breaker half-open for %s - testing telemetry requests"
36
+ LOG_CIRCUIT_BREAKER_OPENED = (
37
+ "Circuit breaker opened for %s - telemetry requests will be blocked"
38
+ )
39
+ LOG_CIRCUIT_BREAKER_CLOSED = (
40
+ "Circuit breaker closed for %s - telemetry requests will be allowed"
41
+ )
42
+ LOG_CIRCUIT_BREAKER_HALF_OPEN = (
43
+ "Circuit breaker half-open for %s - testing telemetry requests"
44
+ )
39
45
40
46
41
47
class CircuitBreakerStateListener (CircuitBreakerListener ):
42
48
"""Listener for circuit breaker state changes."""
43
-
49
+
44
50
def before_call (self , cb : CircuitBreaker , func , * args , ** kwargs ) -> None :
45
51
"""Called before the circuit breaker calls a function."""
46
52
pass
47
-
53
+
48
54
def failure (self , cb : CircuitBreaker , exc : BaseException ) -> None :
49
55
"""Called when a function called by the circuit breaker fails."""
50
56
pass
51
-
57
+
52
58
def success (self , cb : CircuitBreaker ) -> None :
53
59
"""Called when a function called by the circuit breaker succeeds."""
54
60
pass
55
-
61
+
56
62
def state_change (self , cb : CircuitBreaker , old_state , new_state ) -> None :
57
63
"""Called when the circuit breaker state changes."""
58
64
old_state_name = old_state .name if old_state else "None"
59
65
new_state_name = new_state .name if new_state else "None"
60
-
66
+
61
67
logger .info (
62
- LOG_CIRCUIT_BREAKER_STATE_CHANGED ,
63
- old_state_name , new_state_name , cb .name
68
+ LOG_CIRCUIT_BREAKER_STATE_CHANGED , old_state_name , new_state_name , cb .name
64
69
)
65
-
70
+
66
71
if new_state_name == CIRCUIT_BREAKER_STATE_OPEN :
67
- logger .warning (
68
- LOG_CIRCUIT_BREAKER_OPENED ,
69
- cb .name
70
- )
72
+ logger .warning (LOG_CIRCUIT_BREAKER_OPENED , cb .name )
71
73
elif new_state_name == CIRCUIT_BREAKER_STATE_CLOSED :
72
- logger .info (
73
- LOG_CIRCUIT_BREAKER_CLOSED ,
74
- cb .name
75
- )
74
+ logger .info (LOG_CIRCUIT_BREAKER_CLOSED , cb .name )
76
75
elif new_state_name == CIRCUIT_BREAKER_STATE_HALF_OPEN :
77
- logger .info (
78
- LOG_CIRCUIT_BREAKER_HALF_OPEN ,
79
- cb .name
80
- )
76
+ logger .info (LOG_CIRCUIT_BREAKER_HALF_OPEN , cb .name )
81
77
82
78
83
79
@dataclass (frozen = True )
84
80
class CircuitBreakerConfig :
85
81
"""Configuration for circuit breaker behavior.
86
-
82
+
87
83
This class is immutable to prevent modification of circuit breaker settings.
88
84
All configuration values are set to constants defined at the module level.
89
85
"""
90
-
86
+
91
87
# Failure threshold percentage (0.0 to 1.0)
92
88
failure_threshold : float = DEFAULT_FAILURE_THRESHOLD
93
-
89
+
94
90
# Minimum number of calls before circuit can open
95
91
minimum_calls : int = DEFAULT_MINIMUM_CALLS
96
-
92
+
97
93
# Time window for counting failures (in seconds)
98
94
timeout : int = DEFAULT_TIMEOUT
99
-
95
+
100
96
# Time to wait before trying to close circuit (in seconds)
101
97
reset_timeout : int = DEFAULT_RESET_TIMEOUT
102
-
98
+
103
99
# Expected exception types that should trigger circuit breaker
104
100
expected_exception : tuple = DEFAULT_EXPECTED_EXCEPTION
105
-
101
+
106
102
# Name for the circuit breaker (for logging)
107
103
name : str = DEFAULT_NAME
108
104
109
105
110
106
class CircuitBreakerManager :
111
107
"""
112
108
Manages circuit breaker instances for telemetry requests.
113
-
109
+
114
110
This class provides a singleton pattern to manage circuit breaker instances
115
111
per host, ensuring that telemetry failures don't impact main SQL operations.
116
112
"""
117
-
113
+
118
114
_instances : Dict [str , CircuitBreaker ] = {}
119
115
_lock = threading .RLock ()
120
116
_config : Optional [CircuitBreakerConfig ] = None
121
-
117
+
122
118
@classmethod
123
119
def initialize (cls , config : CircuitBreakerConfig ) -> None :
124
120
"""
125
121
Initialize the circuit breaker manager with configuration.
126
-
122
+
127
123
Args:
128
124
config: Circuit breaker configuration
129
125
"""
130
126
with cls ._lock :
131
127
cls ._config = config
132
128
logger .debug ("CircuitBreakerManager initialized with config: %s" , config )
133
-
129
+
134
130
@classmethod
135
131
def get_circuit_breaker (cls , host : str ) -> CircuitBreaker :
136
132
"""
137
133
Get or create a circuit breaker instance for the specified host.
138
-
134
+
139
135
Args:
140
136
host: The hostname for which to get the circuit breaker
141
-
137
+
142
138
Returns:
143
139
CircuitBreaker instance for the host
144
140
"""
145
141
if not cls ._config :
146
142
# Return a no-op circuit breaker if not initialized
147
143
return cls ._create_noop_circuit_breaker ()
148
-
144
+
149
145
with cls ._lock :
150
146
if host not in cls ._instances :
151
147
cls ._instances [host ] = cls ._create_circuit_breaker (host )
152
148
logger .debug ("Created circuit breaker for host: %s" , host )
153
-
149
+
154
150
return cls ._instances [host ]
155
-
151
+
156
152
@classmethod
157
153
def _create_circuit_breaker (cls , host : str ) -> CircuitBreaker :
158
154
"""
159
155
Create a new circuit breaker instance for the specified host.
160
-
156
+
161
157
Args:
162
158
host: The hostname for the circuit breaker
163
-
159
+
164
160
Returns:
165
161
New CircuitBreaker instance
166
162
"""
167
163
config = cls ._config
168
-
164
+ if config is None :
165
+ raise RuntimeError ("CircuitBreakerManager not initialized" )
166
+
169
167
# Create circuit breaker with configuration
170
168
breaker = CircuitBreaker (
171
169
fail_max = config .minimum_calls , # Number of failures before circuit opens
172
170
reset_timeout = config .reset_timeout ,
173
- name = f"{ config .name } -{ host } "
171
+ name = f"{ config .name } -{ host } " ,
174
172
)
175
-
173
+
176
174
# Add state change listeners for logging
177
175
breaker .add_listener (CircuitBreakerStateListener ())
178
-
176
+
179
177
return breaker
180
-
178
+
181
179
@classmethod
182
180
def _create_noop_circuit_breaker (cls ) -> CircuitBreaker :
183
181
"""
184
182
Create a no-op circuit breaker that always allows calls.
185
-
183
+
186
184
Returns:
187
185
CircuitBreaker that never opens
188
186
"""
189
187
# Create a circuit breaker with very high thresholds so it never opens
190
188
breaker = CircuitBreaker (
191
189
fail_max = 1000000 , # Very high threshold
192
- reset_timeout = 1 , # Short reset time
193
- name = "noop-circuit-breaker"
190
+ reset_timeout = 1 , # Short reset time
191
+ name = "noop-circuit-breaker" ,
194
192
)
195
- breaker .failure_threshold = 1.0 # 100% failure threshold
196
193
return breaker
197
-
198
-
194
+
199
195
@classmethod
200
196
def get_circuit_breaker_state (cls , host : str ) -> str :
201
197
"""
202
198
Get the current state of the circuit breaker for a host.
203
-
199
+
204
200
Args:
205
201
host: The hostname
206
-
202
+
207
203
Returns:
208
204
Current state of the circuit breaker
209
205
"""
210
206
if not cls ._config :
211
207
return CIRCUIT_BREAKER_STATE_DISABLED
212
-
208
+
213
209
with cls ._lock :
214
210
if host not in cls ._instances :
215
211
return CIRCUIT_BREAKER_STATE_NOT_INITIALIZED
216
-
212
+
217
213
breaker = cls ._instances [host ]
218
214
return breaker .current_state
219
-
215
+
220
216
@classmethod
221
217
def reset_circuit_breaker (cls , host : str ) -> None :
222
218
"""
223
219
Reset the circuit breaker for a host to closed state.
224
-
220
+
225
221
Args:
226
222
host: The hostname
227
223
"""
@@ -230,20 +226,20 @@ def reset_circuit_breaker(cls, host: str) -> None:
230
226
# pybreaker doesn't have a reset method, we need to recreate the breaker
231
227
del cls ._instances [host ]
232
228
logger .info ("Reset circuit breaker for host: %s" , host )
233
-
229
+
234
230
@classmethod
235
231
def clear_circuit_breaker (cls , host : str ) -> None :
236
232
"""
237
233
Remove the circuit breaker instance for a host.
238
-
234
+
239
235
Args:
240
236
host: The hostname
241
237
"""
242
238
with cls ._lock :
243
239
if host in cls ._instances :
244
240
del cls ._instances [host ]
245
241
logger .debug ("Cleared circuit breaker for host: %s" , host )
246
-
242
+
247
243
@classmethod
248
244
def clear_all_circuit_breakers (cls ) -> None :
249
245
"""Clear all circuit breaker instances."""
@@ -255,10 +251,10 @@ def clear_all_circuit_breakers(cls) -> None:
255
251
def is_circuit_breaker_error (exception : Exception ) -> bool :
256
252
"""
257
253
Check if an exception is a circuit breaker error.
258
-
254
+
259
255
Args:
260
256
exception: The exception to check
261
-
257
+
262
258
Returns:
263
259
True if the exception is a circuit breaker error
264
260
"""
0 commit comments