From 9f0e30c06ba13f7c63d2d9ec202a5ba36425cd4c Mon Sep 17 00:00:00 2001
From: DeshDeepakKant <deshdeepakkant@gmail.com>
Date: Tue, 21 Jan 2025 22:17:05 +0530
Subject: [PATCH 1/4] docs(circuit-breaker): Add comprehensive KMesh Circuit
 Breaker user guide

- Implement detailed documentation for Circuit Breaker feature
- Include technical implementation details
- Provide configuration examples and best practices
- Cover troubleshooting and monitoring aspects

Signed-off-by: Desh Deepak Kant <deshdeepakkant@gmail.com>
Closes #103
Signed-off-by: DeshDeepakKant <deshdeepakkant@gmail.com>
---
 content/en/docs/userguide/circuit_breaker.md | 253 +++++++++++++++++++
 1 file changed, 253 insertions(+)
 create mode 100644 content/en/docs/userguide/circuit_breaker.md

diff --git a/content/en/docs/userguide/circuit_breaker.md b/content/en/docs/userguide/circuit_breaker.md
new file mode 100644
index 0000000..1359d51
--- /dev/null
+++ b/content/en/docs/userguide/circuit_breaker.md
@@ -0,0 +1,253 @@
+---
+draft: false
+linktitle:  Circuit Breaker
+menu:
+  docs:
+    parent: user guide
+    weight: 22
+title: Use Circuit Breaker 
+toc: true
+type: docs
+---
+
+### Preparation
+
+1. Ensure KMesh is installed in your Kubernetes cluster (see [Quick Start Guide](https://kmesh.net/en/docs/setup/quickstart/))
+
+2. Deploy a sample microservice application
+
+3. Verify the default namespace is managed by KMesh
+
+### Circuit Breaker Configuration
+
+##### Deploy a Sample Application
+
+Let's use a simple microservice setup to demonstrate circuit breaking:
+
+```bash
+kubectl apply -f - <<EOF
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: sample-service
+spec:
+  replicas: 2
+  selector:
+    matchLabels:
+      app: sample-service
+  template:
+    metadata:
+      labels:
+        app: sample-service
+    spec:
+      containers:
+      - name: service
+        image: your-service-image
+        ports:
+        - containerPort: 8080
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: sample-service
+spec:
+  selector:
+    app: sample-service
+  ports:
+  - port: 80
+    targetPort: 8080
+EOF
+```
+
+##### Apply Circuit Breaker Configuration
+
+Configure circuit breaker to limit connections and requests:
+
+```bash
+kubectl apply -f - <<EOF
+apiVersion: kmesh.net/v1alpha1
+kind: CircuitBreaker
+metadata:
+  name: sample-service-circuit-breaker
+spec:
+  service: sample-service
+  rules:
+    - priority: HIGH
+      maxConnections: 100
+      maxPendingRequests: 50
+      maxRequests: 200
+      maxRetries: 3
+EOF
+```
+
+### Verify Circuit Breaker Functionality
+
+Use a load testing tool to simulate traffic and observe circuit breaker behavior:
+
+```bash
+# Install hey load testing tool
+go install github.com/rakyll/hey@latest
+
+# Generate load
+hey -n 1000 -c 50 http://sample-service/endpoint
+```
+
+### Monitoring Circuit Breaker Metrics
+
+Check circuit breaker metrics and logs:
+
+```bash
+# View KMesh circuit breaker logs
+kubectl logs -n kmesh -l app=kmesh circuit-breaker
+
+# Get circuit breaker statistics
+kmesh get circuit-breaker-stats sample-service
+```
+
+### Understanding the Circuit Breaker Mechanism
+
+When the circuit breaker is activated:
+- New connections are rejected
+- Existing connections are maintained
+- Service is protected from overload
+
+### Circuit Breaker Configuration Options
+
+| Parameter           | Description                          | Default    |
+|--------------------|--------------------------------------|------------|
+| priority           | Request routing priority             | DEFAULT    |
+| maxConnections     | Maximum concurrent connections       | Unlimited  |
+| maxPendingRequests | Maximum pending requests in queue    | Unlimited  |
+| maxRequests        | Maximum concurrent requests          | Unlimited  |
+| maxRetries         | Maximum retry attempts               | Unlimited  |
+
+### Technical Implementation Details
+
+##### Data Structures
+
+Circuit Breaker configuration in Protocol Buffers:
+
+```protobuf
+message CircuitBreakers {
+  RoutingPriority priority = 1;
+  uint32 max_connections = 2;
+  uint32 max_pending_requests = 3;
+  uint32 max_requests = 4;
+  uint32 max_retries = 5;
+  uint32 max_connection_pools = 7;
+}
+```
+
+Connection Tracking Structure:
+
+```c
+struct cluster_stats {
+    __u32 active_connections;
+};
+
+struct cluster_stats_key {
+    __u64 netns_cookie;
+    __u32 cluster_id;
+};
+```
+
+##### Connection Management Workflow
+
+Connection Binding Logic:
+
+```c
+static inline int on_cluster_sock_bind(
+    ctx_buff_t *ctx, 
+    const Cluster__Cluster *cluster
+) {
+    // Check if connection limits are exceeded
+    if (stats->active_connections >= cbs->max_connections) {
+        // Reject connection
+        return -1;
+    }
+    // Bind socket to cluster
+    return 0;
+}
+```
+
+### Advanced Configuration Example
+
+```yaml
+apiVersion: kmesh.net/v1alpha1
+kind: CircuitBreaker
+metadata:
+  name: complex-service-circuit-breaker
+spec:
+  services:
+    - name: service-a
+      rules:
+        - priority: HIGH
+          maxConnections: 50
+    - name: service-b
+      rules:
+        - priority: MEDIUM
+          maxConnections: 100
+```
+
+### Troubleshooting
+
+##### Common Issues
+- Unexpected connection rejections
+- High error rates
+- Performance degradation
+
+##### Debugging Steps
+
+```bash
+# Check circuit breaker configuration
+kubectl describe circuitbreaker sample-service-circuit-breaker
+
+# View detailed logs
+kubectl logs -n kmesh -l app=kmesh circuit-breaker -c circuit-breaker
+
+# Check cluster status
+kmesh get clusters
+```
+
+### Best Practices
+
+1. Start with conservative limits
+2. Gradually adjust based on service performance
+3. Monitor circuit breaker metrics
+4. Use priority-based configurations
+
+### Performance Considerations
+
+- Kernel-native implementation
+- Minimal overhead
+- Lock-free atomic updates
+- eBPF map-based tracking
+
+### Cleanup
+
+Remove the circuit breaker and sample application:
+
+```bash
+kubectl delete circuitbreaker sample-service-circuit-breaker
+kubectl delete deployment sample-service
+kubectl delete service sample-service
+```
+
+### Limitations
+
+- Currently focuses on TCP connections
+- Kernel version dependencies
+- Per-cluster granularity
+
+### Sample Code Snippet
+
+```go
+// Circuit Breaker Configuration Example
+circuitBreaker := &CircuitBreakers{
+    Priority:           RoutingPriority_HIGH,
+    MaxConnections:     100,
+    MaxPendingRequests: 50,
+    MaxRequests:        200,
+    MaxRetries:         3,
+}
+```
\ No newline at end of file

From 27bda7ecfe7e81f766c8af46284d2653847127ae Mon Sep 17 00:00:00 2001
From: DeshDeepakKant <deshdeepakkant@gmail.com>
Date: Thu, 23 Jan 2025 19:27:07 +0530
Subject: [PATCH 2/4] docs: add circuit breaker user guide with Fortio testing

Signed-off-by: DeshDeepakKant <deshdeepakkant@gmail.com>
---
 content/en/docs/userguide/circuit_breaker.md | 311 ++++++++-----------
 1 file changed, 137 insertions(+), 174 deletions(-)

diff --git a/content/en/docs/userguide/circuit_breaker.md b/content/en/docs/userguide/circuit_breaker.md
index 1359d51..ea883df 100644
--- a/content/en/docs/userguide/circuit_breaker.md
+++ b/content/en/docs/userguide/circuit_breaker.md
@@ -1,253 +1,216 @@
 ---
 draft: false
-linktitle:  Circuit Breaker
+linktitle: Circuit Breaker
 menu:
   docs:
     parent: user guide
     weight: 22
-title: Use Circuit Breaker 
+title: Circuit Breaker
 toc: true
 type: docs
 ---
 
-### Preparation
+This task shows you how to configure circuit breakers in KMesh using Fortio for load testing.
 
-1. Ensure KMesh is installed in your Kubernetes cluster (see [Quick Start Guide](https://kmesh.net/en/docs/setup/quickstart/))
+### Before you begin
 
-2. Deploy a sample microservice application
+- Install KMesh
+  Please refer [quickstart](https://kmesh.net/en/docs/setup/quickstart/) and change into ads mode
 
-3. Verify the default namespace is managed by KMesh
 
-### Circuit Breaker Configuration
+### Deploy the test applications
 
-##### Deploy a Sample Application
-
-Let's use a simple microservice setup to demonstrate circuit breaking:
-
-```bash
-kubectl apply -f - <<EOF
+1. Create the test service:
+```yaml
+# sample-app.yaml
 apiVersion: apps/v1
 kind: Deployment
 metadata:
-  name: sample-service
+  name: test-service
 spec:
-  replicas: 2
+  replicas: 1
   selector:
     matchLabels:
-      app: sample-service
+      app: test-service
   template:
     metadata:
       labels:
-        app: sample-service
+        app: test-service
     spec:
       containers:
-      - name: service
-        image: your-service-image
+      - name: test-service
+        image: nginx:latest
         ports:
-        - containerPort: 8080
+        - containerPort: 80
 ---
 apiVersion: v1
 kind: Service
 metadata:
-  name: sample-service
+  name: test-service
 spec:
   selector:
-    app: sample-service
+    app: test-service
   ports:
   - port: 80
-    targetPort: 8080
-EOF
 ```
 
-##### Apply Circuit Breaker Configuration
-
-Configure circuit breaker to limit connections and requests:
-
-```bash
-kubectl apply -f - <<EOF
-apiVersion: kmesh.net/v1alpha1
-kind: CircuitBreaker
+2. Deploy Fortio load tester:
+```yaml
+# fortio.yaml
+apiVersion: apps/v1
+kind: Deployment
 metadata:
-  name: sample-service-circuit-breaker
+  name: fortio
 spec:
-  service: sample-service
-  rules:
-    - priority: HIGH
-      maxConnections: 100
-      maxPendingRequests: 50
-      maxRequests: 200
-      maxRetries: 3
-EOF
+  replicas: 1
+  selector:
+    matchLabels:
+      app: fortio
+  template:
+    metadata:
+      labels:
+        app: fortio
+    spec:
+      containers:
+      - name: fortio
+        image: fortio/fortio
+        ports:
+        - containerPort: 8080
 ```
 
-### Verify Circuit Breaker Functionality
-
-Use a load testing tool to simulate traffic and observe circuit breaker behavior:
+### Configure circuit breaker
 
-```bash
-# Install hey load testing tool
-go install github.com/rakyll/hey@latest
-
-# Generate load
-hey -n 1000 -c 50 http://sample-service/endpoint
+Apply the circuit breaker configuration:
+```yaml
+# circuit-breaker.yaml
+apiVersion: networking.istio.io/v1alpha3
+kind: DestinationRule
+metadata:
+  name: test-circuit-breaker
+spec:
+  host: test-service
+  trafficPolicy:
+    connectionPool:
+      http:
+        http1MaxPendingRequests: 1
+        maxRequestsPerConnection: 1
+    outlierDetection:
+      consecutive5xxErrors: 3
+      interval: 5s
+      baseEjectionTime: 30s
 ```
 
-### Monitoring Circuit Breaker Metrics
-
-Check circuit breaker metrics and logs:
-
+Apply the configurations:
 ```bash
-# View KMesh circuit breaker logs
-kubectl logs -n kmesh -l app=kmesh circuit-breaker
-
-# Get circuit breaker statistics
-kmesh get circuit-breaker-stats sample-service
+kubectl apply -f sample-app.yaml
+kubectl apply -f fortio.yaml
+kubectl apply -f circuit-breaker.yaml
 ```
 
-### Understanding the Circuit Breaker Mechanism
-
-When the circuit breaker is activated:
-- New connections are rejected
-- Existing connections are maintained
-- Service is protected from overload
-
-### Circuit Breaker Configuration Options
 
-| Parameter           | Description                          | Default    |
-|--------------------|--------------------------------------|------------|
-| priority           | Request routing priority             | DEFAULT    |
-| maxConnections     | Maximum concurrent connections       | Unlimited  |
-| maxPendingRequests | Maximum pending requests in queue    | Unlimited  |
-| maxRequests        | Maximum concurrent requests          | Unlimited  |
-| maxRetries         | Maximum retry attempts               | Unlimited  |
 
-### Technical Implementation Details
+### Monitor circuit breaker behavior
 
-##### Data Structures
-
-Circuit Breaker configuration in Protocol Buffers:
+Monitor metrics:
+```bash
+# View Fortio metrics
+kubectl logs deploy/fortio
 
-```protobuf
-message CircuitBreakers {
-  RoutingPriority priority = 1;
-  uint32 max_connections = 2;
-  uint32 max_pending_requests = 3;
-  uint32 max_requests = 4;
-  uint32 max_retries = 5;
-  uint32 max_connection_pools = 7;
-}
+# Check service status
+kubectl get pods -w
 ```
 
-Connection Tracking Structure:
-
-```c
-struct cluster_stats {
-    __u32 active_connections;
-};
+### Testing Scenarios
 
-struct cluster_stats_key {
-    __u64 netns_cookie;
-    __u32 cluster_id;
-};
+#### 4.1 Basic Load Test
+```bash
+# Normal load
+kubectl exec -it deploy/fortio -- \
+fortio load -c 1 -qps 10 -t 30s http://test-service
 ```
 
-##### Connection Management Workflow
-
-Connection Binding Logic:
-
-```c
-static inline int on_cluster_sock_bind(
-    ctx_buff_t *ctx, 
-    const Cluster__Cluster *cluster
-) {
-    // Check if connection limits are exceeded
-    if (stats->active_connections >= cbs->max_connections) {
-        // Reject connection
-        return -1;
-    }
-    // Bind socket to cluster
-    return 0;
-}
-```
+#### 4.2 Circuit Breaker Test
+```bash
+# Heavy load to trigger circuit breaker
+kubectl exec -it deploy/fortio -- \
+fortio load -c 5 -qps 100 -t 30s http://test-service
 
-### Advanced Configuration Example
+# Verify circuit breaker status
+kubectl get destinationrule test-circuit-breaker -o yaml
 
-```yaml
-apiVersion: kmesh.net/v1alpha1
-kind: CircuitBreaker
-metadata:
-  name: complex-service-circuit-breaker
-spec:
-  services:
-    - name: service-a
-      rules:
-        - priority: HIGH
-          maxConnections: 50
-    - name: service-b
-      rules:
-        - priority: MEDIUM
-          maxConnections: 100
+# Simulate service failure
+kubectl scale deployment test-service --replicas=0
 ```
+#### 4.3 Recovery Test
+```bash
+# Restore service
+kubectl scale deployment test-service --replicas=1
 
-### Troubleshooting
-
-##### Common Issues
-- Unexpected connection rejections
-- High error rates
-- Performance degradation
+# Test recovery
+kubectl exec -it deploy/fortio -- \
+fortio load -c 2 -qps 20 -t 30s http://test-service
+```
 
-##### Debugging Steps
+### Analyzing Results
 
+#### 6.1 Fortio Results
 ```bash
-# Check circuit breaker configuration
-kubectl describe circuitbreaker sample-service-circuit-breaker
+# View test results
+kubectl logs deploy/fortio
 
-# View detailed logs
-kubectl logs -n kmesh -l app=kmesh circuit-breaker -c circuit-breaker
-
-# Check cluster status
-kmesh get clusters
+# Get detailed metrics
+kubectl exec -it deploy/fortio -- /usr/bin/fortio report
 ```
 
-### Best Practices
+#### 6.2 System Metrics
+```bash
+# Check pod status
+kubectl get pods -w
 
-1. Start with conservative limits
-2. Gradually adjust based on service performance
-3. Monitor circuit breaker metrics
-4. Use priority-based configurations
+# View circuit breaker configuration
+kubectl get destinationrule test-circuit-breaker -o yaml
+```
 
-### Performance Considerations
+### Understanding what happened
 
-- Kernel-native implementation
-- Minimal overhead
-- Lock-free atomic updates
-- eBPF map-based tracking
+The circuit breaker configuration:
+- Limits concurrent HTTP requests
+- Ejects hosts after 3 consecutive errors
+- Keeps circuit open for 30 seconds
+- Monitors service health every 5 seconds
 
-### Cleanup
+When the service is overloaded:
+1. Circuit breaker trips after threshold breach
+2. Subsequent requests are blocked
+3. Service recovers after baseEjectionTime
+4. Normal traffic flow resumes
 
-Remove the circuit breaker and sample application:
+### Clean up
 
+Remove test components:
 ```bash
-kubectl delete circuitbreaker sample-service-circuit-breaker
-kubectl delete deployment sample-service
-kubectl delete service sample-service
+kubectl delete -f sample-app.yaml
+kubectl delete -f fortio.yaml
+kubectl delete -f circuit-breaker.yaml
 ```
 
-### Limitations
+### Troubleshooting
 
-- Currently focuses on TCP connections
-- Kernel version dependencies
-- Per-cluster granularity
+If you encounter issues:
+1. Check pod status:
+```bash
+kubectl get pods
+kubectl describe pod <pod-name>
+```
 
-### Sample Code Snippet
+2. Check circuit breaker configuration:
+```bash
+kubectl get destinationrule
+kubectl describe destinationrule test-circuit-breaker
+```
 
-```go
-// Circuit Breaker Configuration Example
-circuitBreaker := &CircuitBreakers{
-    Priority:           RoutingPriority_HIGH,
-    MaxConnections:     100,
-    MaxPendingRequests: 50,
-    MaxRequests:        200,
-    MaxRetries:         3,
-}
+3. View application logs:
+```bash
+kubectl logs deploy/test-service
+kubectl logs deploy/fortio
 ```
\ No newline at end of file

From 6bea74657cfbcd45141ec5484b1f2345f28a1cc3 Mon Sep 17 00:00:00 2001
From: DeshDeepakKant <deshdeepakkant@gmail.com>
Date: Fri, 24 Jan 2025 03:50:54 +0530
Subject: [PATCH 3/4] updated circuit-breaker.yaml

Signed-off-by: DeshDeepakKant <deshdeepakkant@gmail.com>
---
 content/en/docs/userguide/circuit_breaker.md | 21 +++++++++-----------
 1 file changed, 9 insertions(+), 12 deletions(-)

diff --git a/content/en/docs/userguide/circuit_breaker.md b/content/en/docs/userguide/circuit_breaker.md
index ea883df..0d7475d 100644
--- a/content/en/docs/userguide/circuit_breaker.md
+++ b/content/en/docs/userguide/circuit_breaker.md
@@ -83,21 +83,18 @@ spec:
 Apply the circuit breaker configuration:
 ```yaml
 # circuit-breaker.yaml
-apiVersion: networking.istio.io/v1alpha3
-kind: DestinationRule
+apiVersion: kmesh.net/v1alpha1
+kind: CircuitBreaker
 metadata:
   name: test-circuit-breaker
 spec:
-  host: test-service
-  trafficPolicy:
-    connectionPool:
-      http:
-        http1MaxPendingRequests: 1
-        maxRequestsPerConnection: 1
-    outlierDetection:
-      consecutive5xxErrors: 3
-      interval: 5s
-      baseEjectionTime: 30s
+  service: test-service
+  rules:
+    - priority: HIGH
+      maxConnections: 10
+      maxPendingRequests: 5
+      maxRequests: 20
+      maxRetries: 3
 ```
 
 Apply the configurations:

From 0a17ca54bee2ffbe9e634a5b9f623a9dc0182ec7 Mon Sep 17 00:00:00 2001
From: DeshDeepakKant <deshdeepakkant@gmail.com>
Date: Sun, 9 Feb 2025 01:08:06 +0530
Subject: [PATCH 4/4] Fix: Update circuit breaker documentation (removed
 rate-limit.md)

Signed-off-by: DeshDeepakKant <deshdeepakkant@gmail.com>
---
 content/en/docs/userguide/circuit_breaker.md | 18 +++++++++++++++++-
 1 file changed, 17 insertions(+), 1 deletion(-)

diff --git a/content/en/docs/userguide/circuit_breaker.md b/content/en/docs/userguide/circuit_breaker.md
index 0d7475d..9c437df 100644
--- a/content/en/docs/userguide/circuit_breaker.md
+++ b/content/en/docs/userguide/circuit_breaker.md
@@ -148,6 +148,22 @@ kubectl exec -it deploy/fortio -- \
 fortio load -c 2 -qps 20 -t 30s http://test-service
 ```
 
+### Sample Test Results
+
+#### Without Circuit Breaker
+```
+IP addresses distribution: 10.96.230.153:80: 5 
+Code 200 : 3000 (100.0 %)
+```
+
+#### With Circuit Breaker
+```
+IP addresses distribution: 10.96.230.153:80: 5 
+Code 200 : 1914 (63.8%) 
+Code 503 : 1086 (36.2%)
+```
+
+
 ### Analyzing Results
 
 #### 6.1 Fortio Results
@@ -210,4 +226,4 @@ kubectl describe destinationrule test-circuit-breaker
 ```bash
 kubectl logs deploy/test-service
 kubectl logs deploy/fortio
-```
\ No newline at end of file
+```