From 104a5ea735942829a994be20c1b1b713028484a1 Mon Sep 17 00:00:00 2001 From: Kaushik Raina Date: Tue, 7 Oct 2025 20:12:53 +0530 Subject: [PATCH 1/6] Set performance bounds --- tests/ducktape/producer_benchmark_bounds.json | 8 ++++---- tests/ducktape/producer_strategy.py | 10 ++++++++++ 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/tests/ducktape/producer_benchmark_bounds.json b/tests/ducktape/producer_benchmark_bounds.json index d0fbdb44e..ec3031047 100644 --- a/tests/ducktape/producer_benchmark_bounds.json +++ b/tests/ducktape/producer_benchmark_bounds.json @@ -3,10 +3,10 @@ "local": { "_comment": "Default bounds for local development - more relaxed thresholds", "min_throughput_msg_per_sec": 1000.0, - "max_p95_latency_ms": 2000.0, + "max_p95_latency_ms": 6000.0, "max_error_rate": 0.02, "min_success_rate": 0.98, - "max_p99_latency_ms": 3000.0, + "max_p99_latency_ms": 7000.0, "max_memory_growth_mb": 800.0, "max_buffer_full_rate": 0.05, "min_messages_per_poll": 10.0 @@ -14,10 +14,10 @@ "ci": { "_comment": "Stricter bounds for CI environment - production-like requirements", "min_throughput_msg_per_sec": 1500.0, - "max_p95_latency_ms": 1500.0, + "max_p95_latency_ms": 5500.0, "max_error_rate": 0.01, "min_success_rate": 0.99, - "max_p99_latency_ms": 2500.0, + "max_p99_latency_ms": 6500.0, "max_memory_growth_mb": 600.0, "max_buffer_full_rate": 0.03, "min_messages_per_poll": 10.0 diff --git a/tests/ducktape/producer_strategy.py b/tests/ducktape/producer_strategy.py index 5c01f3f30..de697e0b9 100644 --- a/tests/ducktape/producer_strategy.py +++ b/tests/ducktape/producer_strategy.py @@ -509,6 +509,16 @@ def shared_metrics_callback(err, msg): pending_futures.append((delivery_future, message_key)) # Store delivery future messages_sent += 1 + # Use configured polling interval (default to 50 if not set) + poll_interval = getattr(self, 'poll_interval', 50) + + if messages_sent % poll_interval == 0: + poll_start = time.time() + await producer.poll(0) + poll_times.append(time.time() - poll_start) + if self.metrics: + self.metrics.record_poll() + except Exception as e: if failed_container is not None: failed_container.append(e) From d45aaa75065f8ee098e9c0a924610c89b785a7ab Mon Sep 17 00:00:00 2001 From: Kaushik Raina Date: Tue, 7 Oct 2025 20:43:36 +0530 Subject: [PATCH 2/6] Enable error logs --- tests/ducktape/producer_benchmark_bounds.json | 10 +++++----- tests/ducktape/test_consumer.py | 2 +- tests/ducktape/test_producer.py | 10 +++++----- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/tests/ducktape/producer_benchmark_bounds.json b/tests/ducktape/producer_benchmark_bounds.json index ec3031047..5fca529ae 100644 --- a/tests/ducktape/producer_benchmark_bounds.json +++ b/tests/ducktape/producer_benchmark_bounds.json @@ -9,18 +9,18 @@ "max_p99_latency_ms": 7000.0, "max_memory_growth_mb": 800.0, "max_buffer_full_rate": 0.05, - "min_messages_per_poll": 10.0 + "min_messages_per_poll": 5.0 }, "ci": { "_comment": "Stricter bounds for CI environment - production-like requirements", "min_throughput_msg_per_sec": 1500.0, - "max_p95_latency_ms": 5500.0, + "max_p95_latency_ms": 11000.0, "max_error_rate": 0.01, "min_success_rate": 0.99, - "max_p99_latency_ms": 6500.0, - "max_memory_growth_mb": 600.0, + "max_p99_latency_ms": 12000.0, + "max_memory_growth_mb": 800.0, "max_buffer_full_rate": 0.03, - "min_messages_per_poll": 10.0 + "min_messages_per_poll": 5.0 }, "_default_environment": "local" } diff --git a/tests/ducktape/test_consumer.py b/tests/ducktape/test_consumer.py index 4c8959fee..c6caad5bf 100644 --- a/tests/ducktape/test_consumer.py +++ b/tests/ducktape/test_consumer.py @@ -570,7 +570,7 @@ def _run_consumer_performance_benchmark( # Validate against performance bounds if not is_valid: - self.logger.warning( + self.logger.error( "Performance bounds validation failed: %s", "; ".join(violations) ) diff --git a/tests/ducktape/test_producer.py b/tests/ducktape/test_producer.py index 792c6fd60..f082e6808 100644 --- a/tests/ducktape/test_producer.py +++ b/tests/ducktape/test_producer.py @@ -127,7 +127,7 @@ def message_formatter(msg_num): # Validate against performance bounds if not is_valid: - self.logger.warning( + self.logger.error( "Performance bounds validation failed: %s", "; ".join(violations) ) @@ -219,7 +219,7 @@ def message_formatter(msg_num): # Validate against performance bounds if not is_valid: - self.logger.warning( + self.logger.error( "Performance bounds validation failed: %s", "; ".join(violations) ) @@ -325,7 +325,7 @@ def message_formatter(msg_num): # Validate against performance bounds if not is_valid: - self.logger.warning( + self.logger.error( "Performance bounds validation failed for %ds test: %s", test_duration, "; ".join(violations), @@ -466,7 +466,7 @@ def message_formatter(msg_num): # Validate against performance bounds if not is_valid: - self.logger.warning( + self.logger.error( "Performance bounds validation failed for %s compression: %s", compression_type, "; ".join(violations), @@ -607,7 +607,7 @@ def message_formatter(i): ), f"Send throughput too low: {metrics_summary['send_throughput_msg_per_sec']:.2f} msg/s" if not is_valid: - self.logger.warning( + self.logger.error( "Performance bounds validation failed: %s", "; ".join(violations) ) From a8dfa063350ae67c8de404a3f9fdc2836d80b591 Mon Sep 17 00:00:00 2001 From: Kaushik Raina Date: Wed, 8 Oct 2025 00:19:56 +0530 Subject: [PATCH 3/6] Fail test when bounds are in valid --- tests/ducktape/test_consumer.py | 1 + tests/ducktape/test_producer.py | 5 +++++ 2 files changed, 6 insertions(+) diff --git a/tests/ducktape/test_consumer.py b/tests/ducktape/test_consumer.py index c6caad5bf..dfb72c8d4 100644 --- a/tests/ducktape/test_consumer.py +++ b/tests/ducktape/test_consumer.py @@ -573,6 +573,7 @@ def _run_consumer_performance_benchmark( self.logger.error( "Performance bounds validation failed: %s", "; ".join(violations) ) + assert False, f"Performance bounds validation failed: {'; '.join(violations)}" self.logger.info( f"Successfully completed basic {operation_type} test with comprehensive metrics" diff --git a/tests/ducktape/test_producer.py b/tests/ducktape/test_producer.py index f082e6808..f2d23dbf7 100644 --- a/tests/ducktape/test_producer.py +++ b/tests/ducktape/test_producer.py @@ -130,6 +130,7 @@ def message_formatter(msg_num): self.logger.error( "Performance bounds validation failed: %s", "; ".join(violations) ) + assert False, f"Performance bounds validation failed: {'; '.join(violations)}" self.logger.info( "Successfully completed basic production test with comprehensive metrics" @@ -222,6 +223,7 @@ def message_formatter(msg_num): self.logger.error( "Performance bounds validation failed: %s", "; ".join(violations) ) + assert False, f"Performance bounds validation failed: {'; '.join(violations)}" self.logger.info( "Successfully completed basic production test with comprehensive metrics with transaction" @@ -330,6 +332,7 @@ def message_formatter(msg_num): test_duration, "; ".join(violations), ) + assert False, f"Performance bounds validation failed for {test_duration}s test: {'; '.join(violations)}" self.logger.info( "Successfully completed %ds batch production test with comprehensive metrics", @@ -471,6 +474,7 @@ def message_formatter(msg_num): compression_type, "; ".join(violations), ) + assert False, f"Performance bounds validation failed for {compression_type} compression: {'; '.join(violations)}" self.logger.info( "Successfully completed %s compression test with comprehensive metrics", @@ -610,6 +614,7 @@ def message_formatter(i): self.logger.error( "Performance bounds validation failed: %s", "; ".join(violations) ) + assert False, f"Performance bounds validation failed: {'; '.join(violations)}" self.logger.info( "Successfully completed SR production test with comprehensive metrics" From 2fe8669f508dcb817e7429d329859054aaea9e07 Mon Sep 17 00:00:00 2001 From: Kaushik Raina Date: Wed, 8 Oct 2025 00:29:55 +0530 Subject: [PATCH 4/6] Fix lint --- tests/ducktape/test_producer.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/ducktape/test_producer.py b/tests/ducktape/test_producer.py index f2d23dbf7..39b714af8 100644 --- a/tests/ducktape/test_producer.py +++ b/tests/ducktape/test_producer.py @@ -474,7 +474,10 @@ def message_formatter(msg_num): compression_type, "; ".join(violations), ) - assert False, f"Performance bounds validation failed for {compression_type} compression: {'; '.join(violations)}" + assert False, ( + f"Performance bounds validation failed for {compression_type} compression: " + f"{'; '.join(violations)}" + ) self.logger.info( "Successfully completed %s compression test with comprehensive metrics", From f07da429a267f346f7d44703d744e71deedd0a93 Mon Sep 17 00:00:00 2001 From: Kaushik Raina Date: Wed, 8 Oct 2025 01:04:21 +0530 Subject: [PATCH 5/6] Update consume metrics --- tests/ducktape/consumer_benchmark_metrics.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/ducktape/consumer_benchmark_metrics.py b/tests/ducktape/consumer_benchmark_metrics.py index 6a1d2eb3a..91302956b 100644 --- a/tests/ducktape/consumer_benchmark_metrics.py +++ b/tests/ducktape/consumer_benchmark_metrics.py @@ -231,7 +231,7 @@ def __init__(self, min_success_rate: float = 0.95, max_error_rate: float = 0.05, max_memory_growth_mb: float = 600.0, - min_messages_per_consume: float = 1.0, + min_messages_per_consume: float = 0.5, max_empty_consume_rate: float = 0.5): self.min_consumption_rate = min_consumption_rate self.max_avg_latency_ms = max_avg_latency_ms From 773bb0855758f0628979d232496246a144ccfe54 Mon Sep 17 00:00:00 2001 From: Kaushik Raina Date: Wed, 8 Oct 2025 08:25:30 +0530 Subject: [PATCH 6/6] Update consume metrics --- tests/ducktape/consumer_benchmark_metrics.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/ducktape/consumer_benchmark_metrics.py b/tests/ducktape/consumer_benchmark_metrics.py index 91302956b..afafe8264 100644 --- a/tests/ducktape/consumer_benchmark_metrics.py +++ b/tests/ducktape/consumer_benchmark_metrics.py @@ -228,7 +228,7 @@ def __init__(self, min_consumption_rate: float = 1.0, max_avg_latency_ms: float = 5000.0, max_p95_latency_ms: float = 10000.0, - min_success_rate: float = 0.95, + min_success_rate: float = 0.90, max_error_rate: float = 0.05, max_memory_growth_mb: float = 600.0, min_messages_per_consume: float = 0.5,