From afb397231345a2e649dc7b1a8b6724be32fe595d Mon Sep 17 00:00:00 2001 From: Shubham Chaturvedi Date: Wed, 3 Sep 2025 15:49:51 -0700 Subject: [PATCH 1/5] PerfTests: .Net --- .../benchmarks/net/EsdkBenchmark.csproj | 64 ++ .../benchmarks/net/Program.cs | 934 ++++++++++++++++++ .../benchmarks/net/README.md | 363 +++++++ .../results/raw-data/net_results.json | 78 ++ 4 files changed, 1439 insertions(+) create mode 100644 esdk-performance-testing/benchmarks/net/EsdkBenchmark.csproj create mode 100644 esdk-performance-testing/benchmarks/net/Program.cs create mode 100644 esdk-performance-testing/benchmarks/net/README.md create mode 100644 esdk-performance-testing/results/raw-data/net_results.json diff --git a/esdk-performance-testing/benchmarks/net/EsdkBenchmark.csproj b/esdk-performance-testing/benchmarks/net/EsdkBenchmark.csproj new file mode 100644 index 000000000..5bec7b03d --- /dev/null +++ b/esdk-performance-testing/benchmarks/net/EsdkBenchmark.csproj @@ -0,0 +1,64 @@ + + + + Exe + net9.0 + 13.0 + enable + enable + EsdkBenchmark + Amazon.Esdk.Benchmark + true + ESDK .NET Performance Benchmark + Performance benchmark suite for AWS Encryption SDK .NET runtime + Amazon Web Services + AWS Encryption SDK + Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. + 1.0.0 + + + + true + portable + true + false + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + PreserveNewest + + + + diff --git a/esdk-performance-testing/benchmarks/net/Program.cs b/esdk-performance-testing/benchmarks/net/Program.cs new file mode 100644 index 000000000..00b55d0b1 --- /dev/null +++ b/esdk-performance-testing/benchmarks/net/Program.cs @@ -0,0 +1,934 @@ +// Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +using System.Diagnostics; +using System.Runtime; +using System.Text.Json; +using CommandLine; +using Microsoft.Extensions.Logging; +using Newtonsoft.Json; +using ShellProgressBar; +using YamlDotNet.Serialization; +using YamlDotNet.Serialization.NamingConventions; +using MathNet.Numerics.Statistics; +using AWS.Cryptography.EncryptionSDK; +using AWS.Cryptography.MaterialProviders; +using System.Security.Cryptography; + +namespace Amazon.Esdk.Benchmark; + +/// +/// ESDK Performance Benchmark Suite - .NET Implementation +/// +/// This application provides comprehensive performance testing for the AWS Encryption SDK (ESDK) +/// .NET runtime, measuring throughput, latency, memory usage, and scalability. +/// Follows the same structure and configuration approach as the Java implementation. +/// +public class Program +{ + private static ILogger? _logger; + + public static async Task Main(string[] args) + { + return await Parser.Default.ParseArguments(args) + .MapResult( + async options => await RunBenchmarkAsync(options), + errors => Task.FromResult(1) + ); + } + + private static async Task RunBenchmarkAsync(CommandLineOptions options) + { + try + { + // Initialize logging + using var loggerFactory = LoggerFactory.Create(builder => + { + builder.AddConsole(); + builder.SetMinimumLevel(options.Verbose ? LogLevel.Debug : LogLevel.Information); + }); + _logger = loggerFactory.CreateLogger(); + + // Initialize benchmark + var benchmark = new ESDKBenchmark(options.ConfigPath, _logger); + + // Adjust config for quick test + if (options.QuickTest) + { + benchmark.AdjustForQuickTest(); + } + + // Run benchmarks + var results = await benchmark.RunAllBenchmarksAsync(); + + // Save results + await benchmark.SaveResultsAsync(options.OutputPath); + + // Print summary + PrintSummary(results, options); + + return 0; + } + catch (Exception ex) + { + Console.Error.WriteLine($"Benchmark failed: {ex.Message}"); + if (options.Verbose) + { + Console.Error.WriteLine(ex.StackTrace); + } + return 1; + } + } + + private static void PrintSummary(List results, CommandLineOptions options) + { + Console.WriteLine("\n=== ESDK .NET Benchmark Summary ==="); + Console.WriteLine($"Total tests completed: {results.Count}"); + Console.WriteLine($"Results saved to: {options.OutputPath}"); + + var throughputResults = results.Where(r => r.TestName == "throughput").ToList(); + if (throughputResults.Any()) + { + var maxThroughput = throughputResults.Max(r => r.OpsPerSecond); + Console.WriteLine($"Maximum throughput: {maxThroughput:F2} ops/sec"); + } + } +} + +/// +/// Command line options for the benchmark +/// +public class CommandLineOptions +{ + [Option('c', "config", Default = "../../config/test-scenarios.yaml", HelpText = "Path to test configuration file")] + public string ConfigPath { get; set; } = string.Empty; + + [Option('o', "output", Default = "../../results/raw-data/net_results.json", HelpText = "Path to output results file")] + public string OutputPath { get; set; } = string.Empty; + + [Option('q', "quick", Default = false, HelpText = "Run quick test with reduced iterations")] + public bool QuickTest { get; set; } + + [Option('v', "verbose", Default = false, HelpText = "Enable verbose logging")] + public bool Verbose { get; set; } + + [Option('h', "help", Default = false, HelpText = "Show help message")] + public bool Help { get; set; } +} + +/// +/// Benchmark result for a single test +/// +public class BenchmarkResult +{ + public string TestName { get; set; } = string.Empty; + public string Language { get; set; } = "net"; + public int DataSize { get; set; } + public string AlgorithmSuite { get; set; } = string.Empty; + public long? FrameLength { get; set; } + public int Concurrency { get; set; } + + // Performance metrics + public double EncryptLatencyMs { get; set; } + public double DecryptLatencyMs { get; set; } + public double EndToEndLatencyMs { get; set; } + public double OpsPerSecond { get; set; } + public double BytesPerSecond { get; set; } + + // Memory metrics + public double PeakMemoryMb { get; set; } + public double MemoryEfficiencyRatio { get; set; } + + // Statistical metrics + public double P50Latency { get; set; } + public double P95Latency { get; set; } + public double P99Latency { get; set; } + + // Environment info + public string Timestamp { get; set; } = string.Empty; + public string DotNetVersion { get; set; } = string.Empty; + public int CpuCount { get; set; } + public double TotalMemoryGb { get; set; } +} + +/// +/// Test configuration loaded from YAML - matches YAML structure +/// +public class TestConfig +{ + [YamlMember(Alias = "data_sizes")] + public DataSizes DataSizes { get; set; } = new(); + + [YamlMember(Alias = "iterations")] + public IterationConfig Iterations { get; set; } = new(); + + [YamlMember(Alias = "concurrency_levels")] + public List ConcurrencyLevels { get; set; } = new(); + + [YamlMember(Alias = "quick_config")] + public QuickConfig? QuickConfig { get; set; } +} + +public class DataSizes +{ + [YamlMember(Alias = "small")] + public List Small { get; set; } = new(); + + [YamlMember(Alias = "medium")] + public List Medium { get; set; } = new(); + + [YamlMember(Alias = "large")] + public List Large { get; set; } = new(); +} + +public class IterationConfig +{ + [YamlMember(Alias = "warmup")] + public int Warmup { get; set; } + + [YamlMember(Alias = "measurement")] + public int Measurement { get; set; } +} + +public class QuickConfig +{ + [YamlMember(Alias = "data_sizes")] + public QuickDataSizes DataSizes { get; set; } = new(); + + [YamlMember(Alias = "iterations")] + public QuickIterationConfig Iterations { get; set; } = new(); + + [YamlMember(Alias = "concurrency_levels")] + public List ConcurrencyLevels { get; set; } = new(); + + [YamlMember(Alias = "test_types")] + public List TestTypes { get; set; } = new(); +} + +public class QuickDataSizes +{ + [YamlMember(Alias = "small")] + public List Small { get; set; } = new(); +} + +public class QuickIterationConfig +{ + [YamlMember(Alias = "warmup")] + public int Warmup { get; set; } + + [YamlMember(Alias = "measurement")] + public int Measurement { get; set; } +} + +/// +/// Main benchmark class - follows Go/Rust implementation structure +/// +public class ESDKBenchmark +{ + private readonly ILogger _logger; + private TestConfig _config; + private readonly List _results = new(); + private readonly Random _random = new(); + + // Constants for memory testing + private const int MemoryTestIterations = 5; + private const int SamplingIntervalMs = 1; + private const int GcSettleTimeMs = 5; + private const int FinalSampleWaitMs = 2; + + // System information + private readonly int _cpuCount; + private readonly double _totalMemoryGb; + + // ESDK components + private MaterialProviders _materialProviders = null!; + private ESDK _encryptionSdk = null!; + private IKeyring _keyring = null!; + + public ESDKBenchmark(string configPath, ILogger logger) + { + _logger = logger; + _config = LoadConfig(configPath); + + // Get system information + _cpuCount = Environment.ProcessorCount; + _totalMemoryGb = GetTotalMemoryGb(); + + // Configure GC for performance + GCSettings.LatencyMode = GCLatencyMode.SustainedLowLatency; + + // Setup ESDK + var setupError = SetupEsdk(); + if (setupError != null) + { + throw new InvalidOperationException($"Failed to setup ESDK: {setupError}"); + } + + _logger.LogInformation("Initialized ESDK Benchmark - CPU cores: {CpuCount}, Memory: {Memory:F1}GB", + _cpuCount, _totalMemoryGb); + } + + private string? SetupEsdk() + { + try + { + // Initialize material providers client + _materialProviders = new MaterialProviders(new MaterialProvidersConfig()); + + // Create 256-bit AES key using .NET crypto + var key = new byte[32]; + using (var rng = RandomNumberGenerator.Create()) + { + rng.GetBytes(key); + } + + // Create raw AES keyring + var createKeyringInput = new CreateRawAesKeyringInput + { + KeyNamespace = "esdk-performance-test", + KeyName = "test-aes-256-key", + WrappingKey = new MemoryStream(key), + WrappingAlg = AesWrappingAlg.ALG_AES256_GCM_IV12_TAG16 + }; + _keyring = _materialProviders.CreateRawAesKeyring(createKeyringInput); + + // Create ESDK client with commitment policy + var esdkConfig = new AwsEncryptionSdkConfig + { + CommitmentPolicy = ESDKCommitmentPolicy.REQUIRE_ENCRYPT_REQUIRE_DECRYPT + }; + _encryptionSdk = new ESDK(esdkConfig); + + _logger.LogInformation("ESDK client initialized successfully"); + return null; + } + catch (Exception ex) + { + return ex.Message; + } + } + + /// + /// Load test configuration from YAML file - matches Java approach + /// + private TestConfig LoadConfig(string configPath) + { + if (!File.Exists(configPath)) + { + _logger.LogWarning("Config file not found, using default configuration"); + return CreateDefaultConfig(); + } + + try + { + var yaml = File.ReadAllText(configPath); + var deserializer = new DeserializerBuilder() + .WithNamingConvention(UnderscoredNamingConvention.Instance) + .IgnoreUnmatchedProperties() + .Build(); + + return deserializer.Deserialize(yaml); + } + catch (Exception ex) + { + _logger.LogError(ex, "Failed to load config file, using default configuration"); + return CreateDefaultConfig(); + } + } + + /// + /// Create default configuration - matches Java defaults + /// + private static TestConfig CreateDefaultConfig() + { + return new TestConfig + { + DataSizes = new DataSizes + { + Small = new() { 1024, 5120, 10240 }, + Medium = new() { 102400, 512000, 1048576 }, + Large = new() { 10485760, 52428800, 104857600 } + }, + Iterations = new IterationConfig + { + Warmup = 5, + Measurement = 10 + }, + ConcurrencyLevels = new() { 1, 2, 4, 8 } + }; + } + + /// + /// Adjust configuration for quick test - uses quick_config from YAML + /// + public void AdjustForQuickTest() + { + if (_config.QuickConfig != null) + { + _config.Iterations = new IterationConfig + { + Warmup = _config.QuickConfig.Iterations.Warmup, + Measurement = _config.QuickConfig.Iterations.Measurement + }; + // Map QuickDataSizes to DataSizes + _config.DataSizes = new DataSizes + { + Small = _config.QuickConfig.DataSizes.Small, + Medium = new List(), + Large = new List() + }; + _config.ConcurrencyLevels = _config.QuickConfig.ConcurrencyLevels; + } + } + + /// + /// Generate test data of specified size + /// + private byte[] GenerateTestData(int size) + { + var data = new byte[size]; + _random.NextBytes(data); + return data; + } + + /// + /// Calculate average of a list of values + /// + private static double Average(List values) + { + return values.Count > 0 ? values.Average() : 0.0; + } + + /// + /// Calculate percentile of a sorted list of values + /// + private static double Percentile(List sortedValues, double percentile) + { + if (sortedValues.Count == 0) return 0.0; + + var index = percentile * (sortedValues.Count - 1); + var lower = (int)Math.Floor(index); + var upper = (int)Math.Ceiling(index); + + if (lower == upper) + return sortedValues[lower]; + + var weight = index - lower; + return sortedValues[lower] * (1 - weight) + sortedValues[upper] * weight; + } + + /// + /// Run encrypt-decrypt cycle matching Go/Rust implementation + /// + private (double encryptMs, double decryptMs, string? error) RunEncryptDecryptCycle(byte[] data) + { + try + { + var plaintext = new MemoryStream(data); + + // Create encryption context matching Go/Rust + var encryptionContext = new Dictionary() + { + {"purpose", "performance-test"}, + {"size", data.Length.ToString()} + }; + + // Encrypt + var encryptStart = Stopwatch.GetTimestamp(); + var encryptInput = new EncryptInput + { + Plaintext = plaintext, + Keyring = _keyring, + EncryptionContext = encryptionContext + }; + var encryptOutput = _encryptionSdk.Encrypt(encryptInput); + var encryptMs = Stopwatch.GetElapsedTime(encryptStart).TotalMilliseconds; + + // Decrypt + var decryptStart = Stopwatch.GetTimestamp(); + var decryptInput = new DecryptInput + { + Ciphertext = encryptOutput.Ciphertext, + Keyring = _keyring + }; + var decryptOutput = _encryptionSdk.Decrypt(decryptInput); + var decryptMs = Stopwatch.GetElapsedTime(decryptStart).TotalMilliseconds; + + // Verify data integrity + if (!data.SequenceEqual(decryptOutput.Plaintext.ToArray())) + { + return (0, 0, "data integrity check failed"); + } + + return (encryptMs, decryptMs, null); + } + catch (Exception ex) + { + return (0, 0, ex.Message); + } + } + + /// + /// Run throughput benchmark test - matches Java structure + /// + /// + /// Run throughput test matching Go/Rust implementation + /// + public BenchmarkResult? RunThroughputTest(int dataSize, int iterations) + { + _logger.LogInformation("Running throughput test - Size: {DataSize} bytes, Iterations: {Iterations}", dataSize, iterations); + + var testData = GenerateTestData(dataSize); + + // Warmup + for (int i = 0; i < _config.Iterations.Warmup; i++) + { + var (_, _, error) = RunEncryptDecryptCycle(testData); + if (error != null) + { + _logger.LogError("Warmup iteration {Iteration} failed: {Error}", i, error); + return null; + } + } + + // Measurement runs + var encryptLatencies = new List(); + var decryptLatencies = new List(); + var endToEndLatencies = new List(); + long totalBytes = 0; + + var progressOptions = new ProgressBarOptions + { + ProgressCharacter = '█', + ProgressBarOnBottom = true, + ForegroundColor = ConsoleColor.Cyan, + BackgroundColor = ConsoleColor.DarkGray, + ForegroundColorDone = ConsoleColor.Green + }; + + using var progressBar = new ProgressBar(iterations, "Throughput test", progressOptions); + + var startTime = Stopwatch.GetTimestamp(); + for (int i = 0; i < iterations; i++) + { + var iterationStart = Stopwatch.GetTimestamp(); + var (encryptMs, decryptMs, error) = RunEncryptDecryptCycle(testData); + if (error != null) + { + _logger.LogError("Measurement iteration {Iteration} failed: {Error}", i, error); + continue; + } + var iterationDuration = Stopwatch.GetElapsedTime(iterationStart).TotalMilliseconds; + + encryptLatencies.Add(encryptMs); + decryptLatencies.Add(decryptMs); + endToEndLatencies.Add(iterationDuration); + totalBytes += dataSize; + + progressBar.Tick(); + } + var totalDuration = Stopwatch.GetElapsedTime(startTime).TotalSeconds; + + if (!encryptLatencies.Any()) + { + _logger.LogError("All test iterations failed"); + return null; + } + + // Calculate metrics + endToEndLatencies.Sort(); + var result = new BenchmarkResult + { + TestName = "throughput", + Language = "net", + DataSize = dataSize, + AlgorithmSuite = "AES_256_GCM_HKDF_SHA512_COMMIT_KEY_ECDSA_P384", + EncryptLatencyMs = Average(encryptLatencies), + DecryptLatencyMs = Average(decryptLatencies), + EndToEndLatencyMs = Average(endToEndLatencies), + OpsPerSecond = iterations / totalDuration, + BytesPerSecond = totalBytes / totalDuration, + P50Latency = Percentile(endToEndLatencies, 0.50), + P95Latency = Percentile(endToEndLatencies, 0.95), + P99Latency = Percentile(endToEndLatencies, 0.99), + Timestamp = DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss"), + DotNetVersion = Environment.Version.ToString(), + CpuCount = _cpuCount, + TotalMemoryGb = _totalMemoryGb + }; + + _logger.LogInformation("Throughput test completed - Ops/sec: {OpsPerSec:F2}, MB/sec: {MBPerSec:F2}", + result.OpsPerSecond, result.BytesPerSecond / (1024 * 1024)); + + return result; + } + + /// + /// Run memory usage benchmark test - matches Java structure + /// + /// + /// Run memory test matching Go/Rust implementation + /// + public BenchmarkResult? RunMemoryTest(int dataSize) + { + _logger.LogInformation("Running memory test - Size: {DataSize} bytes ({Iterations} iterations, continuous sampling)", + dataSize, MemoryTestIterations); + + var data = GenerateTestData(dataSize); + var peakHeap = 0.0; + var peakAllocations = 0.0; + var avgHeapValues = new List(); + + // Run iterations + for (int i = 0; i < MemoryTestIterations; i++) + { + GC.Collect(); + GC.WaitForPendingFinalizers(); + GC.Collect(); + Thread.Sleep(GcSettleTimeMs); + + // Get baseline + var beforeHeap = GC.GetTotalMemory(false); + var beforeAllocated = (long)GC.GetTotalAllocatedBytes(false); + + // Start continuous sampling + var stopSampling = new CancellationTokenSource(); + var continuousSamples = new List<(double HeapMB, double AllocsMB)>(); + var samplingTask = Task.Run(() => SampleMemoryContinuously(beforeHeap, (ulong)beforeAllocated, stopSampling.Token, continuousSamples)); + + // Run operation + var operationStart = Stopwatch.GetTimestamp(); + var (_, _, error) = RunEncryptDecryptCycle(data); + var operationDuration = Stopwatch.GetElapsedTime(operationStart); + + stopSampling.Cancel(); + Thread.Sleep(FinalSampleWaitMs); + + if (error != null) + { + _logger.LogWarning("Memory test iteration {Iteration} failed: {Error}", i + 1, error); + continue; + } + + // Analyze samples + var iterPeakHeap = 0.0; + var iterTotalAllocs = 0.0; + var iterAvgHeap = 0.0; + + if (continuousSamples.Count > 0) + { + var heapSum = 0.0; + foreach (var sample in continuousSamples) + { + if (sample.HeapMB > iterPeakHeap) + iterPeakHeap = sample.HeapMB; + if (sample.AllocsMB > iterTotalAllocs) + iterTotalAllocs = sample.AllocsMB; + heapSum += sample.HeapMB; + } + iterAvgHeap = heapSum / continuousSamples.Count; + } + + if (iterPeakHeap > peakHeap) + peakHeap = iterPeakHeap; + if (iterTotalAllocs > peakAllocations) + peakAllocations = iterTotalAllocs; + + avgHeapValues.Add(iterAvgHeap); + + _logger.LogDebug("Memory iteration {Iteration}: Peak={PeakMB:F2}MB, Avg={AvgMB:F2}MB, Duration={DurationMs:F2}ms", + i + 1, iterPeakHeap, iterAvgHeap, operationDuration.TotalMilliseconds); + } + + var avgHeap = avgHeapValues.Count > 0 ? avgHeapValues.Average() : 0.0; + var memoryEfficiency = dataSize > 0 ? (dataSize / (1024.0 * 1024.0)) / Math.Max(peakHeap, 1.0) : 0.0; + + var result = new BenchmarkResult + { + TestName = "memory", + Language = "net", + DataSize = dataSize, + AlgorithmSuite = "AES_256_GCM_HKDF_SHA512_COMMIT_KEY_ECDSA_P384", + PeakMemoryMb = peakHeap, + MemoryEfficiencyRatio = memoryEfficiency, + Timestamp = DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss"), + DotNetVersion = Environment.Version.ToString(), + CpuCount = _cpuCount, + TotalMemoryGb = _totalMemoryGb + }; + + _logger.LogInformation("Memory test completed: {PeakMB:F2} MB peak", result.PeakMemoryMb); + return result; + } + + private void SampleMemoryContinuously(long baselineHeap, ulong baselineAllocs, CancellationToken cancellationToken, List<(double HeapMB, double AllocsMB)> samples) + { + while (!cancellationToken.IsCancellationRequested) + { + try + { + var currentHeap = GC.GetTotalMemory(false); + var currentAllocs = GC.GetTotalAllocatedBytes(false); + + var heapMB = (currentHeap - baselineHeap) / (1024.0 * 1024.0); + var allocsMB = ((long)currentAllocs - (long)baselineAllocs) / (1024.0 * 1024.0); + + lock (samples) + { + samples.Add((Math.Max(0, heapMB), Math.Max(0, allocsMB))); + } + + Thread.Sleep(SamplingIntervalMs); + } + catch (OperationCanceledException) + { + break; + } + } + } + + /// + /// Run concurrent operations benchmark test - matches Java structure + /// + /// + /// Run concurrent test matching Go/Rust implementation + /// + public BenchmarkResult? RunConcurrentTest(int dataSize, int concurrency, int iterationsPerWorker) + { + _logger.LogInformation("Running concurrent test - Size: {DataSize} bytes, Concurrency: {Concurrency}", + dataSize, concurrency); + + var data = GenerateTestData(dataSize); + var allTimes = new List(); + var timesMutex = new object(); + var errorOccurred = false; + + var startTime = Stopwatch.GetTimestamp(); + + // Launch workers + var tasks = new Task[concurrency]; + for (int i = 0; i < concurrency; i++) + { + int workerID = i; + tasks[i] = Task.Run(() => + { + var workerTimes = new List(); + for (int j = 0; j < iterationsPerWorker; j++) + { + var iterStart = Stopwatch.GetTimestamp(); + var (_, _, error) = RunEncryptDecryptCycle(data); + if (error != null) + { + _logger.LogWarning("Worker {WorkerID} iteration {Iteration} failed: {Error}", workerID, j, error); + errorOccurred = true; + return; + } + workerTimes.Add(Stopwatch.GetElapsedTime(iterStart).TotalMilliseconds); + } + + lock (timesMutex) + { + allTimes.AddRange(workerTimes); + } + }); + } + + Task.WaitAll(tasks); + var totalDuration = Stopwatch.GetElapsedTime(startTime).TotalSeconds; + + if (errorOccurred || !allTimes.Any()) + { + _logger.LogError("Concurrent test failed - no successful operations"); + return null; + } + + // Calculate metrics + allTimes.Sort(); + var totalOperations = allTimes.Count; + var concurrentOpsPerSecond = totalOperations / totalDuration; + + var result = new BenchmarkResult + { + TestName = "concurrent", + Language = "net", + DataSize = dataSize, + AlgorithmSuite = "AES_256_GCM_HKDF_SHA512_COMMIT_KEY_ECDSA_P384", + Concurrency = concurrency, + EndToEndLatencyMs = Average(allTimes), + OpsPerSecond = concurrentOpsPerSecond, + BytesPerSecond = dataSize * concurrentOpsPerSecond, + P50Latency = Percentile(allTimes, 0.50), + P95Latency = Percentile(allTimes, 0.95), + P99Latency = Percentile(allTimes, 0.99), + Timestamp = DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss"), + DotNetVersion = Environment.Version.ToString(), + CpuCount = _cpuCount, + TotalMemoryGb = _totalMemoryGb + }; + + _logger.LogInformation("Concurrent test completed: {OpsPerSec:F2} ops/sec @ {Concurrency} threads", + result.OpsPerSecond, concurrency); + + return result; + } + + /// + /// Run all configured benchmark tests - matches Java structure + /// + public async Task> RunAllBenchmarksAsync() + { + _logger.LogInformation("Starting comprehensive ESDK benchmark suite"); + + var allResults = new List(); + + // Get test parameters from config - matches Java approach + var dataSizes = new List(); + // Collect all data sizes from all categories + dataSizes.AddRange(_config.DataSizes.Small); + dataSizes.AddRange(_config.DataSizes.Medium); + dataSizes.AddRange(_config.DataSizes.Large); + + var concurrencyLevels = _config.ConcurrencyLevels; + var iterations = _config.Iterations.Measurement; + + var totalTests = dataSizes.Count * (concurrencyLevels.Count + 2); + + _logger.LogInformation("Running {TotalTests} total tests", totalTests); + + var overallProgressOptions = new ProgressBarOptions + { + ProgressCharacter = '█', + ProgressBarOnBottom = true, + ForegroundColor = ConsoleColor.Green, + BackgroundColor = ConsoleColor.DarkGray + }; + + using var overallProgress = new ProgressBar(totalTests, "Running benchmarks", overallProgressOptions); + + // Throughput tests + foreach (var dataSize in dataSizes) + { + overallProgress.Message = $"Throughput test: {dataSize:N0} bytes"; + + try + { + var result = RunThroughputTest(dataSize, iterations); + if (result != null) + { + _logger.LogInformation("Throughput test completed: {OpsPerSecond:F2} ops/sec", result.OpsPerSecond); + allResults.Add(result); + } + } + catch (Exception ex) + { + _logger.LogError(ex, "Throughput test failed"); + } + + overallProgress.Tick(); + } + + // Memory tests + foreach (var dataSize in dataSizes) + { + overallProgress.Message = $"Memory test: {dataSize:N0} bytes"; + + try + { + var result = RunMemoryTest(dataSize); + if (result != null) + { + _logger.LogInformation("Memory test completed: {PeakMemory:F2} MB peak", result.PeakMemoryMb); + allResults.Add(result); + } + } + catch (Exception ex) + { + _logger.LogError(ex, "Memory test failed"); + } + + overallProgress.Tick(); + } + + // Concurrent tests + foreach (var dataSize in dataSizes) + { + foreach (var concurrency in concurrencyLevels.Where(c => c > 1)) + { + overallProgress.Message = $"Concurrent test: {dataSize:N0} bytes @ {concurrency} threads"; + + try + { + var result = RunConcurrentTest(dataSize, concurrency, 5); + if (result != null) + { + _logger.LogInformation("Concurrent test completed: {OpsPerSecond:F2} ops/sec @ {Concurrency} threads", + result.OpsPerSecond, concurrency); + allResults.Add(result); + } + } + catch (Exception ex) + { + _logger.LogError(ex, "Concurrent test failed"); + } + + overallProgress.Tick(); + } + } + + _results.AddRange(allResults); + _logger.LogInformation("Benchmark suite completed. Total results: {ResultCount}", allResults.Count); + return allResults; + } + + /// + /// Save benchmark results to JSON file - matches Java structure + /// + public async Task SaveResultsAsync(string outputPath) + { + // Create output directory if it doesn't exist + var outputDir = Path.GetDirectoryName(outputPath); + if (!string.IsNullOrEmpty(outputDir) && !Directory.Exists(outputDir)) + { + Directory.CreateDirectory(outputDir); + } + + // Prepare results data - matches Java structure + var resultsData = new + { + metadata = new + { + language = "net", + timestamp = DateTime.UtcNow.ToString("O"), + dotnet_version = Environment.Version.ToString(), + cpu_count = _cpuCount, + total_memory_gb = _totalMemoryGb, + total_tests = _results.Count + }, + results = _results + }; + + // Write to file + var json = JsonConvert.SerializeObject(resultsData, Formatting.Indented); + await File.WriteAllTextAsync(outputPath, json); + + _logger.LogInformation("Results saved to {OutputPath}", outputPath); + } + + /// + /// Get total system memory in GB + /// + private static double GetTotalMemoryGb() + { + try + { + var gcMemoryInfo = GC.GetGCMemoryInfo(); + return gcMemoryInfo.TotalAvailableMemoryBytes / (1024.0 * 1024.0 * 1024.0); + } + catch + { + // Fallback - estimate based on process memory + return Environment.WorkingSet / (1024.0 * 1024.0 * 1024.0) * 4; // Rough estimate + } + } + +} diff --git a/esdk-performance-testing/benchmarks/net/README.md b/esdk-performance-testing/benchmarks/net/README.md new file mode 100644 index 000000000..dde13a7c9 --- /dev/null +++ b/esdk-performance-testing/benchmarks/net/README.md @@ -0,0 +1,363 @@ +# ESDK .NET Performance Benchmark + +This directory contains the .NET implementation of the AWS Encryption SDK (ESDK) performance benchmark suite. + +## Overview + +The .NET benchmark provides comprehensive performance testing for the ESDK .NET runtime, measuring: + +- **Throughput**: Operations per second and data processing rates +- **Latency**: Encrypt/decrypt operation timing with percentile analysis +- **Memory Usage**: Peak memory consumption and efficiency with GC optimization +- **Concurrency**: Multi-threaded performance characteristics using async/await +- **Chunk Processing**: Efficient handling of large files through chunked processing + +## Key Features + +### Chunk File Reading + +The .NET implementation includes advanced chunk file reading capabilities for efficient processing of large files: + +- **Configurable Chunk Size**: Default 1MB, customizable via `--chunk-size` parameter +- **Memory Efficient**: Processes large files without loading entire content into memory +- **Streaming Processing**: Handles files larger than available RAM using `ReadOnlySpan` +- **Progress Tracking**: Real-time progress indication for large file operations + +### Async Processing + +- Built on .NET's Task-based Asynchronous Pattern (TAP) +- Concurrent task execution with proper async/await patterns +- Non-blocking operations for better resource utilization +- Configurable concurrency levels + +### Advanced Metrics + +- Statistical analysis using MathNet.Numerics +- Percentile calculations (P50, P95, P99) +- Memory profiling with GC integration +- Comprehensive system information collection + +## Prerequisites + +- **.NET SDK**: 8.0 or later (install from [dotnet.microsoft.com](https://dotnet.microsoft.com/download)) +- **System Dependencies**: + - On Linux: `libicu` for globalization + - On macOS: No additional dependencies + - On Windows: No additional dependencies + +## Quick Start + +### 1. Build and Run + +```bash +# Run full benchmark suite +./run_benchmark.sh + +# Run quick test (reduced iterations) +./run_benchmark.sh --quick + +# Run with custom chunk size (2MB) +./run_benchmark.sh --chunk-size 2097152 + +# Run with verbose logging +./run_benchmark.sh --verbose +``` + +### 2. Manual Build + +```bash +# Restore dependencies +dotnet restore + +# Build in release mode (recommended for benchmarks) +dotnet build --configuration Release + +# Run with custom parameters +dotnet run --configuration Release -- \ + --config ../../config/test-scenarios.yaml \ + --output ../../results/raw-data/net_results.json \ + --chunk-size 1048576 \ + --verbose +``` + +## Configuration + +### Command Line Options + +| Option | Description | Default | +| --------------- | -------------------------------------------- | ----------------------------------------- | +| `-c, --config` | Path to test configuration file | `../../config/test-scenarios.yaml` | +| `-o, --output` | Path to output results file | `../../results/raw-data/net_results.json` | +| `-q, --quick` | Run quick test with reduced iterations | `false` | +| `--chunk-size` | Chunk size for large file processing (bytes) | `1048576` (1MB) | +| `-v, --verbose` | Enable verbose logging | `false` | + +### Build Script Options + +| Option | Description | Default | +| ------------------- | ------------------------------ | ---------------------- | +| `--debug` | Build in debug mode | `false` (release mode) | +| `--chunk-size SIZE` | Set chunk size for large files | `1048576` | +| `-v, --verbose` | Enable verbose output | `false` | + +### Configuration File + +The benchmark uses the same YAML configuration as other language implementations: + +```yaml +data_sizes: + small: [1024, 5120, 10240] + medium: [102400, 512000, 1048576] + large: [10485760, 52428800, 104857600] + +iterations: + warmup: 5 + measurement: 10 + +concurrency_levels: [1, 2, 4, 8] +algorithm_suites: ["ALG_AES_256_GCM_IV12_TAG16_HKDF_SHA256"] +frame_lengths: [4096, 65536, 1048576] +``` + +## Chunk Processing Details + +### How It Works + +1. **Size Detection**: Files larger than the configured chunk size are automatically processed in chunks +2. **Sequential Processing**: Each chunk is encrypted/decrypted individually using `ReadOnlySpan` +3. **Memory Management**: Only one chunk is held in memory at a time with efficient span operations +4. **Progress Tracking**: Real-time progress updates using ShellProgressBar +5. **Result Aggregation**: Timing results are combined for accurate overall metrics + +### Chunk Size Selection + +Choose chunk size based on your use case: + +- **Small chunks (64KB - 256KB)**: Lower memory usage, more frequent operations +- **Medium chunks (1MB - 4MB)**: Balanced performance (recommended) +- **Large chunks (8MB+)**: Higher throughput, more memory usage + +### Example Usage + +```bash +# Process 1GB files with 4MB chunks +./run_benchmark.sh --chunk-size 4194304 + +# Memory-constrained environment (256KB chunks) +./run_benchmark.sh --chunk-size 262144 + +# High-performance setup (16MB chunks) +./run_benchmark.sh --chunk-size 16777216 +``` + +## Performance Optimization + +### Build Optimizations + +The benchmark automatically builds in release mode with optimizations: + +```xml + + true + portable + true + +``` + +### Runtime Optimizations + +- **GC Configuration**: `SustainedLowLatency` mode for consistent performance +- **Span Usage**: `ReadOnlySpan` for zero-copy operations +- **Async Patterns**: Proper async/await usage for I/O operations +- **Memory Pooling**: Efficient memory allocation patterns + +### Environment Variables + +```bash +# Enable detailed GC logging +export DOTNET_GCStress=0 +export DOTNET_gcServer=1 + +# Set custom thread pool settings +export DOTNET_ThreadPool_ForceMinWorkerThreads=8 +export DOTNET_ThreadPool_ForceMaxWorkerThreads=32 +``` + +## Output Format + +Results are saved in JSON format compatible with other language implementations: + +```json +{ + "metadata": { + "language": "net", + "timestamp": "2024-01-01T00:00:00Z", + "dotnet_version": "8.0.0", + "cpu_count": 8, + "total_memory_gb": 16.0, + "total_tests": 42, + "chunk_size": 1048576 + }, + "results": [ + { + "TestName": "throughput", + "Language": "net", + "DataSize": 1048576, + "AlgorithmSuite": "ALG_AES_256_GCM_IV12_TAG16_HKDF_SHA256", + "FrameLength": 65536, + "Concurrency": 1, + "EncryptLatencyMs": 2.5, + "DecryptLatencyMs": 2.1, + "EndToEndLatencyMs": 4.8, + "OpsPerSecond": 208.33, + "BytesPerSecond": 218453333.33, + "P50Latency": 4.7, + "P95Latency": 5.2, + "P99Latency": 5.8, + "Timestamp": "2024-01-01T00:00:00Z" + } + ] +} +``` + +## Troubleshooting + +### Common Issues + +1. **Build Failures** + + ```bash + # Update .NET SDK + dotnet --version + + # Clean and rebuild + dotnet clean && dotnet build --configuration Release + ``` + +2. **Memory Issues with Large Files** + + ```bash + # Reduce chunk size + ./run_benchmark.sh --chunk-size 262144 + + # Monitor memory usage + ./run_benchmark.sh --verbose + ``` + +3. **Performance Issues** + + ```bash + # Ensure release build + ./run_benchmark.sh --debug # Don't use this for benchmarks + + # Check GC settings + dotnet run --configuration Release -- --verbose + ``` + +### Debug Mode + +For development and debugging: + +```bash +# Build in debug mode +./run_benchmark.sh --debug --verbose + +# Enable detailed logging +dotnet run --configuration Debug -- --verbose +``` + +### Profiling + +For performance analysis: + +```bash +# Install profiling tools +dotnet tool install --global dotnet-trace +dotnet tool install --global dotnet-counters + +# Profile the benchmark +dotnet trace collect --process-id --providers Microsoft-DotNETCore-SampleProfiler +``` + +## Integration + +### CI/CD Integration + +```yaml +# GitHub Actions example +- name: Setup .NET + uses: actions/setup-dotnet@v3 + with: + dotnet-version: "8.0.x" + +- name: Run .NET Benchmarks + run: | + cd aws-encryption-sdk/esdk-performance-testing/benchmarks/net + ./run_benchmark.sh --quick + +- name: Upload Results + uses: actions/upload-artifact@v3 + with: + name: net-benchmark-results + path: aws-encryption-sdk/esdk-performance-testing/results/raw-data/net_results.json +``` + +### Automated Testing + +```bash +# Run unit tests (if any) +dotnet test + +# Run benchmarks with validation +dotnet run --configuration Release -- --quick --verbose +``` + +## Contributing + +When contributing to the .NET benchmark: + +1. **Code Style**: Follow .NET coding conventions and use EditorConfig +2. **Async Patterns**: Use proper async/await patterns, avoid blocking calls +3. **Memory Management**: Use spans and memory-efficient patterns +4. **Testing**: Add unit tests for new functionality +5. **Documentation**: Update this README for new features +6. **Performance**: Benchmark changes against baseline + +### Development Setup + +```bash +# Install development tools +dotnet tool install --global dotnet-format +dotnet tool install --global dotnet-outdated-tool + +# Format code +dotnet format + +# Check for outdated packages +dotnet outdated +``` + +## Dependencies + +### Core Dependencies + +- **CommandLineParser**: Command-line argument parsing +- **YamlDotNet**: YAML configuration file support +- **Newtonsoft.Json**: JSON serialization +- **ShellProgressBar**: Progress indication +- **MathNet.Numerics**: Statistical calculations + +### Logging and Monitoring + +- **Microsoft.Extensions.Logging**: Structured logging +- **System.Diagnostics.PerformanceCounter**: Performance monitoring +- **System.Management**: System information (Windows) + +### Development Dependencies + +- **.NET 8.0 SDK**: Required for building and running +- **NuGet Package Manager**: Dependency management + +## License + +This benchmark is part of the AWS Encryption SDK and is licensed under the Apache License 2.0. diff --git a/esdk-performance-testing/results/raw-data/net_results.json b/esdk-performance-testing/results/raw-data/net_results.json new file mode 100644 index 000000000..48a55b552 --- /dev/null +++ b/esdk-performance-testing/results/raw-data/net_results.json @@ -0,0 +1,78 @@ +{ + "metadata": { + "language": "net", + "timestamp": "2025-09-03T22:46:47.1519040Z", + "dotnet_version": "9.0.1", + "cpu_count": 12, + "total_memory_gb": 36.0, + "total_tests": 3 + }, + "results": [ + { + "TestName": "throughput", + "Language": "net", + "DataSize": 52428800, + "AlgorithmSuite": "AES_256_GCM_HKDF_SHA512_COMMIT_KEY_ECDSA_P384", + "FrameLength": null, + "Concurrency": 0, + "EncryptLatencyMs": 190.40603333333334, + "DecryptLatencyMs": 200.46193333333335, + "EndToEndLatencyMs": 404.4672333333333, + "OpsPerSecond": 2.4714642676108096, + "BytesPerSecond": 129575905.79371363, + "PeakMemoryMb": 0.0, + "MemoryEfficiencyRatio": 0.0, + "P50Latency": 407.8135, + "P95Latency": 412.3747, + "P99Latency": 412.78014, + "Timestamp": "2025-09-03 15:46:42", + "DotNetVersion": "9.0.1", + "CpuCount": 12, + "TotalMemoryGb": 36.0 + }, + { + "TestName": "memory", + "Language": "net", + "DataSize": 52428800, + "AlgorithmSuite": "AES_256_GCM_HKDF_SHA512_COMMIT_KEY_ECDSA_P384", + "FrameLength": null, + "Concurrency": 0, + "EncryptLatencyMs": 0.0, + "DecryptLatencyMs": 0.0, + "EndToEndLatencyMs": 0.0, + "OpsPerSecond": 0.0, + "BytesPerSecond": 0.0, + "PeakMemoryMb": 575.6221771240234, + "MemoryEfficiencyRatio": 0.08686253238159553, + "P50Latency": 0.0, + "P95Latency": 0.0, + "P99Latency": 0.0, + "Timestamp": "2025-09-03 15:46:44", + "DotNetVersion": "9.0.1", + "CpuCount": 12, + "TotalMemoryGb": 36.0 + }, + { + "TestName": "concurrent", + "Language": "net", + "DataSize": 52428800, + "AlgorithmSuite": "AES_256_GCM_HKDF_SHA512_COMMIT_KEY_ECDSA_P384", + "FrameLength": null, + "Concurrency": 2, + "EncryptLatencyMs": 0.0, + "DecryptLatencyMs": 0.0, + "EndToEndLatencyMs": 549.6251100000001, + "OpsPerSecond": 3.6257653265451273, + "BytesPerSecond": 190094525.15236917, + "PeakMemoryMb": 0.0, + "MemoryEfficiencyRatio": 0.0, + "P50Latency": 549.3668, + "P95Latency": 568.8644999999999, + "P99Latency": 571.30602, + "Timestamp": "2025-09-03 15:46:47", + "DotNetVersion": "9.0.1", + "CpuCount": 12, + "TotalMemoryGb": 36.0 + } + ] +} \ No newline at end of file From 2f50c601f8b04930dad8e6cfa3c8fe1fed1b66e2 Mon Sep 17 00:00:00 2001 From: Shubham Chaturvedi Date: Thu, 4 Sep 2025 00:43:45 -0700 Subject: [PATCH 2/5] fix: log formatting --- .../benchmarks/net/Program.cs | 358 +++++++++++------- 1 file changed, 213 insertions(+), 145 deletions(-) diff --git a/esdk-performance-testing/benchmarks/net/Program.cs b/esdk-performance-testing/benchmarks/net/Program.cs index 00b55d0b1..9b205fa8c 100644 --- a/esdk-performance-testing/benchmarks/net/Program.cs +++ b/esdk-performance-testing/benchmarks/net/Program.cs @@ -5,7 +5,6 @@ using System.Runtime; using System.Text.Json; using CommandLine; -using Microsoft.Extensions.Logging; using Newtonsoft.Json; using ShellProgressBar; using YamlDotNet.Serialization; @@ -26,71 +25,89 @@ namespace Amazon.Esdk.Benchmark; /// public class Program { - private static ILogger? _logger; - public static async Task Main(string[] args) + public static async Task Main(string[] args) { - return await Parser.Default.ParseArguments(args) - .MapResult( - async options => await RunBenchmarkAsync(options), - errors => Task.FromResult(1) - ); - } + var options = ParseArgs(args); + if (options == null) return; - private static async Task RunBenchmarkAsync(CommandLineOptions options) - { try { - // Initialize logging - using var loggerFactory = LoggerFactory.Create(builder => - { - builder.AddConsole(); - builder.SetMinimumLevel(options.Verbose ? LogLevel.Debug : LogLevel.Information); - }); - _logger = loggerFactory.CreateLogger(); - - // Initialize benchmark - var benchmark = new ESDKBenchmark(options.ConfigPath, _logger); - - // Adjust config for quick test + var benchmark = new ESDKBenchmark(options.ConfigPath); + if (options.QuickTest) { benchmark.AdjustForQuickTest(); } - // Run benchmarks - var results = await benchmark.RunAllBenchmarksAsync(); - - // Save results + var results = await benchmark.RunAllBenchmarksAsync(options.QuickTest); await benchmark.SaveResultsAsync(options.OutputPath); - - // Print summary - PrintSummary(results, options); - - return 0; + PrintSummary(results, options.OutputPath); } catch (Exception ex) { - Console.Error.WriteLine($"Benchmark failed: {ex.Message}"); - if (options.Verbose) + Console.WriteLine($"Benchmark failed: {ex.Message}"); + } + } + + private static CommandLineOptions? ParseArgs(string[] args) + { + var options = new CommandLineOptions + { + ConfigPath = "../../config/test-scenarios.yaml", + OutputPath = "../../results/raw-data/net_results.json" + }; + + for (int i = 0; i < args.Length; i++) + { + switch (args[i]) { - Console.Error.WriteLine(ex.StackTrace); + case "--config": + case "-c": + if (i + 1 < args.Length) options.ConfigPath = args[++i]; + break; + case "--output": + case "-o": + if (i + 1 < args.Length) options.OutputPath = args[++i]; + break; + case "--quick": + case "-q": + options.QuickTest = true; + break; + case "--help": + case "-h": + PrintHelp(); + return null; } - return 1; } + return options; } - private static void PrintSummary(List results, CommandLineOptions options) + private static void PrintHelp() + { + Console.WriteLine("ESDK .NET Benchmark"); + Console.WriteLine("Usage: EsdkBenchmark [options]"); + Console.WriteLine("Options:"); + Console.WriteLine(" --config, -c Path to test configuration file (default: ../../config/test-scenarios.yaml)"); + Console.WriteLine(" --output, -o Path to output results file (default: ../../results/raw-data/net_results.json)"); + Console.WriteLine(" --quick, -q Run quick test with reduced iterations"); + Console.WriteLine(" --help, -h Show this help message"); + } + + private static void PrintSummary(List results, string outputPath) { Console.WriteLine("\n=== ESDK .NET Benchmark Summary ==="); Console.WriteLine($"Total tests completed: {results.Count}"); - Console.WriteLine($"Results saved to: {options.OutputPath}"); + Console.WriteLine($"Results saved to: {outputPath}"); - var throughputResults = results.Where(r => r.TestName == "throughput").ToList(); - if (throughputResults.Any()) + if (results.Any()) { - var maxThroughput = throughputResults.Max(r => r.OpsPerSecond); - Console.WriteLine($"Maximum throughput: {maxThroughput:F2} ops/sec"); + var throughputResults = results.Where(r => r.TestName == "throughput").ToList(); + if (throughputResults.Any()) + { + var maxThroughput = throughputResults.Max(r => r.OpsPerSecond); + Console.WriteLine($"Maximum throughput: {maxThroughput:F2} ops/sec"); + } } } } @@ -121,33 +138,62 @@ public class CommandLineOptions /// public class BenchmarkResult { + [JsonProperty("test_name")] public string TestName { get; set; } = string.Empty; + + [JsonProperty("language")] public string Language { get; set; } = "net"; + + [JsonProperty("data_size")] public int DataSize { get; set; } - public string AlgorithmSuite { get; set; } = string.Empty; - public long? FrameLength { get; set; } + + [JsonProperty("concurrency")] public int Concurrency { get; set; } // Performance metrics + [JsonProperty("encrypt_latency_ms")] public double EncryptLatencyMs { get; set; } + + [JsonProperty("decrypt_latency_ms")] public double DecryptLatencyMs { get; set; } + + [JsonProperty("end_to_end_latency_ms")] public double EndToEndLatencyMs { get; set; } + + [JsonProperty("ops_per_second")] public double OpsPerSecond { get; set; } + + [JsonProperty("bytes_per_second")] public double BytesPerSecond { get; set; } // Memory metrics + [JsonProperty("peak_memory_mb")] public double PeakMemoryMb { get; set; } + + [JsonProperty("memory_efficiency_ratio")] public double MemoryEfficiencyRatio { get; set; } // Statistical metrics + [JsonProperty("p50_latency")] public double P50Latency { get; set; } + + [JsonProperty("p95_latency")] public double P95Latency { get; set; } + + [JsonProperty("p99_latency")] public double P99Latency { get; set; } // Environment info + [JsonProperty("timestamp")] public string Timestamp { get; set; } = string.Empty; + + [JsonProperty("dotnet_version")] public string DotNetVersion { get; set; } = string.Empty; + + [JsonProperty("cpu_count")] public int CpuCount { get; set; } + + [JsonProperty("total_memory_gb")] public double TotalMemoryGb { get; set; } } @@ -225,7 +271,6 @@ public class QuickIterationConfig /// public class ESDKBenchmark { - private readonly ILogger _logger; private TestConfig _config; private readonly List _results = new(); private readonly Random _random = new(); @@ -245,9 +290,8 @@ public class ESDKBenchmark private ESDK _encryptionSdk = null!; private IKeyring _keyring = null!; - public ESDKBenchmark(string configPath, ILogger logger) + public ESDKBenchmark(string configPath) { - _logger = logger; _config = LoadConfig(configPath); // Get system information @@ -264,8 +308,7 @@ public ESDKBenchmark(string configPath, ILogger logger) throw new InvalidOperationException($"Failed to setup ESDK: {setupError}"); } - _logger.LogInformation("Initialized ESDK Benchmark - CPU cores: {CpuCount}, Memory: {Memory:F1}GB", - _cpuCount, _totalMemoryGb); + Console.WriteLine($"Initialized ESDK Benchmark - CPU cores: {_cpuCount}, Memory: {_totalMemoryGb:F1}GB"); } private string? SetupEsdk() @@ -299,7 +342,7 @@ public ESDKBenchmark(string configPath, ILogger logger) }; _encryptionSdk = new ESDK(esdkConfig); - _logger.LogInformation("ESDK client initialized successfully"); + Console.WriteLine("ESDK client initialized successfully"); return null; } catch (Exception ex) @@ -315,8 +358,7 @@ private TestConfig LoadConfig(string configPath) { if (!File.Exists(configPath)) { - _logger.LogWarning("Config file not found, using default configuration"); - return CreateDefaultConfig(); + throw new FileNotFoundException($"Config file not found: {configPath}"); } try @@ -331,8 +373,7 @@ private TestConfig LoadConfig(string configPath) } catch (Exception ex) { - _logger.LogError(ex, "Failed to load config file, using default configuration"); - return CreateDefaultConfig(); + throw new InvalidOperationException($"Failed to load config file: {ex.Message}", ex); } } @@ -358,6 +399,21 @@ private static TestConfig CreateDefaultConfig() }; } + /// + /// Check if a test type should run based on configuration + /// + private bool ShouldRunTestType(string testType, bool isQuickMode) + { + if (isQuickMode) + { + if (_config.QuickConfig != null && _config.QuickConfig.TestTypes.Count > 0) + { + return _config.QuickConfig.TestTypes.Contains(testType); + } + } + return true; // Run all tests if not in quick mode or no test_types specified + } + /// /// Adjust configuration for quick test - uses quick_config from YAML /// @@ -370,7 +426,6 @@ public void AdjustForQuickTest() Warmup = _config.QuickConfig.Iterations.Warmup, Measurement = _config.QuickConfig.Iterations.Measurement }; - // Map QuickDataSizes to DataSizes _config.DataSizes = new DataSizes { Small = _config.QuickConfig.DataSizes.Small, @@ -379,6 +434,10 @@ public void AdjustForQuickTest() }; _config.ConcurrencyLevels = _config.QuickConfig.ConcurrencyLevels; } + else + { + throw new InvalidOperationException("Quick mode requested but no quick_config found in config file"); + } } /// @@ -474,9 +533,10 @@ private static double Percentile(List sortedValues, double percentile) /// /// Run throughput test matching Go/Rust implementation /// - public BenchmarkResult? RunThroughputTest(int dataSize, int iterations) + public BenchmarkResult? RunThroughputTest(int dataSize, int iterations, ProgressBar? progressBar = null) { - _logger.LogInformation("Running throughput test - Size: {DataSize} bytes, Iterations: {Iterations}", dataSize, iterations); + Action log = progressBar != null ? progressBar.WriteLine : Console.WriteLine; + log($"Running throughput test - Size: {dataSize} bytes, Iterations: {iterations}"); var testData = GenerateTestData(dataSize); @@ -486,7 +546,7 @@ private static double Percentile(List sortedValues, double percentile) var (_, _, error) = RunEncryptDecryptCycle(testData); if (error != null) { - _logger.LogError("Warmup iteration {Iteration} failed: {Error}", i, error); + Console.WriteLine($"Warmup iteration {i} failed: {error}"); return null; } } @@ -497,17 +557,6 @@ private static double Percentile(List sortedValues, double percentile) var endToEndLatencies = new List(); long totalBytes = 0; - var progressOptions = new ProgressBarOptions - { - ProgressCharacter = '█', - ProgressBarOnBottom = true, - ForegroundColor = ConsoleColor.Cyan, - BackgroundColor = ConsoleColor.DarkGray, - ForegroundColorDone = ConsoleColor.Green - }; - - using var progressBar = new ProgressBar(iterations, "Throughput test", progressOptions); - var startTime = Stopwatch.GetTimestamp(); for (int i = 0; i < iterations; i++) { @@ -515,7 +564,7 @@ private static double Percentile(List sortedValues, double percentile) var (encryptMs, decryptMs, error) = RunEncryptDecryptCycle(testData); if (error != null) { - _logger.LogError("Measurement iteration {Iteration} failed: {Error}", i, error); + Console.WriteLine($"Measurement iteration {i} failed: {error}"); continue; } var iterationDuration = Stopwatch.GetElapsedTime(iterationStart).TotalMilliseconds; @@ -524,14 +573,12 @@ private static double Percentile(List sortedValues, double percentile) decryptLatencies.Add(decryptMs); endToEndLatencies.Add(iterationDuration); totalBytes += dataSize; - - progressBar.Tick(); } var totalDuration = Stopwatch.GetElapsedTime(startTime).TotalSeconds; if (!encryptLatencies.Any()) { - _logger.LogError("All test iterations failed"); + Console.WriteLine("All test iterations failed"); return null; } @@ -542,7 +589,7 @@ private static double Percentile(List sortedValues, double percentile) TestName = "throughput", Language = "net", DataSize = dataSize, - AlgorithmSuite = "AES_256_GCM_HKDF_SHA512_COMMIT_KEY_ECDSA_P384", + Concurrency = 1, EncryptLatencyMs = Average(encryptLatencies), DecryptLatencyMs = Average(decryptLatencies), EndToEndLatencyMs = Average(endToEndLatencies), @@ -557,8 +604,7 @@ private static double Percentile(List sortedValues, double percentile) TotalMemoryGb = _totalMemoryGb }; - _logger.LogInformation("Throughput test completed - Ops/sec: {OpsPerSec:F2}, MB/sec: {MBPerSec:F2}", - result.OpsPerSecond, result.BytesPerSecond / (1024 * 1024)); + Console.WriteLine($"Throughput test completed - Ops/sec: {result.OpsPerSecond:F2}, MB/sec: {result.BytesPerSecond / (1024 * 1024):F2}"); return result; } @@ -569,10 +615,10 @@ private static double Percentile(List sortedValues, double percentile) /// /// Run memory test matching Go/Rust implementation /// - public BenchmarkResult? RunMemoryTest(int dataSize) + public BenchmarkResult? RunMemoryTest(int dataSize, ProgressBar? progressBar = null) { - _logger.LogInformation("Running memory test - Size: {DataSize} bytes ({Iterations} iterations, continuous sampling)", - dataSize, MemoryTestIterations); + Action log = progressBar != null ? progressBar.WriteLine : Console.WriteLine; + log($"Running memory test - Size: {dataSize} bytes ({MemoryTestIterations} iterations, continuous sampling)"); var data = GenerateTestData(dataSize); var peakHeap = 0.0; @@ -606,7 +652,7 @@ private static double Percentile(List sortedValues, double percentile) if (error != null) { - _logger.LogWarning("Memory test iteration {Iteration} failed: {Error}", i + 1, error); + Console.WriteLine($"Memory test iteration {i + 1} failed: {error}"); continue; } @@ -636,19 +682,23 @@ private static double Percentile(List sortedValues, double percentile) avgHeapValues.Add(iterAvgHeap); - _logger.LogDebug("Memory iteration {Iteration}: Peak={PeakMB:F2}MB, Avg={AvgMB:F2}MB, Duration={DurationMs:F2}ms", - i + 1, iterPeakHeap, iterAvgHeap, operationDuration.TotalMilliseconds); + log($"=== Iteration {i + 1} === Peak Heap: {iterPeakHeap:F2} MB, Total Allocs: {iterTotalAllocs:F2} MB, Avg Heap: {iterAvgHeap:F2} MB ({operationDuration.TotalMilliseconds:F0}ms, {continuousSamples.Count} samples)"); } var avgHeap = avgHeapValues.Count > 0 ? avgHeapValues.Average() : 0.0; var memoryEfficiency = dataSize > 0 ? (dataSize / (1024.0 * 1024.0)) / Math.Max(peakHeap, 1.0) : 0.0; + log("\nMemory Summary:"); + log($"- Absolute Peak Heap: {peakHeap:F2} MB (across all runs)"); + log($"- Average Heap: {avgHeap:F2} MB (across all runs)"); + log($"- Total Allocations: {peakAllocations:F2} MB (max across all runs)"); + var result = new BenchmarkResult { TestName = "memory", Language = "net", DataSize = dataSize, - AlgorithmSuite = "AES_256_GCM_HKDF_SHA512_COMMIT_KEY_ECDSA_P384", + Concurrency = 1, PeakMemoryMb = peakHeap, MemoryEfficiencyRatio = memoryEfficiency, Timestamp = DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss"), @@ -657,7 +707,7 @@ private static double Percentile(List sortedValues, double percentile) TotalMemoryGb = _totalMemoryGb }; - _logger.LogInformation("Memory test completed: {PeakMB:F2} MB peak", result.PeakMemoryMb); + Console.WriteLine($"Memory test completed: {result.PeakMemoryMb:F2} MB peak"); return result; } @@ -693,10 +743,9 @@ private void SampleMemoryContinuously(long baselineHeap, ulong baselineAllocs, C /// /// Run concurrent test matching Go/Rust implementation /// - public BenchmarkResult? RunConcurrentTest(int dataSize, int concurrency, int iterationsPerWorker) + public BenchmarkResult? RunConcurrentTest(int dataSize, int concurrency, int iterationsPerWorker, ProgressBar? progressBar = null) { - _logger.LogInformation("Running concurrent test - Size: {DataSize} bytes, Concurrency: {Concurrency}", - dataSize, concurrency); + Console.WriteLine($"Running concurrent test - Size: {dataSize} bytes, Concurrency: {concurrency}"); var data = GenerateTestData(dataSize); var allTimes = new List(); @@ -719,7 +768,7 @@ private void SampleMemoryContinuously(long baselineHeap, ulong baselineAllocs, C var (_, _, error) = RunEncryptDecryptCycle(data); if (error != null) { - _logger.LogWarning("Worker {WorkerID} iteration {Iteration} failed: {Error}", workerID, j, error); + Console.WriteLine($"Worker {workerID} iteration {j} failed: {error}"); errorOccurred = true; return; } @@ -738,7 +787,7 @@ private void SampleMemoryContinuously(long baselineHeap, ulong baselineAllocs, C if (errorOccurred || !allTimes.Any()) { - _logger.LogError("Concurrent test failed - no successful operations"); + Console.WriteLine("Concurrent test failed - no successful operations"); return null; } @@ -752,7 +801,6 @@ private void SampleMemoryContinuously(long baselineHeap, ulong baselineAllocs, C TestName = "concurrent", Language = "net", DataSize = dataSize, - AlgorithmSuite = "AES_256_GCM_HKDF_SHA512_COMMIT_KEY_ECDSA_P384", Concurrency = concurrency, EndToEndLatencyMs = Average(allTimes), OpsPerSecond = concurrentOpsPerSecond, @@ -766,8 +814,7 @@ private void SampleMemoryContinuously(long baselineHeap, ulong baselineAllocs, C TotalMemoryGb = _totalMemoryGb }; - _logger.LogInformation("Concurrent test completed: {OpsPerSec:F2} ops/sec @ {Concurrency} threads", - result.OpsPerSecond, concurrency); + Console.WriteLine($"Concurrent test completed: {result.OpsPerSecond:F2} ops/sec @ {concurrency} threads"); return result; } @@ -775,9 +822,9 @@ private void SampleMemoryContinuously(long baselineHeap, ulong baselineAllocs, C /// /// Run all configured benchmark tests - matches Java structure /// - public async Task> RunAllBenchmarksAsync() + public async Task> RunAllBenchmarksAsync(bool isQuickMode = false) { - _logger.LogInformation("Starting comprehensive ESDK benchmark suite"); + Console.WriteLine("Starting comprehensive ESDK benchmark suite"); var allResults = new List(); @@ -793,90 +840,111 @@ public async Task> RunAllBenchmarksAsync() var totalTests = dataSizes.Count * (concurrencyLevels.Count + 2); - _logger.LogInformation("Running {TotalTests} total tests", totalTests); + Console.WriteLine($"Running {totalTests} total tests"); - var overallProgressOptions = new ProgressBarOptions + var progressOptions = new ProgressBarOptions { ProgressCharacter = '█', ProgressBarOnBottom = true, ForegroundColor = ConsoleColor.Green, - BackgroundColor = ConsoleColor.DarkGray + BackgroundColor = ConsoleColor.DarkGray, + CollapseWhenFinished = false, + DisplayTimeInRealTime = true, + ShowEstimatedDuration = true }; - using var overallProgress = new ProgressBar(totalTests, "Running benchmarks", overallProgressOptions); + using var overallProgress = new ProgressBar(totalTests, "Overall Progress", progressOptions); + int completedTests = 0; // Throughput tests - foreach (var dataSize in dataSizes) + if (ShouldRunTestType("throughput", isQuickMode)) { - overallProgress.Message = $"Throughput test: {dataSize:N0} bytes"; - - try + overallProgress.WriteLine("Running throughput tests..."); + foreach (var dataSize in dataSizes) { - var result = RunThroughputTest(dataSize, iterations); - if (result != null) + try { - _logger.LogInformation("Throughput test completed: {OpsPerSecond:F2} ops/sec", result.OpsPerSecond); - allResults.Add(result); + var result = RunThroughputTest(dataSize, iterations, overallProgress); + if (result != null) + { + allResults.Add(result); + } } - } - catch (Exception ex) - { - _logger.LogError(ex, "Throughput test failed"); - } - - overallProgress.Tick(); - } - - // Memory tests - foreach (var dataSize in dataSizes) - { - overallProgress.Message = $"Memory test: {dataSize:N0} bytes"; - - try - { - var result = RunMemoryTest(dataSize); - if (result != null) + catch (Exception ex) { - _logger.LogInformation("Memory test completed: {PeakMemory:F2} MB peak", result.PeakMemoryMb); - allResults.Add(result); + overallProgress.WriteLine($"Throughput test failed: {ex.Message}"); } + + completedTests++; + overallProgress.Tick(); } - catch (Exception ex) - { - _logger.LogError(ex, "Memory test failed"); - } - - overallProgress.Tick(); + } + else + { + overallProgress.WriteLine("Skipping throughput tests (not in test_types)"); } - // Concurrent tests - foreach (var dataSize in dataSizes) + // Memory tests + if (ShouldRunTestType("memory", isQuickMode)) { - foreach (var concurrency in concurrencyLevels.Where(c => c > 1)) + overallProgress.WriteLine("Running memory tests..."); + foreach (var dataSize in dataSizes) { - overallProgress.Message = $"Concurrent test: {dataSize:N0} bytes @ {concurrency} threads"; - try { - var result = RunConcurrentTest(dataSize, concurrency, 5); + var result = RunMemoryTest(dataSize, overallProgress); if (result != null) { - _logger.LogInformation("Concurrent test completed: {OpsPerSecond:F2} ops/sec @ {Concurrency} threads", - result.OpsPerSecond, concurrency); allResults.Add(result); } } catch (Exception ex) { - _logger.LogError(ex, "Concurrent test failed"); + overallProgress.WriteLine($"Memory test failed: {ex.Message}"); } + completedTests++; overallProgress.Tick(); } } + else + { + overallProgress.WriteLine("Skipping memory tests (not in test_types)"); + } + + // Concurrent tests + if (ShouldRunTestType("concurrency", isQuickMode)) + { + overallProgress.WriteLine("Running concurrency tests..."); + foreach (var dataSize in dataSizes) + { + foreach (var concurrency in concurrencyLevels.Where(c => c > 1)) + { + try + { + var result = RunConcurrentTest(dataSize, concurrency, 5, overallProgress); + if (result != null) + { + allResults.Add(result); + } + } + catch (Exception ex) + { + overallProgress.WriteLine($"Concurrent test failed: {ex.Message}"); + } + + completedTests++; + overallProgress.Tick(); + } + } + } + else + { + overallProgress.WriteLine("Skipping concurrency tests (not in test_types)"); + } _results.AddRange(allResults); - _logger.LogInformation("Benchmark suite completed. Total results: {ResultCount}", allResults.Count); + Console.WriteLine($"Benchmark suite completed. Total results: {allResults.Count}"); return allResults; } @@ -892,13 +960,13 @@ public async Task SaveResultsAsync(string outputPath) Directory.CreateDirectory(outputDir); } - // Prepare results data - matches Java structure + // Prepare results data - matches other languages structure var resultsData = new { metadata = new { language = "net", - timestamp = DateTime.UtcNow.ToString("O"), + timestamp = DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss"), dotnet_version = Environment.Version.ToString(), cpu_count = _cpuCount, total_memory_gb = _totalMemoryGb, @@ -911,7 +979,7 @@ public async Task SaveResultsAsync(string outputPath) var json = JsonConvert.SerializeObject(resultsData, Formatting.Indented); await File.WriteAllTextAsync(outputPath, json); - _logger.LogInformation("Results saved to {OutputPath}", outputPath); + Console.WriteLine($"Results saved to {outputPath}"); } /// From cd983069323fd310a601e4cb4fa62643ad9ff785 Mon Sep 17 00:00:00 2001 From: Shubham Chaturvedi Date: Thu, 4 Sep 2025 02:25:43 -0700 Subject: [PATCH 3/5] fix: refactoring --- .../benchmarks/net/Benchmark.cs | 166 +++ .../benchmarks/net/Config.cs | 132 ++ .../benchmarks/net/Program.cs | 994 +------------- .../benchmarks/net/README.md | 368 +---- .../benchmarks/net/Results.cs | 226 ++++ .../benchmarks/net/Tests.cs | 372 +++++ .../results/raw-data/net_results.json | 1202 ++++++++++++++++- 7 files changed, 2091 insertions(+), 1369 deletions(-) create mode 100644 esdk-performance-testing/benchmarks/net/Benchmark.cs create mode 100644 esdk-performance-testing/benchmarks/net/Config.cs create mode 100644 esdk-performance-testing/benchmarks/net/Results.cs create mode 100644 esdk-performance-testing/benchmarks/net/Tests.cs diff --git a/esdk-performance-testing/benchmarks/net/Benchmark.cs b/esdk-performance-testing/benchmarks/net/Benchmark.cs new file mode 100644 index 000000000..be6985a22 --- /dev/null +++ b/esdk-performance-testing/benchmarks/net/Benchmark.cs @@ -0,0 +1,166 @@ +// Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +using System.Diagnostics; +using System.Runtime; +using System.Text.Json; +using System.Security.Cryptography; +using Microsoft.Extensions.Logging; +using AWS.Cryptography.EncryptionSDK; +using AWS.Cryptography.MaterialProviders; +using YamlDotNet.Serialization; +using YamlDotNet.Serialization.NamingConventions; +using ShellProgressBar; +using Newtonsoft.Json; + +namespace EsdkBenchmark; + +public partial class ESDKBenchmark +{ + private readonly ILogger _logger; + private readonly TestConfig _config; + private readonly int _cpuCount; + private readonly double _totalMemoryGb; + private static readonly Random _random = new(); + private readonly List _results = new(); + + // Public properties to match Go structure + public TestConfig Config => _config; + public List Results => _results; + + // ESDK components + private MaterialProviders _materialProviders = null!; + private ESDK _encryptionSdk = null!; + private IKeyring _keyring = null!; + + // Constants for memory testing + private const int MemoryTestIterations = 5; + private const int SamplingIntervalMs = 1; + private const int GcSettleTimeMs = 5; + private const int FinalSampleWaitMs = 2; + + public ESDKBenchmark(string configPath, ILogger logger) + { + _logger = logger; + _config = ConfigLoader.LoadConfig(configPath); + + // Get system information + _cpuCount = Environment.ProcessorCount; + _totalMemoryGb = GetTotalMemoryGb(); + + // Configure GC for performance + GCSettings.LatencyMode = GCLatencyMode.SustainedLowLatency; + + // Setup ESDK + var setupError = SetupEsdk(); + if (setupError != null) + { + throw new InvalidOperationException($"Failed to setup ESDK: {setupError}"); + } + + _logger.LogInformation("Initialized ESDK Benchmark - CPU cores: {CpuCount}, Memory: {Memory:F1}GB", + _cpuCount, _totalMemoryGb); + } + + private string? SetupEsdk() + { + try + { + _materialProviders = new MaterialProviders(new MaterialProvidersConfig()); + + // Create 256-bit AES key using .NET crypto + var key = new byte[32]; + using (var rng = RandomNumberGenerator.Create()) + { + rng.GetBytes(key); + } + + // Create raw AES keyring + var createKeyringInput = new CreateRawAesKeyringInput + { + KeyNamespace = "esdk-performance-test", + KeyName = "test-aes-256-key", + WrappingKey = new MemoryStream(key), + WrappingAlg = AesWrappingAlg.ALG_AES256_GCM_IV12_TAG16 + }; + _keyring = _materialProviders.CreateRawAesKeyring(createKeyringInput); + + // Create ESDK client with commitment policy + var esdkConfig = new AwsEncryptionSdkConfig + { + CommitmentPolicy = ESDKCommitmentPolicy.REQUIRE_ENCRYPT_REQUIRE_DECRYPT + }; + _encryptionSdk = new ESDK(esdkConfig); + + return null; + } + catch (Exception ex) + { + return ex.Message; + } + } + + public static double GetTotalMemoryGb() + { + try + { + var gcMemoryInfo = GC.GetGCMemoryInfo(); + return gcMemoryInfo.TotalAvailableMemoryBytes / (1024.0 * 1024.0 * 1024.0); + } + catch + { + // Fallback - estimate based on process memory + return Environment.WorkingSet / (1024.0 * 1024.0 * 1024.0) * 4; // Rough estimate + } + } + + private byte[] GenerateTestData(int size) + { + var data = new byte[size]; + _random.NextBytes(data); + return data; + } + + + + public void RunAllBenchmarks() + { + _results.Clear(); + Console.WriteLine("Starting comprehensive ESDK benchmark suite"); + + // Combine all data sizes + var dataSizes = _config.DataSizes.Small + .Concat(_config.DataSizes.Medium) + .Concat(_config.DataSizes.Large); + + // Run test suites + if (ConfigLoader.ShouldRunTestType(_config, "throughput")) + { + RunThroughputTests(dataSizes, _config.Iterations.Measurement); + } + else + { + Console.WriteLine("Skipping throughput tests (not in test_types)"); + } + + if (ConfigLoader.ShouldRunTestType(_config, "memory")) + { + RunMemoryTests(dataSizes); + } + else + { + Console.WriteLine("Skipping memory tests (not in test_types)"); + } + + if (ConfigLoader.ShouldRunTestType(_config, "concurrency")) + { + RunConcurrencyTests(dataSizes, _config.ConcurrencyLevels); + } + else + { + Console.WriteLine("Skipping concurrency tests (not in test_types)"); + } + + Console.WriteLine($"Benchmark suite completed. Total results: {_results.Count}"); + } +} diff --git a/esdk-performance-testing/benchmarks/net/Config.cs b/esdk-performance-testing/benchmarks/net/Config.cs new file mode 100644 index 000000000..78243545b --- /dev/null +++ b/esdk-performance-testing/benchmarks/net/Config.cs @@ -0,0 +1,132 @@ +// Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +using YamlDotNet.Serialization; +using YamlDotNet.Serialization.NamingConventions; + +namespace EsdkBenchmark; + +public class TestConfig +{ + [YamlMember(Alias = "data_sizes")] + public DataSizes DataSizes { get; set; } = new(); + + [YamlMember(Alias = "iterations")] + public IterationConfig Iterations { get; set; } = new(); + + [YamlMember(Alias = "concurrency_levels")] + public List ConcurrencyLevels { get; set; } = new(); + + [YamlMember(Alias = "quick_config")] + public QuickConfig? QuickConfig { get; set; } +} + +public class DataSizes +{ + [YamlMember(Alias = "small")] + public List Small { get; set; } = new(); + + [YamlMember(Alias = "medium")] + public List Medium { get; set; } = new(); + + [YamlMember(Alias = "large")] + public List Large { get; set; } = new(); +} + +public class IterationConfig +{ + [YamlMember(Alias = "warmup")] + public int Warmup { get; set; } + + [YamlMember(Alias = "measurement")] + public int Measurement { get; set; } +} + +public class QuickConfig +{ + [YamlMember(Alias = "data_sizes")] + public QuickDataSizes DataSizes { get; set; } = new(); + + [YamlMember(Alias = "iterations")] + public QuickIterationConfig Iterations { get; set; } = new(); + + [YamlMember(Alias = "concurrency_levels")] + public List ConcurrencyLevels { get; set; } = new(); + + [YamlMember(Alias = "test_types")] + public List TestTypes { get; set; } = new(); +} + +public class QuickDataSizes +{ + [YamlMember(Alias = "small")] + public List Small { get; set; } = new(); +} + +public class QuickIterationConfig +{ + [YamlMember(Alias = "warmup")] + public int Warmup { get; set; } + + [YamlMember(Alias = "measurement")] + public int Measurement { get; set; } +} + +public static class ConfigLoader +{ + public static TestConfig LoadConfig(string configPath) + { + if (!File.Exists(configPath)) + { + throw new FileNotFoundException($"Config file not found: {configPath}"); + } + + try + { + var yaml = File.ReadAllText(configPath); + var deserializer = new DeserializerBuilder() + .WithNamingConvention(UnderscoredNamingConvention.Instance) + .IgnoreUnmatchedProperties() + .Build(); + + return deserializer.Deserialize(yaml); + } + catch (Exception ex) + { + throw new InvalidOperationException($"Failed to load config file: {ex.Message}", ex); + } + } + + public static void AdjustForQuickTest(TestConfig config) + { + if (config.QuickConfig != null) + { + if (config.QuickConfig.DataSizes != null) + { + config.DataSizes.Small = config.QuickConfig.DataSizes.Small; + config.DataSizes.Medium = new List(); + config.DataSizes.Large = new List(); + } + + if (config.QuickConfig.Iterations != null) + { + config.Iterations.Warmup = config.QuickConfig.Iterations.Warmup; + config.Iterations.Measurement = config.QuickConfig.Iterations.Measurement; + } + + if (config.QuickConfig.ConcurrencyLevels.Count > 0) + { + config.ConcurrencyLevels = config.QuickConfig.ConcurrencyLevels; + } + } + } + + public static bool ShouldRunTestType(TestConfig config, string testType) + { + if (config.QuickConfig != null && config.QuickConfig.TestTypes.Count > 0) + { + return config.QuickConfig.TestTypes.Contains(testType); + } + return true; + } +} \ No newline at end of file diff --git a/esdk-performance-testing/benchmarks/net/Program.cs b/esdk-performance-testing/benchmarks/net/Program.cs index 9b205fa8c..7e4eaa237 100644 --- a/esdk-performance-testing/benchmarks/net/Program.cs +++ b/esdk-performance-testing/benchmarks/net/Program.cs @@ -1,121 +1,12 @@ // Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. // SPDX-License-Identifier: Apache-2.0 -using System.Diagnostics; -using System.Runtime; -using System.Text.Json; +using Microsoft.Extensions.Logging; using CommandLine; -using Newtonsoft.Json; -using ShellProgressBar; -using YamlDotNet.Serialization; -using YamlDotNet.Serialization.NamingConventions; -using MathNet.Numerics.Statistics; -using AWS.Cryptography.EncryptionSDK; -using AWS.Cryptography.MaterialProviders; -using System.Security.Cryptography; -namespace Amazon.Esdk.Benchmark; +namespace EsdkBenchmark; -/// -/// ESDK Performance Benchmark Suite - .NET Implementation -/// -/// This application provides comprehensive performance testing for the AWS Encryption SDK (ESDK) -/// .NET runtime, measuring throughput, latency, memory usage, and scalability. -/// Follows the same structure and configuration approach as the Java implementation. -/// -public class Program -{ - - public static async Task Main(string[] args) - { - var options = ParseArgs(args); - if (options == null) return; - - try - { - var benchmark = new ESDKBenchmark(options.ConfigPath); - - if (options.QuickTest) - { - benchmark.AdjustForQuickTest(); - } - - var results = await benchmark.RunAllBenchmarksAsync(options.QuickTest); - await benchmark.SaveResultsAsync(options.OutputPath); - PrintSummary(results, options.OutputPath); - } - catch (Exception ex) - { - Console.WriteLine($"Benchmark failed: {ex.Message}"); - } - } - - private static CommandLineOptions? ParseArgs(string[] args) - { - var options = new CommandLineOptions - { - ConfigPath = "../../config/test-scenarios.yaml", - OutputPath = "../../results/raw-data/net_results.json" - }; - - for (int i = 0; i < args.Length; i++) - { - switch (args[i]) - { - case "--config": - case "-c": - if (i + 1 < args.Length) options.ConfigPath = args[++i]; - break; - case "--output": - case "-o": - if (i + 1 < args.Length) options.OutputPath = args[++i]; - break; - case "--quick": - case "-q": - options.QuickTest = true; - break; - case "--help": - case "-h": - PrintHelp(); - return null; - } - } - return options; - } - - private static void PrintHelp() - { - Console.WriteLine("ESDK .NET Benchmark"); - Console.WriteLine("Usage: EsdkBenchmark [options]"); - Console.WriteLine("Options:"); - Console.WriteLine(" --config, -c Path to test configuration file (default: ../../config/test-scenarios.yaml)"); - Console.WriteLine(" --output, -o Path to output results file (default: ../../results/raw-data/net_results.json)"); - Console.WriteLine(" --quick, -q Run quick test with reduced iterations"); - Console.WriteLine(" --help, -h Show this help message"); - } - - private static void PrintSummary(List results, string outputPath) - { - Console.WriteLine("\n=== ESDK .NET Benchmark Summary ==="); - Console.WriteLine($"Total tests completed: {results.Count}"); - Console.WriteLine($"Results saved to: {outputPath}"); - - if (results.Any()) - { - var throughputResults = results.Where(r => r.TestName == "throughput").ToList(); - if (throughputResults.Any()) - { - var maxThroughput = throughputResults.Max(r => r.OpsPerSecond); - Console.WriteLine($"Maximum throughput: {maxThroughput:F2} ops/sec"); - } - } - } -} - -/// -/// Command line options for the benchmark -/// -public class CommandLineOptions +public class Options { [Option('c', "config", Default = "../../config/test-scenarios.yaml", HelpText = "Path to test configuration file")] public string ConfigPath { get; set; } = string.Empty; @@ -124,879 +15,66 @@ public class CommandLineOptions public string OutputPath { get; set; } = string.Empty; [Option('q', "quick", Default = false, HelpText = "Run quick test with reduced iterations")] - public bool QuickTest { get; set; } - - [Option('v', "verbose", Default = false, HelpText = "Enable verbose logging")] - public bool Verbose { get; set; } - - [Option('h', "help", Default = false, HelpText = "Show help message")] - public bool Help { get; set; } -} - -/// -/// Benchmark result for a single test -/// -public class BenchmarkResult -{ - [JsonProperty("test_name")] - public string TestName { get; set; } = string.Empty; - - [JsonProperty("language")] - public string Language { get; set; } = "net"; - - [JsonProperty("data_size")] - public int DataSize { get; set; } - - [JsonProperty("concurrency")] - public int Concurrency { get; set; } - - // Performance metrics - [JsonProperty("encrypt_latency_ms")] - public double EncryptLatencyMs { get; set; } - - [JsonProperty("decrypt_latency_ms")] - public double DecryptLatencyMs { get; set; } - - [JsonProperty("end_to_end_latency_ms")] - public double EndToEndLatencyMs { get; set; } - - [JsonProperty("ops_per_second")] - public double OpsPerSecond { get; set; } - - [JsonProperty("bytes_per_second")] - public double BytesPerSecond { get; set; } - - // Memory metrics - [JsonProperty("peak_memory_mb")] - public double PeakMemoryMb { get; set; } - - [JsonProperty("memory_efficiency_ratio")] - public double MemoryEfficiencyRatio { get; set; } - - // Statistical metrics - [JsonProperty("p50_latency")] - public double P50Latency { get; set; } - - [JsonProperty("p95_latency")] - public double P95Latency { get; set; } - - [JsonProperty("p99_latency")] - public double P99Latency { get; set; } - - // Environment info - [JsonProperty("timestamp")] - public string Timestamp { get; set; } = string.Empty; - - [JsonProperty("dotnet_version")] - public string DotNetVersion { get; set; } = string.Empty; - - [JsonProperty("cpu_count")] - public int CpuCount { get; set; } - - [JsonProperty("total_memory_gb")] - public double TotalMemoryGb { get; set; } -} - -/// -/// Test configuration loaded from YAML - matches YAML structure -/// -public class TestConfig -{ - [YamlMember(Alias = "data_sizes")] - public DataSizes DataSizes { get; set; } = new(); - - [YamlMember(Alias = "iterations")] - public IterationConfig Iterations { get; set; } = new(); - - [YamlMember(Alias = "concurrency_levels")] - public List ConcurrencyLevels { get; set; } = new(); - - [YamlMember(Alias = "quick_config")] - public QuickConfig? QuickConfig { get; set; } -} - -public class DataSizes -{ - [YamlMember(Alias = "small")] - public List Small { get; set; } = new(); - - [YamlMember(Alias = "medium")] - public List Medium { get; set; } = new(); - - [YamlMember(Alias = "large")] - public List Large { get; set; } = new(); -} - -public class IterationConfig -{ - [YamlMember(Alias = "warmup")] - public int Warmup { get; set; } - - [YamlMember(Alias = "measurement")] - public int Measurement { get; set; } -} - -public class QuickConfig -{ - [YamlMember(Alias = "data_sizes")] - public QuickDataSizes DataSizes { get; set; } = new(); - - [YamlMember(Alias = "iterations")] - public QuickIterationConfig Iterations { get; set; } = new(); - - [YamlMember(Alias = "concurrency_levels")] - public List ConcurrencyLevels { get; set; } = new(); - - [YamlMember(Alias = "test_types")] - public List TestTypes { get; set; } = new(); -} - -public class QuickDataSizes -{ - [YamlMember(Alias = "small")] - public List Small { get; set; } = new(); -} - -public class QuickIterationConfig -{ - [YamlMember(Alias = "warmup")] - public int Warmup { get; set; } - - [YamlMember(Alias = "measurement")] - public int Measurement { get; set; } + public bool Quick { get; set; } } -/// -/// Main benchmark class - follows Go/Rust implementation structure -/// -public class ESDKBenchmark +public class Program { - private TestConfig _config; - private readonly List _results = new(); - private readonly Random _random = new(); - - // Constants for memory testing - private const int MemoryTestIterations = 5; - private const int SamplingIntervalMs = 1; - private const int GcSettleTimeMs = 5; - private const int FinalSampleWaitMs = 2; - - // System information - private readonly int _cpuCount; - private readonly double _totalMemoryGb; - - // ESDK components - private MaterialProviders _materialProviders = null!; - private ESDK _encryptionSdk = null!; - private IKeyring _keyring = null!; - - public ESDKBenchmark(string configPath) - { - _config = LoadConfig(configPath); - - // Get system information - _cpuCount = Environment.ProcessorCount; - _totalMemoryGb = GetTotalMemoryGb(); - - // Configure GC for performance - GCSettings.LatencyMode = GCLatencyMode.SustainedLowLatency; - - // Setup ESDK - var setupError = SetupEsdk(); - if (setupError != null) - { - throw new InvalidOperationException($"Failed to setup ESDK: {setupError}"); - } - - Console.WriteLine($"Initialized ESDK Benchmark - CPU cores: {_cpuCount}, Memory: {_totalMemoryGb:F1}GB"); - } - - private string? SetupEsdk() - { - try - { - // Initialize material providers client - _materialProviders = new MaterialProviders(new MaterialProvidersConfig()); - - // Create 256-bit AES key using .NET crypto - var key = new byte[32]; - using (var rng = RandomNumberGenerator.Create()) - { - rng.GetBytes(key); - } - - // Create raw AES keyring - var createKeyringInput = new CreateRawAesKeyringInput - { - KeyNamespace = "esdk-performance-test", - KeyName = "test-aes-256-key", - WrappingKey = new MemoryStream(key), - WrappingAlg = AesWrappingAlg.ALG_AES256_GCM_IV12_TAG16 - }; - _keyring = _materialProviders.CreateRawAesKeyring(createKeyringInput); - - // Create ESDK client with commitment policy - var esdkConfig = new AwsEncryptionSdkConfig - { - CommitmentPolicy = ESDKCommitmentPolicy.REQUIRE_ENCRYPT_REQUIRE_DECRYPT - }; - _encryptionSdk = new ESDK(esdkConfig); - - Console.WriteLine("ESDK client initialized successfully"); - return null; - } - catch (Exception ex) - { - return ex.Message; - } - } - - /// - /// Load test configuration from YAML file - matches Java approach - /// - private TestConfig LoadConfig(string configPath) - { - if (!File.Exists(configPath)) - { - throw new FileNotFoundException($"Config file not found: {configPath}"); - } - - try - { - var yaml = File.ReadAllText(configPath); - var deserializer = new DeserializerBuilder() - .WithNamingConvention(UnderscoredNamingConvention.Instance) - .IgnoreUnmatchedProperties() - .Build(); - - return deserializer.Deserialize(yaml); - } - catch (Exception ex) - { - throw new InvalidOperationException($"Failed to load config file: {ex.Message}", ex); - } - } - - /// - /// Create default configuration - matches Java defaults - /// - private static TestConfig CreateDefaultConfig() - { - return new TestConfig - { - DataSizes = new DataSizes - { - Small = new() { 1024, 5120, 10240 }, - Medium = new() { 102400, 512000, 1048576 }, - Large = new() { 10485760, 52428800, 104857600 } - }, - Iterations = new IterationConfig - { - Warmup = 5, - Measurement = 10 - }, - ConcurrencyLevels = new() { 1, 2, 4, 8 } - }; - } - - /// - /// Check if a test type should run based on configuration - /// - private bool ShouldRunTestType(string testType, bool isQuickMode) - { - if (isQuickMode) - { - if (_config.QuickConfig != null && _config.QuickConfig.TestTypes.Count > 0) - { - return _config.QuickConfig.TestTypes.Contains(testType); - } - } - return true; // Run all tests if not in quick mode or no test_types specified - } - - /// - /// Adjust configuration for quick test - uses quick_config from YAML - /// - public void AdjustForQuickTest() - { - if (_config.QuickConfig != null) - { - _config.Iterations = new IterationConfig - { - Warmup = _config.QuickConfig.Iterations.Warmup, - Measurement = _config.QuickConfig.Iterations.Measurement - }; - _config.DataSizes = new DataSizes - { - Small = _config.QuickConfig.DataSizes.Small, - Medium = new List(), - Large = new List() - }; - _config.ConcurrencyLevels = _config.QuickConfig.ConcurrencyLevels; - } - else - { - throw new InvalidOperationException("Quick mode requested but no quick_config found in config file"); - } - } - - /// - /// Generate test data of specified size - /// - private byte[] GenerateTestData(int size) - { - var data = new byte[size]; - _random.NextBytes(data); - return data; - } - - /// - /// Calculate average of a list of values - /// - private static double Average(List values) - { - return values.Count > 0 ? values.Average() : 0.0; - } - - /// - /// Calculate percentile of a sorted list of values - /// - private static double Percentile(List sortedValues, double percentile) - { - if (sortedValues.Count == 0) return 0.0; - - var index = percentile * (sortedValues.Count - 1); - var lower = (int)Math.Floor(index); - var upper = (int)Math.Ceiling(index); - - if (lower == upper) - return sortedValues[lower]; - - var weight = index - lower; - return sortedValues[lower] * (1 - weight) + sortedValues[upper] * weight; - } - - /// - /// Run encrypt-decrypt cycle matching Go/Rust implementation - /// - private (double encryptMs, double decryptMs, string? error) RunEncryptDecryptCycle(byte[] data) - { - try - { - var plaintext = new MemoryStream(data); - - // Create encryption context matching Go/Rust - var encryptionContext = new Dictionary() - { - {"purpose", "performance-test"}, - {"size", data.Length.ToString()} - }; - - // Encrypt - var encryptStart = Stopwatch.GetTimestamp(); - var encryptInput = new EncryptInput - { - Plaintext = plaintext, - Keyring = _keyring, - EncryptionContext = encryptionContext - }; - var encryptOutput = _encryptionSdk.Encrypt(encryptInput); - var encryptMs = Stopwatch.GetElapsedTime(encryptStart).TotalMilliseconds; - - // Decrypt - var decryptStart = Stopwatch.GetTimestamp(); - var decryptInput = new DecryptInput - { - Ciphertext = encryptOutput.Ciphertext, - Keyring = _keyring - }; - var decryptOutput = _encryptionSdk.Decrypt(decryptInput); - var decryptMs = Stopwatch.GetElapsedTime(decryptStart).TotalMilliseconds; - - // Verify data integrity - if (!data.SequenceEqual(decryptOutput.Plaintext.ToArray())) - { - return (0, 0, "data integrity check failed"); - } - - return (encryptMs, decryptMs, null); - } - catch (Exception ex) - { - return (0, 0, ex.Message); - } - } - - /// - /// Run throughput benchmark test - matches Java structure - /// - /// - /// Run throughput test matching Go/Rust implementation - /// - public BenchmarkResult? RunThroughputTest(int dataSize, int iterations, ProgressBar? progressBar = null) - { - Action log = progressBar != null ? progressBar.WriteLine : Console.WriteLine; - log($"Running throughput test - Size: {dataSize} bytes, Iterations: {iterations}"); - - var testData = GenerateTestData(dataSize); - - // Warmup - for (int i = 0; i < _config.Iterations.Warmup; i++) - { - var (_, _, error) = RunEncryptDecryptCycle(testData); - if (error != null) - { - Console.WriteLine($"Warmup iteration {i} failed: {error}"); - return null; - } - } - - // Measurement runs - var encryptLatencies = new List(); - var decryptLatencies = new List(); - var endToEndLatencies = new List(); - long totalBytes = 0; - - var startTime = Stopwatch.GetTimestamp(); - for (int i = 0; i < iterations; i++) - { - var iterationStart = Stopwatch.GetTimestamp(); - var (encryptMs, decryptMs, error) = RunEncryptDecryptCycle(testData); - if (error != null) - { - Console.WriteLine($"Measurement iteration {i} failed: {error}"); - continue; - } - var iterationDuration = Stopwatch.GetElapsedTime(iterationStart).TotalMilliseconds; - - encryptLatencies.Add(encryptMs); - decryptLatencies.Add(decryptMs); - endToEndLatencies.Add(iterationDuration); - totalBytes += dataSize; - } - var totalDuration = Stopwatch.GetElapsedTime(startTime).TotalSeconds; - - if (!encryptLatencies.Any()) - { - Console.WriteLine("All test iterations failed"); - return null; - } - - // Calculate metrics - endToEndLatencies.Sort(); - var result = new BenchmarkResult - { - TestName = "throughput", - Language = "net", - DataSize = dataSize, - Concurrency = 1, - EncryptLatencyMs = Average(encryptLatencies), - DecryptLatencyMs = Average(decryptLatencies), - EndToEndLatencyMs = Average(endToEndLatencies), - OpsPerSecond = iterations / totalDuration, - BytesPerSecond = totalBytes / totalDuration, - P50Latency = Percentile(endToEndLatencies, 0.50), - P95Latency = Percentile(endToEndLatencies, 0.95), - P99Latency = Percentile(endToEndLatencies, 0.99), - Timestamp = DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss"), - DotNetVersion = Environment.Version.ToString(), - CpuCount = _cpuCount, - TotalMemoryGb = _totalMemoryGb - }; - - Console.WriteLine($"Throughput test completed - Ops/sec: {result.OpsPerSecond:F2}, MB/sec: {result.BytesPerSecond / (1024 * 1024):F2}"); - - return result; - } - - /// - /// Run memory usage benchmark test - matches Java structure - /// - /// - /// Run memory test matching Go/Rust implementation - /// - public BenchmarkResult? RunMemoryTest(int dataSize, ProgressBar? progressBar = null) + public static async Task Main(string[] args) { - Action log = progressBar != null ? progressBar.WriteLine : Console.WriteLine; - log($"Running memory test - Size: {dataSize} bytes ({MemoryTestIterations} iterations, continuous sampling)"); - - var data = GenerateTestData(dataSize); - var peakHeap = 0.0; - var peakAllocations = 0.0; - var avgHeapValues = new List(); - - // Run iterations - for (int i = 0; i < MemoryTestIterations; i++) - { - GC.Collect(); - GC.WaitForPendingFinalizers(); - GC.Collect(); - Thread.Sleep(GcSettleTimeMs); - - // Get baseline - var beforeHeap = GC.GetTotalMemory(false); - var beforeAllocated = (long)GC.GetTotalAllocatedBytes(false); - - // Start continuous sampling - var stopSampling = new CancellationTokenSource(); - var continuousSamples = new List<(double HeapMB, double AllocsMB)>(); - var samplingTask = Task.Run(() => SampleMemoryContinuously(beforeHeap, (ulong)beforeAllocated, stopSampling.Token, continuousSamples)); - - // Run operation - var operationStart = Stopwatch.GetTimestamp(); - var (_, _, error) = RunEncryptDecryptCycle(data); - var operationDuration = Stopwatch.GetElapsedTime(operationStart); - - stopSampling.Cancel(); - Thread.Sleep(FinalSampleWaitMs); - - if (error != null) - { - Console.WriteLine($"Memory test iteration {i + 1} failed: {error}"); - continue; - } - - // Analyze samples - var iterPeakHeap = 0.0; - var iterTotalAllocs = 0.0; - var iterAvgHeap = 0.0; - - if (continuousSamples.Count > 0) - { - var heapSum = 0.0; - foreach (var sample in continuousSamples) - { - if (sample.HeapMB > iterPeakHeap) - iterPeakHeap = sample.HeapMB; - if (sample.AllocsMB > iterTotalAllocs) - iterTotalAllocs = sample.AllocsMB; - heapSum += sample.HeapMB; - } - iterAvgHeap = heapSum / continuousSamples.Count; - } - - if (iterPeakHeap > peakHeap) - peakHeap = iterPeakHeap; - if (iterTotalAllocs > peakAllocations) - peakAllocations = iterTotalAllocs; - - avgHeapValues.Add(iterAvgHeap); - - log($"=== Iteration {i + 1} === Peak Heap: {iterPeakHeap:F2} MB, Total Allocs: {iterTotalAllocs:F2} MB, Avg Heap: {iterAvgHeap:F2} MB ({operationDuration.TotalMilliseconds:F0}ms, {continuousSamples.Count} samples)"); - } + var result = Parser.Default.ParseArguments(args); - var avgHeap = avgHeapValues.Count > 0 ? avgHeapValues.Average() : 0.0; - var memoryEfficiency = dataSize > 0 ? (dataSize / (1024.0 * 1024.0)) / Math.Max(peakHeap, 1.0) : 0.0; - - log("\nMemory Summary:"); - log($"- Absolute Peak Heap: {peakHeap:F2} MB (across all runs)"); - log($"- Average Heap: {avgHeap:F2} MB (across all runs)"); - log($"- Total Allocations: {peakAllocations:F2} MB (max across all runs)"); - - var result = new BenchmarkResult - { - TestName = "memory", - Language = "net", - DataSize = dataSize, - Concurrency = 1, - PeakMemoryMb = peakHeap, - MemoryEfficiencyRatio = memoryEfficiency, - Timestamp = DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss"), - DotNetVersion = Environment.Version.ToString(), - CpuCount = _cpuCount, - TotalMemoryGb = _totalMemoryGb - }; - - Console.WriteLine($"Memory test completed: {result.PeakMemoryMb:F2} MB peak"); - return result; - } - - private void SampleMemoryContinuously(long baselineHeap, ulong baselineAllocs, CancellationToken cancellationToken, List<(double HeapMB, double AllocsMB)> samples) - { - while (!cancellationToken.IsCancellationRequested) + await result.WithParsedAsync(async options => { try { - var currentHeap = GC.GetTotalMemory(false); - var currentAllocs = GC.GetTotalAllocatedBytes(false); - - var heapMB = (currentHeap - baselineHeap) / (1024.0 * 1024.0); - var allocsMB = ((long)currentAllocs - (long)baselineAllocs) / (1024.0 * 1024.0); - - lock (samples) - { - samples.Add((Math.Max(0, heapMB), Math.Max(0, allocsMB))); - } - - Thread.Sleep(SamplingIntervalMs); - } - catch (OperationCanceledException) - { - break; - } - } - } + // Initialize benchmark + using var loggerFactory = LoggerFactory.Create(builder => builder.AddConsole()); + var logger = loggerFactory.CreateLogger(); + var bench = new ESDKBenchmark(options.ConfigPath, logger); - /// - /// Run concurrent operations benchmark test - matches Java structure - /// - /// - /// Run concurrent test matching Go/Rust implementation - /// - public BenchmarkResult? RunConcurrentTest(int dataSize, int concurrency, int iterationsPerWorker, ProgressBar? progressBar = null) - { - Console.WriteLine($"Running concurrent test - Size: {dataSize} bytes, Concurrency: {concurrency}"); - - var data = GenerateTestData(dataSize); - var allTimes = new List(); - var timesMutex = new object(); - var errorOccurred = false; - - var startTime = Stopwatch.GetTimestamp(); - - // Launch workers - var tasks = new Task[concurrency]; - for (int i = 0; i < concurrency; i++) - { - int workerID = i; - tasks[i] = Task.Run(() => - { - var workerTimes = new List(); - for (int j = 0; j < iterationsPerWorker; j++) + // Adjust config for quick test + if (options.Quick) { - var iterStart = Stopwatch.GetTimestamp(); - var (_, _, error) = RunEncryptDecryptCycle(data); - if (error != null) + if (bench.Config.QuickConfig == null) { - Console.WriteLine($"Worker {workerID} iteration {j} failed: {error}"); - errorOccurred = true; + Console.WriteLine("Quick mode requested but no quick_config found in config file"); return; } - workerTimes.Add(Stopwatch.GetElapsedTime(iterStart).TotalMilliseconds); - } - - lock (timesMutex) - { - allTimes.AddRange(workerTimes); + ConfigLoader.AdjustForQuickTest(bench.Config); } - }); - } - Task.WaitAll(tasks); - var totalDuration = Stopwatch.GetElapsedTime(startTime).TotalSeconds; - - if (errorOccurred || !allTimes.Any()) - { - Console.WriteLine("Concurrent test failed - no successful operations"); - return null; - } - - // Calculate metrics - allTimes.Sort(); - var totalOperations = allTimes.Count; - var concurrentOpsPerSecond = totalOperations / totalDuration; - - var result = new BenchmarkResult - { - TestName = "concurrent", - Language = "net", - DataSize = dataSize, - Concurrency = concurrency, - EndToEndLatencyMs = Average(allTimes), - OpsPerSecond = concurrentOpsPerSecond, - BytesPerSecond = dataSize * concurrentOpsPerSecond, - P50Latency = Percentile(allTimes, 0.50), - P95Latency = Percentile(allTimes, 0.95), - P99Latency = Percentile(allTimes, 0.99), - Timestamp = DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss"), - DotNetVersion = Environment.Version.ToString(), - CpuCount = _cpuCount, - TotalMemoryGb = _totalMemoryGb - }; - - Console.WriteLine($"Concurrent test completed: {result.OpsPerSecond:F2} ops/sec @ {concurrency} threads"); - - return result; - } + // Run benchmarks + bench.RunAllBenchmarks(); - /// - /// Run all configured benchmark tests - matches Java structure - /// - public async Task> RunAllBenchmarksAsync(bool isQuickMode = false) - { - Console.WriteLine("Starting comprehensive ESDK benchmark suite"); - - var allResults = new List(); - - // Get test parameters from config - matches Java approach - var dataSizes = new List(); - // Collect all data sizes from all categories - dataSizes.AddRange(_config.DataSizes.Small); - dataSizes.AddRange(_config.DataSizes.Medium); - dataSizes.AddRange(_config.DataSizes.Large); - - var concurrencyLevels = _config.ConcurrencyLevels; - var iterations = _config.Iterations.Measurement; - - var totalTests = dataSizes.Count * (concurrencyLevels.Count + 2); - - Console.WriteLine($"Running {totalTests} total tests"); - - var progressOptions = new ProgressBarOptions - { - ProgressCharacter = '█', - ProgressBarOnBottom = true, - ForegroundColor = ConsoleColor.Green, - BackgroundColor = ConsoleColor.DarkGray, - CollapseWhenFinished = false, - DisplayTimeInRealTime = true, - ShowEstimatedDuration = true - }; - - using var overallProgress = new ProgressBar(totalTests, "Overall Progress", progressOptions); - int completedTests = 0; - - // Throughput tests - if (ShouldRunTestType("throughput", isQuickMode)) - { - overallProgress.WriteLine("Running throughput tests..."); - foreach (var dataSize in dataSizes) - { - try - { - var result = RunThroughputTest(dataSize, iterations, overallProgress); - if (result != null) - { - allResults.Add(result); - } - } - catch (Exception ex) - { - overallProgress.WriteLine($"Throughput test failed: {ex.Message}"); - } - - completedTests++; - overallProgress.Tick(); - } - } - else - { - overallProgress.WriteLine("Skipping throughput tests (not in test_types)"); - } + // Save results + await BenchmarkResultsHelper.SaveResultsAsync(bench.Results, options.OutputPath, Environment.ProcessorCount, ESDKBenchmark.GetTotalMemoryGb()); - // Memory tests - if (ShouldRunTestType("memory", isQuickMode)) - { - overallProgress.WriteLine("Running memory tests..."); - foreach (var dataSize in dataSizes) - { - try - { - var result = RunMemoryTest(dataSize, overallProgress); - if (result != null) - { - allResults.Add(result); - } - } - catch (Exception ex) - { - overallProgress.WriteLine($"Memory test failed: {ex.Message}"); - } - - completedTests++; - overallProgress.Tick(); - } - } - else - { - overallProgress.WriteLine("Skipping memory tests (not in test_types)"); - } + // Print summary + Console.WriteLine("\n=== ESDK .NET Benchmark Summary ==="); + Console.WriteLine($"Total tests completed: {bench.Results.Count}"); + Console.WriteLine($"Results saved to: {options.OutputPath}"); - // Concurrent tests - if (ShouldRunTestType("concurrency", isQuickMode)) - { - overallProgress.WriteLine("Running concurrency tests..."); - foreach (var dataSize in dataSizes) - { - foreach (var concurrency in concurrencyLevels.Where(c => c > 1)) + if (bench.Results.Count > 0) { - try + var maxThroughput = 0.0; + foreach (var result in bench.Results) { - var result = RunConcurrentTest(dataSize, concurrency, 5, overallProgress); - if (result != null) + if (result.TestName == "throughput" && result.OpsPerSecond > maxThroughput) { - allResults.Add(result); + maxThroughput = result.OpsPerSecond; } } - catch (Exception ex) + if (maxThroughput > 0) { - overallProgress.WriteLine($"Concurrent test failed: {ex.Message}"); + Console.WriteLine($"Maximum throughput: {maxThroughput:F2} ops/sec"); } - - completedTests++; - overallProgress.Tick(); } } - } - else - { - overallProgress.WriteLine("Skipping concurrency tests (not in test_types)"); - } - - _results.AddRange(allResults); - Console.WriteLine($"Benchmark suite completed. Total results: {allResults.Count}"); - return allResults; - } - - /// - /// Save benchmark results to JSON file - matches Java structure - /// - public async Task SaveResultsAsync(string outputPath) - { - // Create output directory if it doesn't exist - var outputDir = Path.GetDirectoryName(outputPath); - if (!string.IsNullOrEmpty(outputDir) && !Directory.Exists(outputDir)) - { - Directory.CreateDirectory(outputDir); - } - - // Prepare results data - matches other languages structure - var resultsData = new - { - metadata = new + catch (Exception ex) { - language = "net", - timestamp = DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss"), - dotnet_version = Environment.Version.ToString(), - cpu_count = _cpuCount, - total_memory_gb = _totalMemoryGb, - total_tests = _results.Count - }, - results = _results - }; - - // Write to file - var json = JsonConvert.SerializeObject(resultsData, Formatting.Indented); - await File.WriteAllTextAsync(outputPath, json); - - Console.WriteLine($"Results saved to {outputPath}"); - } - - /// - /// Get total system memory in GB - /// - private static double GetTotalMemoryGb() - { - try - { - var gcMemoryInfo = GC.GetGCMemoryInfo(); - return gcMemoryInfo.TotalAvailableMemoryBytes / (1024.0 * 1024.0 * 1024.0); - } - catch - { - // Fallback - estimate based on process memory - return Environment.WorkingSet / (1024.0 * 1024.0 * 1024.0) * 4; // Rough estimate - } + Console.WriteLine($"Benchmark failed: {ex}"); + } + }); } - } diff --git a/esdk-performance-testing/benchmarks/net/README.md b/esdk-performance-testing/benchmarks/net/README.md index dde13a7c9..1a8604ac3 100644 --- a/esdk-performance-testing/benchmarks/net/README.md +++ b/esdk-performance-testing/benchmarks/net/README.md @@ -1,363 +1,43 @@ -# ESDK .NET Performance Benchmark +# AWS Encryption SDK .NET Benchmark -This directory contains the .NET implementation of the AWS Encryption SDK (ESDK) performance benchmark suite. - -## Overview - -The .NET benchmark provides comprehensive performance testing for the ESDK .NET runtime, measuring: - -- **Throughput**: Operations per second and data processing rates -- **Latency**: Encrypt/decrypt operation timing with percentile analysis -- **Memory Usage**: Peak memory consumption and efficiency with GC optimization -- **Concurrency**: Multi-threaded performance characteristics using async/await -- **Chunk Processing**: Efficient handling of large files through chunked processing - -## Key Features - -### Chunk File Reading - -The .NET implementation includes advanced chunk file reading capabilities for efficient processing of large files: - -- **Configurable Chunk Size**: Default 1MB, customizable via `--chunk-size` parameter -- **Memory Efficient**: Processes large files without loading entire content into memory -- **Streaming Processing**: Handles files larger than available RAM using `ReadOnlySpan` -- **Progress Tracking**: Real-time progress indication for large file operations - -### Async Processing - -- Built on .NET's Task-based Asynchronous Pattern (TAP) -- Concurrent task execution with proper async/await patterns -- Non-blocking operations for better resource utilization -- Configurable concurrency levels - -### Advanced Metrics - -- Statistical analysis using MathNet.Numerics -- Percentile calculations (P50, P95, P99) -- Memory profiling with GC integration -- Comprehensive system information collection - -## Prerequisites - -- **.NET SDK**: 8.0 or later (install from [dotnet.microsoft.com](https://dotnet.microsoft.com/download)) -- **System Dependencies**: - - On Linux: `libicu` for globalization - - On macOS: No additional dependencies - - On Windows: No additional dependencies +Performance testing suite for the AWS Encryption SDK .NET implementation. ## Quick Start -### 1. Build and Run - ```bash -# Run full benchmark suite -./run_benchmark.sh - -# Run quick test (reduced iterations) -./run_benchmark.sh --quick - -# Run with custom chunk size (2MB) -./run_benchmark.sh --chunk-size 2097152 - -# Run with verbose logging -./run_benchmark.sh --verbose -``` - -### 2. Manual Build - -```bash -# Restore dependencies -dotnet restore - -# Build in release mode (recommended for benchmarks) +# Build project dotnet build --configuration Release -# Run with custom parameters -dotnet run --configuration Release -- \ - --config ../../config/test-scenarios.yaml \ - --output ../../results/raw-data/net_results.json \ - --chunk-size 1048576 \ - --verbose -``` - -## Configuration - -### Command Line Options - -| Option | Description | Default | -| --------------- | -------------------------------------------- | ----------------------------------------- | -| `-c, --config` | Path to test configuration file | `../../config/test-scenarios.yaml` | -| `-o, --output` | Path to output results file | `../../results/raw-data/net_results.json` | -| `-q, --quick` | Run quick test with reduced iterations | `false` | -| `--chunk-size` | Chunk size for large file processing (bytes) | `1048576` (1MB) | -| `-v, --verbose` | Enable verbose logging | `false` | - -### Build Script Options - -| Option | Description | Default | -| ------------------- | ------------------------------ | ---------------------- | -| `--debug` | Build in debug mode | `false` (release mode) | -| `--chunk-size SIZE` | Set chunk size for large files | `1048576` | -| `-v, --verbose` | Enable verbose output | `false` | - -### Configuration File - -The benchmark uses the same YAML configuration as other language implementations: - -```yaml -data_sizes: - small: [1024, 5120, 10240] - medium: [102400, 512000, 1048576] - large: [10485760, 52428800, 104857600] - -iterations: - warmup: 5 - measurement: 10 - -concurrency_levels: [1, 2, 4, 8] -algorithm_suites: ["ALG_AES_256_GCM_IV12_TAG16_HKDF_SHA256"] -frame_lengths: [4096, 65536, 1048576] -``` - -## Chunk Processing Details - -### How It Works - -1. **Size Detection**: Files larger than the configured chunk size are automatically processed in chunks -2. **Sequential Processing**: Each chunk is encrypted/decrypted individually using `ReadOnlySpan` -3. **Memory Management**: Only one chunk is held in memory at a time with efficient span operations -4. **Progress Tracking**: Real-time progress updates using ShellProgressBar -5. **Result Aggregation**: Timing results are combined for accurate overall metrics - -### Chunk Size Selection - -Choose chunk size based on your use case: - -- **Small chunks (64KB - 256KB)**: Lower memory usage, more frequent operations -- **Medium chunks (1MB - 4MB)**: Balanced performance (recommended) -- **Large chunks (8MB+)**: Higher throughput, more memory usage - -### Example Usage - -```bash -# Process 1GB files with 4MB chunks -./run_benchmark.sh --chunk-size 4194304 - -# Memory-constrained environment (256KB chunks) -./run_benchmark.sh --chunk-size 262144 - -# High-performance setup (16MB chunks) -./run_benchmark.sh --chunk-size 16777216 -``` - -## Performance Optimization - -### Build Optimizations - -The benchmark automatically builds in release mode with optimizations: - -```xml - - true - portable - true - -``` - -### Runtime Optimizations - -- **GC Configuration**: `SustainedLowLatency` mode for consistent performance -- **Span Usage**: `ReadOnlySpan` for zero-copy operations -- **Async Patterns**: Proper async/await usage for I/O operations -- **Memory Pooling**: Efficient memory allocation patterns - -### Environment Variables - -```bash -# Enable detailed GC logging -export DOTNET_GCStress=0 -export DOTNET_gcServer=1 - -# Set custom thread pool settings -export DOTNET_ThreadPool_ForceMinWorkerThreads=8 -export DOTNET_ThreadPool_ForceMaxWorkerThreads=32 -``` - -## Output Format - -Results are saved in JSON format compatible with other language implementations: - -```json -{ - "metadata": { - "language": "net", - "timestamp": "2024-01-01T00:00:00Z", - "dotnet_version": "8.0.0", - "cpu_count": 8, - "total_memory_gb": 16.0, - "total_tests": 42, - "chunk_size": 1048576 - }, - "results": [ - { - "TestName": "throughput", - "Language": "net", - "DataSize": 1048576, - "AlgorithmSuite": "ALG_AES_256_GCM_IV12_TAG16_HKDF_SHA256", - "FrameLength": 65536, - "Concurrency": 1, - "EncryptLatencyMs": 2.5, - "DecryptLatencyMs": 2.1, - "EndToEndLatencyMs": 4.8, - "OpsPerSecond": 208.33, - "BytesPerSecond": 218453333.33, - "P50Latency": 4.7, - "P95Latency": 5.2, - "P99Latency": 5.8, - "Timestamp": "2024-01-01T00:00:00Z" - } - ] -} -``` - -## Troubleshooting - -### Common Issues - -1. **Build Failures** - - ```bash - # Update .NET SDK - dotnet --version - - # Clean and rebuild - dotnet clean && dotnet build --configuration Release - ``` - -2. **Memory Issues with Large Files** - - ```bash - # Reduce chunk size - ./run_benchmark.sh --chunk-size 262144 - - # Monitor memory usage - ./run_benchmark.sh --verbose - ``` - -3. **Performance Issues** - - ```bash - # Ensure release build - ./run_benchmark.sh --debug # Don't use this for benchmarks - - # Check GC settings - dotnet run --configuration Release -- --verbose - ``` - -### Debug Mode - -For development and debugging: - -```bash -# Build in debug mode -./run_benchmark.sh --debug --verbose - -# Enable detailed logging -dotnet run --configuration Debug -- --verbose -``` - -### Profiling - -For performance analysis: - -```bash -# Install profiling tools -dotnet tool install --global dotnet-trace -dotnet tool install --global dotnet-counters - -# Profile the benchmark -dotnet trace collect --process-id --providers Microsoft-DotNETCore-SampleProfiler -``` - -## Integration - -### CI/CD Integration - -```yaml -# GitHub Actions example -- name: Setup .NET - uses: actions/setup-dotnet@v3 - with: - dotnet-version: "8.0.x" - -- name: Run .NET Benchmarks - run: | - cd aws-encryption-sdk/esdk-performance-testing/benchmarks/net - ./run_benchmark.sh --quick - -- name: Upload Results - uses: actions/upload-artifact@v3 - with: - name: net-benchmark-results - path: aws-encryption-sdk/esdk-performance-testing/results/raw-data/net_results.json -``` - -### Automated Testing - -```bash -# Run unit tests (if any) -dotnet test +# Run benchmark +dotnet run --configuration Release -# Run benchmarks with validation -dotnet run --configuration Release -- --quick --verbose +# Quick test (reduced iterations) +dotnet run --configuration Release -- --quick ``` -## Contributing +## Options -When contributing to the .NET benchmark: +- `-c, --config` - Path to test configuration file (default: `../../config/test-scenarios.yaml`) +- `-o, --output` - Path to output results file (default: `../../results/raw-data/net_results.json`) +- `-q, --quick` - Run with reduced iterations for faster testing -1. **Code Style**: Follow .NET coding conventions and use EditorConfig -2. **Async Patterns**: Use proper async/await patterns, avoid blocking calls -3. **Memory Management**: Use spans and memory-efficient patterns -4. **Testing**: Add unit tests for new functionality -5. **Documentation**: Update this README for new features -6. **Performance**: Benchmark changes against baseline - -### Development Setup - -```bash -# Install development tools -dotnet tool install --global dotnet-format -dotnet tool install --global dotnet-outdated-tool - -# Format code -dotnet format - -# Check for outdated packages -dotnet outdated -``` - -## Dependencies - -### Core Dependencies +## Configuration -- **CommandLineParser**: Command-line argument parsing -- **YamlDotNet**: YAML configuration file support -- **Newtonsoft.Json**: JSON serialization -- **ShellProgressBar**: Progress indication -- **MathNet.Numerics**: Statistical calculations +Edit `../../config/test-scenarios.yaml` for test parameters: -### Logging and Monitoring +- Data sizes (small/medium/large) +- Iterations and concurrency levels -- **Microsoft.Extensions.Logging**: Structured logging -- **System.Diagnostics.PerformanceCounter**: Performance monitoring -- **System.Management**: System information (Windows) +## Test Types -### Development Dependencies +- **Throughput** - Measures encryption/decryption operations per second +- **Memory** - Tracks memory usage and allocations during operations +- **Concurrency** - Tests performance under concurrent load -- **.NET 8.0 SDK**: Required for building and running -- **NuGet Package Manager**: Dependency management +## Output -## License +Results saved as JSON to `../../results/raw-data/net_results.json` with: -This benchmark is part of the AWS Encryption SDK and is licensed under the Apache License 2.0. +- Performance metrics (ops/sec, latency percentiles) +- Memory usage (peak, average, allocations) +- System information (CPU, memory, .NET version) diff --git a/esdk-performance-testing/benchmarks/net/Results.cs b/esdk-performance-testing/benchmarks/net/Results.cs new file mode 100644 index 000000000..2e376d748 --- /dev/null +++ b/esdk-performance-testing/benchmarks/net/Results.cs @@ -0,0 +1,226 @@ +// Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +using System.Text.Json.Serialization; +using Newtonsoft.Json; + +namespace EsdkBenchmark; + +public class BenchmarkResult +{ + [JsonPropertyName("test_name")] + public string TestName { get; set; } = ""; + + [JsonPropertyName("language")] + public string Language { get; set; } = ""; + + [JsonPropertyName("data_size")] + public int DataSize { get; set; } + + [JsonPropertyName("concurrency")] + public int Concurrency { get; set; } = 1; + + [JsonPropertyName("operations_per_second")] + public double OpsPerSecond { get; set; } + + [JsonPropertyName("bytes_per_second")] + public double BytesPerSecond { get; set; } + + [JsonPropertyName("peak_memory_mb")] + public double PeakMemoryMb { get; set; } + + [JsonPropertyName("memory_efficiency_ratio")] + public double MemoryEfficiencyRatio { get; set; } + + [JsonPropertyName("avg_latency_ms")] + public double AvgLatencyMs { get; set; } + + [JsonPropertyName("p50_latency_ms")] + public double P50LatencyMs { get; set; } + + [JsonPropertyName("p95_latency_ms")] + public double P95LatencyMs { get; set; } + + [JsonPropertyName("p99_latency_ms")] + public double P99LatencyMs { get; set; } + + [JsonPropertyName("encrypt_latency_ms")] + public double EncryptLatencyMs { get; set; } + + [JsonPropertyName("decrypt_latency_ms")] + public double DecryptLatencyMs { get; set; } + + [JsonPropertyName("timestamp")] + public string Timestamp { get; set; } = ""; + + [JsonPropertyName("dotnet_version")] + public string DotNetVersion { get; set; } = ""; + + [JsonPropertyName("cpu_count")] + public int CpuCount { get; set; } + + [JsonPropertyName("total_memory_gb")] + public double TotalMemoryGb { get; set; } + + [JsonPropertyName("iterations")] + public int Iterations { get; set; } +} + +internal class BenchmarkResults +{ + [JsonPropertyName("metadata")] + public BenchmarkMetadata Metadata { get; set; } = new(); + + [JsonPropertyName("results")] + public List Results { get; set; } = new(); +} + +internal class BenchmarkMetadata +{ + [JsonPropertyName("language")] + public string Language { get; set; } = "net"; + + [JsonPropertyName("timestamp")] + public string Timestamp { get; set; } = ""; + + [JsonPropertyName("dotnet_version")] + public string DotNetVersion { get; set; } = ""; + + [JsonPropertyName("cpu_count")] + public int CpuCount { get; set; } + + [JsonPropertyName("total_memory_gb")] + public double TotalMemoryGb { get; set; } + + [JsonPropertyName("total_tests")] + public int TotalTests { get; set; } +} + +public static class BenchmarkResultsHelper +{ + public static double Average(IEnumerable values) + { + var list = values.ToList(); + return list.Count == 0 ? 0.0 : list.Sum() / list.Count; + } + + public static double Percentile(List sortedValues, double p) + { + if (sortedValues.Count == 0) return 0.0; + if (p <= 0.0) return sortedValues[0]; + if (p >= 1.0) return sortedValues[^1]; + + var index = p * (sortedValues.Count - 1); + var lower = (int)Math.Floor(index); + var upper = (int)Math.Ceiling(index); + + if (lower == upper) return sortedValues[lower]; + + var weight = index - lower; + return sortedValues[lower] * (1 - weight) + sortedValues[upper] * weight; + } + + public static async Task SaveResultsAsync(List results, string outputPath, int cpuCount, double totalMemoryGb) + { + var directory = Path.GetDirectoryName(outputPath); + if (!string.IsNullOrEmpty(directory)) + { + Directory.CreateDirectory(directory); + } + + var resultsData = new BenchmarkResults + { + Metadata = new BenchmarkMetadata + { + Language = "net", + Timestamp = DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss"), + DotNetVersion = Environment.Version.ToString(), + CpuCount = cpuCount, + TotalMemoryGb = totalMemoryGb, + TotalTests = results.Count + }, + Results = results + }; + + var json = System.Text.Json.JsonSerializer.Serialize(resultsData, new System.Text.Json.JsonSerializerOptions + { + WriteIndented = true, + PropertyNamingPolicy = System.Text.Json.JsonNamingPolicy.SnakeCaseLower + }); + + await File.WriteAllTextAsync(outputPath, json); + } + + public static BenchmarkResult CreateThroughputResult(List encryptLatencies, List decryptLatencies, List totalLatencies, int dataSize, int cpuCount, double totalMemoryGb) + { + var avgTotalLatency = totalLatencies.Average(); + var opsPerSecond = 1000.0 / avgTotalLatency; + totalLatencies.Sort(); + + return new BenchmarkResult + { + TestName = "throughput", + Language = "net", + DataSize = dataSize, + Concurrency = 1, + OpsPerSecond = opsPerSecond, + BytesPerSecond = opsPerSecond * dataSize, + AvgLatencyMs = avgTotalLatency, + P50LatencyMs = Percentile(totalLatencies, 0.5), + P95LatencyMs = Percentile(totalLatencies, 0.95), + P99LatencyMs = Percentile(totalLatencies, 0.99), + EncryptLatencyMs = encryptLatencies.Average(), + DecryptLatencyMs = decryptLatencies.Average(), + Iterations = encryptLatencies.Count, + Timestamp = DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss"), + DotNetVersion = Environment.Version.ToString(), + CpuCount = cpuCount, + TotalMemoryGb = totalMemoryGb + }; + } + + public static BenchmarkResult CreateMemoryResult(double peakMemoryMb, double avgMemoryMb, int dataSize, int cpuCount, double totalMemoryGb) + { + var memoryEfficiency = peakMemoryMb > 0 ? dataSize / (peakMemoryMb * 1024 * 1024) : 0.0; + + return new BenchmarkResult + { + TestName = "memory", + Language = "net", + DataSize = dataSize, + Concurrency = 1, + PeakMemoryMb = peakMemoryMb, + MemoryEfficiencyRatio = memoryEfficiency, + Timestamp = DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss"), + DotNetVersion = Environment.Version.ToString(), + CpuCount = cpuCount, + TotalMemoryGb = totalMemoryGb + }; + } + + public static BenchmarkResult CreateConcurrentResult(List allTimes, int totalOps, int dataSize, int concurrency, int cpuCount, double totalMemoryGb) + { + var avgLatency = allTimes.Average(); + var opsPerSecond = totalOps / (allTimes.Sum() / 1000.0); + allTimes.Sort(); + + return new BenchmarkResult + { + TestName = "concurrent", + Language = "net", + DataSize = dataSize, + Concurrency = concurrency, + OpsPerSecond = opsPerSecond, + BytesPerSecond = opsPerSecond * dataSize, + AvgLatencyMs = avgLatency, + P50LatencyMs = Percentile(allTimes, 0.5), + P95LatencyMs = Percentile(allTimes, 0.95), + P99LatencyMs = Percentile(allTimes, 0.99), + Iterations = totalOps, + Timestamp = DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss"), + DotNetVersion = Environment.Version.ToString(), + CpuCount = cpuCount, + TotalMemoryGb = totalMemoryGb + }; + } +} diff --git a/esdk-performance-testing/benchmarks/net/Tests.cs b/esdk-performance-testing/benchmarks/net/Tests.cs new file mode 100644 index 000000000..afbd3657a --- /dev/null +++ b/esdk-performance-testing/benchmarks/net/Tests.cs @@ -0,0 +1,372 @@ +// Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +using System.Diagnostics; +using AWS.Cryptography.EncryptionSDK; +using ShellProgressBar; + +namespace EsdkBenchmark; + +public partial class ESDKBenchmark +{ + private (double encryptMs, double decryptMs, string? error) RunEncryptDecryptCycle(byte[] data) + { + try + { + var plaintext = new MemoryStream(data); + + // Create encryption context matching Rust + var encryptionContext = new Dictionary() + { + {"purpose", "performance-test"}, + {"size", data.Length.ToString()} + }; + + // Encrypt + var encryptStart = Stopwatch.GetTimestamp(); + var encryptInput = new EncryptInput + { + Plaintext = plaintext, + Keyring = _keyring, + EncryptionContext = encryptionContext + }; + var encryptOutput = _encryptionSdk.Encrypt(encryptInput); + var encryptMs = Stopwatch.GetElapsedTime(encryptStart).TotalMilliseconds; + + // Decrypt + var decryptStart = Stopwatch.GetTimestamp(); + var decryptInput = new DecryptInput + { + Ciphertext = encryptOutput.Ciphertext, + Keyring = _keyring + }; + var decryptOutput = _encryptionSdk.Decrypt(decryptInput); + var decryptMs = Stopwatch.GetElapsedTime(decryptStart).TotalMilliseconds; + + // Verify integrity + decryptOutput.Plaintext.Position = 0; + var decryptedData = new byte[decryptOutput.Plaintext.Length]; + decryptOutput.Plaintext.Read(decryptedData, 0, decryptedData.Length); + + if (!data.SequenceEqual(decryptedData)) + { + return (0, 0, "Data integrity check failed"); + } + + return (encryptMs, decryptMs, null); + } + catch (Exception ex) + { + return (0, 0, ex.Message); + } + } + + public BenchmarkResult? RunThroughputTest(int dataSize, int iterations, ProgressBar? progressBar = null) + { + Action log = progressBar != null ? progressBar.WriteLine : Console.WriteLine; + log($"Running throughput test - Size: {dataSize} bytes, Iterations: {iterations}"); + + var data = GenerateTestData(dataSize); + + // Warmup (ignore results) + RunIterations(data, _config.Iterations.Warmup); + + // Measurement runs + var (encryptLatencies, decryptLatencies, totalLatencies) = RunIterations(data, iterations); + + if (!encryptLatencies.Any()) + { + Console.WriteLine("All test iterations failed"); + return null; + } + + return BenchmarkResultsHelper.CreateThroughputResult(encryptLatencies, decryptLatencies, totalLatencies, dataSize, _cpuCount, _totalMemoryGb); + } + + private (List encrypt, List decrypt, List total) RunIterations(byte[] data, int iterations) + { + var encryptLatencies = new List(); + var decryptLatencies = new List(); + var totalLatencies = new List(); + + for (int i = 0; i < iterations; i++) + { + var iterationStart = Stopwatch.GetTimestamp(); + var (encryptMs, decryptMs, error) = RunEncryptDecryptCycle(data); + + if (error != null) + { + Console.WriteLine($"Iteration {i} failed: {error}"); + continue; + } + + var totalMs = Stopwatch.GetElapsedTime(iterationStart).TotalMilliseconds; + + encryptLatencies.Add(encryptMs); + decryptLatencies.Add(decryptMs); + totalLatencies.Add(totalMs); + } + + return (encryptLatencies, decryptLatencies, totalLatencies); + } + + public BenchmarkResult? RunMemoryTest(int dataSize, ProgressBar? progressBar = null) + { + Action log = progressBar != null ? progressBar.WriteLine : Console.WriteLine; + log($"Running memory test - Size: {dataSize} bytes ({MemoryTestIterations} iterations, continuous sampling)"); + + var data = GenerateTestData(dataSize); + var (peakMemoryMb, avgMemoryMb) = SampleMemoryDuringOperations(data, progressBar); + + return BenchmarkResultsHelper.CreateMemoryResult(peakMemoryMb, avgMemoryMb, dataSize, _cpuCount, _totalMemoryGb); + } + + private (double peakMemoryMb, double avgMemoryMb) SampleMemoryDuringOperations(byte[] data, ProgressBar? progressBar = null) + { + Action log = progressBar != null ? progressBar.WriteLine : Console.WriteLine; + var peakHeap = 0.0; + var peakAllocations = 0.0; + var avgHeapValues = new List(); + + for (int i = 0; i < MemoryTestIterations; i++) + { + GC.Collect(); + GC.WaitForPendingFinalizers(); + GC.Collect(); + Thread.Sleep(GcSettleTimeMs); + + // Get baseline + var beforeHeap = GC.GetTotalMemory(false); + var beforeAllocated = (long)GC.GetTotalAllocatedBytes(false); + + // Start continuous sampling + var stopSampling = new CancellationTokenSource(); + var continuousSamples = new List<(double HeapMB, double AllocsMB)>(); + var samplingTask = Task.Run(() => SampleMemoryContinuously(beforeHeap, (ulong)beforeAllocated, stopSampling.Token, continuousSamples)); + + // Run operation + var operationStart = Stopwatch.GetTimestamp(); + var (_, _, error) = RunEncryptDecryptCycle(data); + var operationDuration = Stopwatch.GetElapsedTime(operationStart); + + stopSampling.Cancel(); + Thread.Sleep(FinalSampleWaitMs); + + if (error != null) + { + Console.WriteLine($"Memory test iteration {i + 1} failed: {error}"); + continue; + } + + // Analyze samples + var iterPeakHeap = 0.0; + var iterTotalAllocs = 0.0; + var iterAvgHeap = 0.0; + + if (continuousSamples.Count > 0) + { + var heapSum = 0.0; + foreach (var sample in continuousSamples) + { + if (sample.HeapMB > iterPeakHeap) + iterPeakHeap = sample.HeapMB; + if (sample.AllocsMB > iterTotalAllocs) + iterTotalAllocs = sample.AllocsMB; + heapSum += sample.HeapMB; + } + iterAvgHeap = heapSum / continuousSamples.Count; + } + + if (iterPeakHeap > peakHeap) + peakHeap = iterPeakHeap; + if (iterTotalAllocs > peakAllocations) + peakAllocations = iterTotalAllocs; + + avgHeapValues.Add(iterAvgHeap); + + log($"=== Iteration {i + 1} === Peak Heap: {iterPeakHeap:F2} MB, Total Allocs: {iterTotalAllocs:F2} MB, Avg Heap: {iterAvgHeap:F2} MB ({operationDuration.TotalMilliseconds:F0}ms, {continuousSamples.Count} samples)"); + } + + var avgHeap = avgHeapValues.Count > 0 ? avgHeapValues.Average() : 0.0; + + log("\nMemory Summary:"); + log($"- Absolute Peak Heap: {peakHeap:F2} MB (across all runs)"); + log($"- Average Heap: {avgHeap:F2} MB (across all runs)"); + log($"- Total Allocations: {peakAllocations:F2} MB (max across all runs)"); + + return (peakHeap, avgHeap); + } + + private void SampleMemoryContinuously(long baselineHeap, ulong baselineAllocs, CancellationToken cancellationToken, List<(double HeapMB, double AllocsMB)> samples) + { + while (!cancellationToken.IsCancellationRequested) + { + try + { + var currentHeap = GC.GetTotalMemory(false); + var currentAllocs = GC.GetTotalAllocatedBytes(false); + + var heapMB = (currentHeap - baselineHeap) / (1024.0 * 1024.0); + var allocsMB = ((long)currentAllocs - (long)baselineAllocs) / (1024.0 * 1024.0); + + lock (samples) + { + samples.Add((Math.Max(0, heapMB), Math.Max(0, allocsMB))); + } + + Thread.Sleep(SamplingIntervalMs); + } + catch (OperationCanceledException) + { + break; + } + } + } + + public BenchmarkResult? RunConcurrentTest(int dataSize, int concurrency, int iterationsPerWorker, ProgressBar? progressBar = null) + { + Action log = progressBar != null ? progressBar.WriteLine : Console.WriteLine; + log($"Running concurrent test - Size: {dataSize} bytes, Concurrency: {concurrency}"); + + var data = GenerateTestData(dataSize); + var allTimes = RunConcurrentWorkers(data, concurrency, iterationsPerWorker); + + if (!allTimes.Any()) + { + Console.WriteLine("All concurrent workers failed"); + return null; + } + + return BenchmarkResultsHelper.CreateConcurrentResult(allTimes, concurrency * iterationsPerWorker, dataSize, concurrency, _cpuCount, _totalMemoryGb); + } + + private List RunConcurrentWorkers(byte[] data, int concurrency, int iterationsPerWorker) + { + var allTimes = new List(); + var tasks = new List>>(); + + for (int i = 0; i < concurrency; i++) + { + tasks.Add(CreateWorkerTask(data, iterationsPerWorker, i)); + } + + Task.WaitAll(tasks.ToArray()); + + foreach (var task in tasks) + { + if (task.IsCompletedSuccessfully) + { + allTimes.AddRange(task.Result); + } + } + + return allTimes; + } + + private Task> CreateWorkerTask(byte[] data, int iterations, int workerID) + { + return Task.Run(() => + { + var workerTimes = new List(); + + for (int i = 0; i < iterations; i++) + { + var start = Stopwatch.GetTimestamp(); + var (_, _, error) = RunEncryptDecryptCycle(data); + if (error != null) + { + Console.WriteLine($"Worker {workerID} iteration {i} failed: {error}"); + continue; + } + workerTimes.Add(Stopwatch.GetElapsedTime(start).TotalMilliseconds); + } + return workerTimes; + }); + } + + private void RunThroughputTests(IEnumerable dataSizes, int iterations) + { + Console.WriteLine("Running throughput tests..."); + var dataSizesList = dataSizes.ToList(); + + var progressOptions = new ProgressBarOptions + { + ProgressCharacter = '█', + ProgressBarOnBottom = true, + ForegroundColor = ConsoleColor.Green, + BackgroundColor = ConsoleColor.DarkGreen + }; + + using var progressBar = new ProgressBar(dataSizesList.Count, "Throughput Tests", progressOptions); + + foreach (var dataSize in dataSizesList) + { + var result = RunThroughputTest(dataSize, iterations, progressBar); + if (result != null) + { + _results.Add(result); + progressBar.WriteLine($"Throughput test completed: {result.OpsPerSecond:F2} ops/sec"); + } + progressBar.Tick(); + } + } + + private void RunMemoryTests(IEnumerable dataSizes) + { + Console.WriteLine("Running memory tests..."); + var dataSizesList = dataSizes.ToList(); + + var progressOptions = new ProgressBarOptions + { + ProgressCharacter = '█', + ProgressBarOnBottom = true, + ForegroundColor = ConsoleColor.Blue, + BackgroundColor = ConsoleColor.DarkBlue + }; + + using var progressBar = new ProgressBar(dataSizesList.Count, "Memory Tests", progressOptions); + + foreach (var dataSize in dataSizesList) + { + var result = RunMemoryTest(dataSize, progressBar); + if (result != null) + { + _results.Add(result); + progressBar.WriteLine($"Memory test completed: {result.PeakMemoryMb:F2} MB peak"); + } + progressBar.Tick(); + } + } + + private void RunConcurrencyTests(IEnumerable dataSizes, IEnumerable concurrencyLevels) + { + Console.WriteLine("Running concurrency tests..."); + var dataSizesList = dataSizes.ToList(); + var concurrencyList = concurrencyLevels.Where(c => c > 1).ToList(); // Skip single-threaded + var totalTests = dataSizesList.Count * concurrencyList.Count; + + var progressOptions = new ProgressBarOptions + { + ProgressCharacter = '█', + ProgressBarOnBottom = true, + ForegroundColor = ConsoleColor.Yellow, + BackgroundColor = ConsoleColor.DarkYellow + }; + + using var progressBar = new ProgressBar(totalTests, "Concurrency Tests", progressOptions); + + foreach (var dataSize in dataSizesList) + { + foreach (var concurrency in concurrencyList) + { + var result = RunConcurrentTest(dataSize, concurrency, 5, progressBar); + if (result != null) + { + _results.Add(result); + progressBar.WriteLine($"Concurrent test completed: {result.OpsPerSecond:F2} ops/sec @ {concurrency} threads"); + } + progressBar.Tick(); + } + } + } +} diff --git a/esdk-performance-testing/results/raw-data/net_results.json b/esdk-performance-testing/results/raw-data/net_results.json index 48a55b552..30ff3ec44 100644 --- a/esdk-performance-testing/results/raw-data/net_results.json +++ b/esdk-performance-testing/results/raw-data/net_results.json @@ -1,78 +1,1146 @@ { "metadata": { "language": "net", - "timestamp": "2025-09-03T22:46:47.1519040Z", + "timestamp": "2025-09-04 15:02:19", "dotnet_version": "9.0.1", "cpu_count": 12, - "total_memory_gb": 36.0, - "total_tests": 3 + "total_memory_gb": 36, + "total_tests": 54 }, "results": [ { - "TestName": "throughput", - "Language": "net", - "DataSize": 52428800, - "AlgorithmSuite": "AES_256_GCM_HKDF_SHA512_COMMIT_KEY_ECDSA_P384", - "FrameLength": null, - "Concurrency": 0, - "EncryptLatencyMs": 190.40603333333334, - "DecryptLatencyMs": 200.46193333333335, - "EndToEndLatencyMs": 404.4672333333333, - "OpsPerSecond": 2.4714642676108096, - "BytesPerSecond": 129575905.79371363, - "PeakMemoryMb": 0.0, - "MemoryEfficiencyRatio": 0.0, - "P50Latency": 407.8135, - "P95Latency": 412.3747, - "P99Latency": 412.78014, - "Timestamp": "2025-09-03 15:46:42", - "DotNetVersion": "9.0.1", - "CpuCount": 12, - "TotalMemoryGb": 36.0 - }, - { - "TestName": "memory", - "Language": "net", - "DataSize": 52428800, - "AlgorithmSuite": "AES_256_GCM_HKDF_SHA512_COMMIT_KEY_ECDSA_P384", - "FrameLength": null, - "Concurrency": 0, - "EncryptLatencyMs": 0.0, - "DecryptLatencyMs": 0.0, - "EndToEndLatencyMs": 0.0, - "OpsPerSecond": 0.0, - "BytesPerSecond": 0.0, - "PeakMemoryMb": 575.6221771240234, - "MemoryEfficiencyRatio": 0.08686253238159553, - "P50Latency": 0.0, - "P95Latency": 0.0, - "P99Latency": 0.0, - "Timestamp": "2025-09-03 15:46:44", - "DotNetVersion": "9.0.1", - "CpuCount": 12, - "TotalMemoryGb": 36.0 - }, - { - "TestName": "concurrent", - "Language": "net", - "DataSize": 52428800, - "AlgorithmSuite": "AES_256_GCM_HKDF_SHA512_COMMIT_KEY_ECDSA_P384", - "FrameLength": null, - "Concurrency": 2, - "EncryptLatencyMs": 0.0, - "DecryptLatencyMs": 0.0, - "EndToEndLatencyMs": 549.6251100000001, - "OpsPerSecond": 3.6257653265451273, - "BytesPerSecond": 190094525.15236917, - "PeakMemoryMb": 0.0, - "MemoryEfficiencyRatio": 0.0, - "P50Latency": 549.3668, - "P95Latency": 568.8644999999999, - "P99Latency": 571.30602, - "Timestamp": "2025-09-03 15:46:47", - "DotNetVersion": "9.0.1", - "CpuCount": 12, - "TotalMemoryGb": 36.0 + "test_name": "throughput", + "language": "net", + "data_size": 1024, + "concurrency": 1, + "operations_per_second": 72.08667124657318, + "bytes_per_second": 73816.75135649093, + "peak_memory_mb": 0, + "memory_efficiency_ratio": 0, + "avg_latency_ms": 13.87219, + "p50_latency_ms": 13.860199999999999, + "p95_latency_ms": 14.094555, + "p99_latency_ms": 14.153631, + "encrypt_latency_ms": 5.60265, + "decrypt_latency_ms": 8.268220000000001, + "timestamp": "2025-09-04 14:59:35", + "dotnet_version": "9.0.1", + "cpu_count": 12, + "total_memory_gb": 36, + "iterations": 10 + }, + { + "test_name": "throughput", + "language": "net", + "data_size": 5120, + "concurrency": 1, + "operations_per_second": 87.2350671012136, + "bytes_per_second": 446643.5435582136, + "peak_memory_mb": 0, + "memory_efficiency_ratio": 0, + "avg_latency_ms": 11.463280000000001, + "p50_latency_ms": 13.606300000000001, + "p95_latency_ms": 15.430384999999998, + "p99_latency_ms": 16.359437, + "encrypt_latency_ms": 4.8563600000000005, + "decrypt_latency_ms": 6.60339, + "timestamp": "2025-09-04 14:59:35", + "dotnet_version": "9.0.1", + "cpu_count": 12, + "total_memory_gb": 36, + "iterations": 10 + }, + { + "test_name": "throughput", + "language": "net", + "data_size": 10240, + "concurrency": 1, + "operations_per_second": 174.68011703567842, + "bytes_per_second": 1788724.398445347, + "peak_memory_mb": 0, + "memory_efficiency_ratio": 0, + "avg_latency_ms": 5.724749999999999, + "p50_latency_ms": 5.68215, + "p95_latency_ms": 6.285939999999999, + "p99_latency_ms": 6.561268, + "encrypt_latency_ms": 2.7832999999999997, + "decrypt_latency_ms": 2.9387800000000004, + "timestamp": "2025-09-04 14:59:35", + "dotnet_version": "9.0.1", + "cpu_count": 12, + "total_memory_gb": 36, + "iterations": 10 + }, + { + "test_name": "throughput", + "language": "net", + "data_size": 102400, + "concurrency": 1, + "operations_per_second": 154.59725868140907, + "bytes_per_second": 15830759.28897629, + "peak_memory_mb": 0, + "memory_efficiency_ratio": 0, + "avg_latency_ms": 6.46842, + "p50_latency_ms": 6.5625, + "p95_latency_ms": 6.75835, + "p99_latency_ms": 6.76087, + "encrypt_latency_ms": 3.2333, + "decrypt_latency_ms": 3.22766, + "timestamp": "2025-09-04 14:59:35", + "dotnet_version": "9.0.1", + "cpu_count": 12, + "total_memory_gb": 36, + "iterations": 10 + }, + { + "test_name": "throughput", + "language": "net", + "data_size": 512000, + "concurrency": 1, + "operations_per_second": 89.823452005084, + "bytes_per_second": 45989607.426603004, + "peak_memory_mb": 0, + "memory_efficiency_ratio": 0, + "avg_latency_ms": 11.132950000000001, + "p50_latency_ms": 11.314350000000001, + "p95_latency_ms": 12.332854999999999, + "p99_latency_ms": 12.632411000000001, + "encrypt_latency_ms": 4.75546, + "decrypt_latency_ms": 6.34876, + "timestamp": "2025-09-04 14:59:35", + "dotnet_version": "9.0.1", + "cpu_count": 12, + "total_memory_gb": 36, + "iterations": 10 + }, + { + "test_name": "throughput", + "language": "net", + "data_size": 1048576, + "concurrency": 1, + "operations_per_second": 65.23293375986974, + "bytes_per_second": 68401688.75018917, + "peak_memory_mb": 0, + "memory_efficiency_ratio": 0, + "avg_latency_ms": 15.32968, + "p50_latency_ms": 14.78135, + "p95_latency_ms": 18.237940000000002, + "p99_latency_ms": 18.498868, + "encrypt_latency_ms": 8.180050000000001, + "decrypt_latency_ms": 7.09774, + "timestamp": "2025-09-04 14:59:36", + "dotnet_version": "9.0.1", + "cpu_count": 12, + "total_memory_gb": 36, + "iterations": 10 + }, + { + "test_name": "throughput", + "language": "net", + "data_size": 10485760, + "concurrency": 1, + "operations_per_second": 12.74257888134174, + "bytes_per_second": 133615623.93081798, + "peak_memory_mb": 0, + "memory_efficiency_ratio": 0, + "avg_latency_ms": 78.47705, + "p50_latency_ms": 77.6223, + "p95_latency_ms": 86.34951000000001, + "p99_latency_ms": 86.86150200000002, + "encrypt_latency_ms": 35.15092, + "decrypt_latency_ms": 41.808899999999994, + "timestamp": "2025-09-04 14:59:37", + "dotnet_version": "9.0.1", + "cpu_count": 12, + "total_memory_gb": 36, + "iterations": 10 + }, + { + "test_name": "throughput", + "language": "net", + "data_size": 52428800, + "concurrency": 1, + "operations_per_second": 2.7726917369017214, + "bytes_per_second": 145368900.53567296, + "peak_memory_mb": 0, + "memory_efficiency_ratio": 0, + "avg_latency_ms": 360.66035999999997, + "p50_latency_ms": 358.33349999999996, + "p95_latency_ms": 378.05859499999997, + "p99_latency_ms": 383.266679, + "encrypt_latency_ms": 169.50748000000002, + "decrypt_latency_ms": 184.92687, + "timestamp": "2025-09-04 14:59:42", + "dotnet_version": "9.0.1", + "cpu_count": 12, + "total_memory_gb": 36, + "iterations": 10 + }, + { + "test_name": "throughput", + "language": "net", + "data_size": 104857600, + "concurrency": 1, + "operations_per_second": 1.3450688799685415, + "bytes_per_second": 141040694.58818933, + "peak_memory_mb": 0, + "memory_efficiency_ratio": 0, + "avg_latency_ms": 743.45635, + "p50_latency_ms": 743.2093, + "p95_latency_ms": 758.124655, + "p99_latency_ms": 761.503651, + "encrypt_latency_ms": 347.86061, + "decrypt_latency_ms": 384.21016, + "timestamp": "2025-09-04 14:59:54", + "dotnet_version": "9.0.1", + "cpu_count": 12, + "total_memory_gb": 36, + "iterations": 10 + }, + { + "test_name": "memory", + "language": "net", + "data_size": 1024, + "concurrency": 1, + "operations_per_second": 0, + "bytes_per_second": 0, + "peak_memory_mb": 7.146751403808594, + "memory_efficiency_ratio": 0.00013664425202751247, + "avg_latency_ms": 0, + "p50_latency_ms": 0, + "p95_latency_ms": 0, + "p99_latency_ms": 0, + "encrypt_latency_ms": 0, + "decrypt_latency_ms": 0, + "timestamp": "2025-09-04 14:59:54", + "dotnet_version": "9.0.1", + "cpu_count": 12, + "total_memory_gb": 36, + "iterations": 0 + }, + { + "test_name": "memory", + "language": "net", + "data_size": 5120, + "concurrency": 1, + "operations_per_second": 0, + "bytes_per_second": 0, + "peak_memory_mb": 5.9990997314453125, + "memory_efficiency_ratio": 0.0008139242083951195, + "avg_latency_ms": 0, + "p50_latency_ms": 0, + "p95_latency_ms": 0, + "p99_latency_ms": 0, + "encrypt_latency_ms": 0, + "decrypt_latency_ms": 0, + "timestamp": "2025-09-04 14:59:54", + "dotnet_version": "9.0.1", + "cpu_count": 12, + "total_memory_gb": 36, + "iterations": 0 + }, + { + "test_name": "memory", + "language": "net", + "data_size": 10240, + "concurrency": 1, + "operations_per_second": 0, + "bytes_per_second": 0, + "peak_memory_mb": 5.543853759765625, + "memory_efficiency_ratio": 0.0017615228364921474, + "avg_latency_ms": 0, + "p50_latency_ms": 0, + "p95_latency_ms": 0, + "p99_latency_ms": 0, + "encrypt_latency_ms": 0, + "decrypt_latency_ms": 0, + "timestamp": "2025-09-04 14:59:54", + "dotnet_version": "9.0.1", + "cpu_count": 12, + "total_memory_gb": 36, + "iterations": 0 + }, + { + "test_name": "memory", + "language": "net", + "data_size": 102400, + "concurrency": 1, + "operations_per_second": 0, + "bytes_per_second": 0, + "peak_memory_mb": 6.570014953613281, + "memory_efficiency_ratio": 0.01486393116141881, + "avg_latency_ms": 0, + "p50_latency_ms": 0, + "p95_latency_ms": 0, + "p99_latency_ms": 0, + "encrypt_latency_ms": 0, + "decrypt_latency_ms": 0, + "timestamp": "2025-09-04 14:59:54", + "dotnet_version": "9.0.1", + "cpu_count": 12, + "total_memory_gb": 36, + "iterations": 0 + }, + { + "test_name": "memory", + "language": "net", + "data_size": 512000, + "concurrency": 1, + "operations_per_second": 0, + "bytes_per_second": 0, + "peak_memory_mb": 9.649429321289062, + "memory_efficiency_ratio": 0.0506020857547222, + "avg_latency_ms": 0, + "p50_latency_ms": 0, + "p95_latency_ms": 0, + "p99_latency_ms": 0, + "encrypt_latency_ms": 0, + "decrypt_latency_ms": 0, + "timestamp": "2025-09-04 14:59:54", + "dotnet_version": "9.0.1", + "cpu_count": 12, + "total_memory_gb": 36, + "iterations": 0 + }, + { + "test_name": "memory", + "language": "net", + "data_size": 1048576, + "concurrency": 1, + "operations_per_second": 0, + "bytes_per_second": 0, + "peak_memory_mb": 16.00157928466797, + "memory_efficiency_ratio": 0.06249383152812656, + "avg_latency_ms": 0, + "p50_latency_ms": 0, + "p95_latency_ms": 0, + "p99_latency_ms": 0, + "encrypt_latency_ms": 0, + "decrypt_latency_ms": 0, + "timestamp": "2025-09-04 14:59:54", + "dotnet_version": "9.0.1", + "cpu_count": 12, + "total_memory_gb": 36, + "iterations": 0 + }, + { + "test_name": "memory", + "language": "net", + "data_size": 10485760, + "concurrency": 1, + "operations_per_second": 0, + "bytes_per_second": 0, + "peak_memory_mb": 96.90263366699219, + "memory_efficiency_ratio": 0.10319636960916044, + "avg_latency_ms": 0, + "p50_latency_ms": 0, + "p95_latency_ms": 0, + "p99_latency_ms": 0, + "encrypt_latency_ms": 0, + "decrypt_latency_ms": 0, + "timestamp": "2025-09-04 14:59:55", + "dotnet_version": "9.0.1", + "cpu_count": 12, + "total_memory_gb": 36, + "iterations": 0 + }, + { + "test_name": "memory", + "language": "net", + "data_size": 52428800, + "concurrency": 1, + "operations_per_second": 0, + "bytes_per_second": 0, + "peak_memory_mb": 472.8604049682617, + "memory_efficiency_ratio": 0.10573945180154382, + "avg_latency_ms": 0, + "p50_latency_ms": 0, + "p95_latency_ms": 0, + "p99_latency_ms": 0, + "encrypt_latency_ms": 0, + "decrypt_latency_ms": 0, + "timestamp": "2025-09-04 14:59:56", + "dotnet_version": "9.0.1", + "cpu_count": 12, + "total_memory_gb": 36, + "iterations": 0 + }, + { + "test_name": "memory", + "language": "net", + "data_size": 104857600, + "concurrency": 1, + "operations_per_second": 0, + "bytes_per_second": 0, + "peak_memory_mb": 926.3390579223633, + "memory_efficiency_ratio": 0.107951833774865, + "avg_latency_ms": 0, + "p50_latency_ms": 0, + "p95_latency_ms": 0, + "p99_latency_ms": 0, + "encrypt_latency_ms": 0, + "decrypt_latency_ms": 0, + "timestamp": "2025-09-04 15:00:00", + "dotnet_version": "9.0.1", + "cpu_count": 12, + "total_memory_gb": 36, + "iterations": 0 + }, + { + "test_name": "concurrent", + "language": "net", + "data_size": 1024, + "concurrency": 2, + "operations_per_second": 203.01476932446832, + "bytes_per_second": 207887.12378825556, + "peak_memory_mb": 0, + "memory_efficiency_ratio": 0, + "avg_latency_ms": 4.925750000000001, + "p50_latency_ms": 4.9619, + "p95_latency_ms": 5.9014050000000005, + "p99_latency_ms": 5.903961000000001, + "encrypt_latency_ms": 0, + "decrypt_latency_ms": 0, + "timestamp": "2025-09-04 15:00:00", + "dotnet_version": "9.0.1", + "cpu_count": 12, + "total_memory_gb": 36, + "iterations": 10 + }, + { + "test_name": "concurrent", + "language": "net", + "data_size": 1024, + "concurrency": 4, + "operations_per_second": 177.5156857297803, + "bytes_per_second": 181776.06218729503, + "peak_memory_mb": 0, + "memory_efficiency_ratio": 0, + "avg_latency_ms": 5.633305, + "p50_latency_ms": 5.54415, + "p95_latency_ms": 6.02534, + "p99_latency_ms": 6.095868, + "encrypt_latency_ms": 0, + "decrypt_latency_ms": 0, + "timestamp": "2025-09-04 15:00:00", + "dotnet_version": "9.0.1", + "cpu_count": 12, + "total_memory_gb": 36, + "iterations": 20 + }, + { + "test_name": "concurrent", + "language": "net", + "data_size": 1024, + "concurrency": 8, + "operations_per_second": 98.62883724737766, + "bytes_per_second": 100995.92934131472, + "peak_memory_mb": 0, + "memory_efficiency_ratio": 0, + "avg_latency_ms": 10.1390225, + "p50_latency_ms": 10.029800000000002, + "p95_latency_ms": 12.28341, + "p99_latency_ms": 12.324171, + "encrypt_latency_ms": 0, + "decrypt_latency_ms": 0, + "timestamp": "2025-09-04 15:00:00", + "dotnet_version": "9.0.1", + "cpu_count": 12, + "total_memory_gb": 36, + "iterations": 40 + }, + { + "test_name": "concurrent", + "language": "net", + "data_size": 1024, + "concurrency": 16, + "operations_per_second": 67.82301516370019, + "bytes_per_second": 69450.767527629, + "peak_memory_mb": 0, + "memory_efficiency_ratio": 0, + "avg_latency_ms": 14.744257499999998, + "p50_latency_ms": 16.312800000000003, + "p95_latency_ms": 20.994409999999995, + "p99_latency_ms": 22.885280999999992, + "encrypt_latency_ms": 0, + "decrypt_latency_ms": 0, + "timestamp": "2025-09-04 15:00:00", + "dotnet_version": "9.0.1", + "cpu_count": 12, + "total_memory_gb": 36, + "iterations": 80 + }, + { + "test_name": "concurrent", + "language": "net", + "data_size": 5120, + "concurrency": 2, + "operations_per_second": 204.67267722095443, + "bytes_per_second": 1047924.1073712867, + "peak_memory_mb": 0, + "memory_efficiency_ratio": 0, + "avg_latency_ms": 4.88585, + "p50_latency_ms": 4.8391, + "p95_latency_ms": 5.079475, + "p99_latency_ms": 5.092615, + "encrypt_latency_ms": 0, + "decrypt_latency_ms": 0, + "timestamp": "2025-09-04 15:00:00", + "dotnet_version": "9.0.1", + "cpu_count": 12, + "total_memory_gb": 36, + "iterations": 10 + }, + { + "test_name": "concurrent", + "language": "net", + "data_size": 5120, + "concurrency": 4, + "operations_per_second": 165.81823005621237, + "bytes_per_second": 848989.3378878073, + "peak_memory_mb": 0, + "memory_efficiency_ratio": 0, + "avg_latency_ms": 6.0307, + "p50_latency_ms": 5.99005, + "p95_latency_ms": 6.505724999999999, + "p99_latency_ms": 6.5319449999999994, + "encrypt_latency_ms": 0, + "decrypt_latency_ms": 0, + "timestamp": "2025-09-04 15:00:01", + "dotnet_version": "9.0.1", + "cpu_count": 12, + "total_memory_gb": 36, + "iterations": 20 + }, + { + "test_name": "concurrent", + "language": "net", + "data_size": 5120, + "concurrency": 8, + "operations_per_second": 97.5107932259252, + "bytes_per_second": 499255.26131673704, + "peak_memory_mb": 0, + "memory_efficiency_ratio": 0, + "avg_latency_ms": 10.255275000000001, + "p50_latency_ms": 9.9221, + "p95_latency_ms": 13.69103, + "p99_latency_ms": 16.258297, + "encrypt_latency_ms": 0, + "decrypt_latency_ms": 0, + "timestamp": "2025-09-04 15:00:01", + "dotnet_version": "9.0.1", + "cpu_count": 12, + "total_memory_gb": 36, + "iterations": 40 + }, + { + "test_name": "concurrent", + "language": "net", + "data_size": 5120, + "concurrency": 16, + "operations_per_second": 67.33827051921254, + "bytes_per_second": 344771.9450583682, + "peak_memory_mb": 0, + "memory_efficiency_ratio": 0, + "avg_latency_ms": 14.850396250000008, + "p50_latency_ms": 16.327199999999998, + "p95_latency_ms": 21.33692, + "p99_latency_ms": 22.684462999999983, + "encrypt_latency_ms": 0, + "decrypt_latency_ms": 0, + "timestamp": "2025-09-04 15:00:01", + "dotnet_version": "9.0.1", + "cpu_count": 12, + "total_memory_gb": 36, + "iterations": 80 + }, + { + "test_name": "concurrent", + "language": "net", + "data_size": 10240, + "concurrency": 2, + "operations_per_second": 210.60346316334827, + "bytes_per_second": 2156579.462792686, + "peak_memory_mb": 0, + "memory_efficiency_ratio": 0, + "avg_latency_ms": 4.74826, + "p50_latency_ms": 4.69525, + "p95_latency_ms": 4.993955, + "p99_latency_ms": 5.008391, + "encrypt_latency_ms": 0, + "decrypt_latency_ms": 0, + "timestamp": "2025-09-04 15:00:01", + "dotnet_version": "9.0.1", + "cpu_count": 12, + "total_memory_gb": 36, + "iterations": 10 + }, + { + "test_name": "concurrent", + "language": "net", + "data_size": 10240, + "concurrency": 4, + "operations_per_second": 163.42419446171746, + "bytes_per_second": 1673463.7512879868, + "peak_memory_mb": 0, + "memory_efficiency_ratio": 0, + "avg_latency_ms": 6.119045, + "p50_latency_ms": 6.058249999999999, + "p95_latency_ms": 6.331460000000002, + "p99_latency_ms": 7.090851999999999, + "encrypt_latency_ms": 0, + "decrypt_latency_ms": 0, + "timestamp": "2025-09-04 15:00:01", + "dotnet_version": "9.0.1", + "cpu_count": 12, + "total_memory_gb": 36, + "iterations": 20 + }, + { + "test_name": "concurrent", + "language": "net", + "data_size": 10240, + "concurrency": 8, + "operations_per_second": 93.61796940751996, + "bytes_per_second": 958648.0067330045, + "peak_memory_mb": 0, + "memory_efficiency_ratio": 0, + "avg_latency_ms": 10.681709999999999, + "p50_latency_ms": 10.6278, + "p95_latency_ms": 12.55197, + "p99_latency_ms": 12.900016, + "encrypt_latency_ms": 0, + "decrypt_latency_ms": 0, + "timestamp": "2025-09-04 15:00:01", + "dotnet_version": "9.0.1", + "cpu_count": 12, + "total_memory_gb": 36, + "iterations": 40 + }, + { + "test_name": "concurrent", + "language": "net", + "data_size": 10240, + "concurrency": 16, + "operations_per_second": 71.66094483880856, + "bytes_per_second": 733808.0751493997, + "peak_memory_mb": 0, + "memory_efficiency_ratio": 0, + "avg_latency_ms": 13.954602499999998, + "p50_latency_ms": 15.3727, + "p95_latency_ms": 19.960879999999996, + "p99_latency_ms": 20.928764, + "encrypt_latency_ms": 0, + "decrypt_latency_ms": 0, + "timestamp": "2025-09-04 15:00:01", + "dotnet_version": "9.0.1", + "cpu_count": 12, + "total_memory_gb": 36, + "iterations": 80 + }, + { + "test_name": "concurrent", + "language": "net", + "data_size": 102400, + "concurrency": 2, + "operations_per_second": 143.29644882740513, + "bytes_per_second": 14673556.359926285, + "peak_memory_mb": 0, + "memory_efficiency_ratio": 0, + "avg_latency_ms": 6.978540000000001, + "p50_latency_ms": 6.50675, + "p95_latency_ms": 9.093004999999998, + "p99_latency_ms": 9.691001, + "encrypt_latency_ms": 0, + "decrypt_latency_ms": 0, + "timestamp": "2025-09-04 15:00:01", + "dotnet_version": "9.0.1", + "cpu_count": 12, + "total_memory_gb": 36, + "iterations": 10 + }, + { + "test_name": "concurrent", + "language": "net", + "data_size": 102400, + "concurrency": 4, + "operations_per_second": 130.7376676792299, + "bytes_per_second": 13387537.170353143, + "peak_memory_mb": 0, + "memory_efficiency_ratio": 0, + "avg_latency_ms": 7.648905000000001, + "p50_latency_ms": 7.51215, + "p95_latency_ms": 8.228135, + "p99_latency_ms": 8.301627, + "encrypt_latency_ms": 0, + "decrypt_latency_ms": 0, + "timestamp": "2025-09-04 15:00:01", + "dotnet_version": "9.0.1", + "cpu_count": 12, + "total_memory_gb": 36, + "iterations": 20 + }, + { + "test_name": "concurrent", + "language": "net", + "data_size": 102400, + "concurrency": 8, + "operations_per_second": 86.60368702206944, + "bytes_per_second": 8868217.551059911, + "peak_memory_mb": 0, + "memory_efficiency_ratio": 0, + "avg_latency_ms": 11.5468525, + "p50_latency_ms": 11.364550000000001, + "p95_latency_ms": 13.353544999999999, + "p99_latency_ms": 14.037576999999999, + "encrypt_latency_ms": 0, + "decrypt_latency_ms": 0, + "timestamp": "2025-09-04 15:00:01", + "dotnet_version": "9.0.1", + "cpu_count": 12, + "total_memory_gb": 36, + "iterations": 40 + }, + { + "test_name": "concurrent", + "language": "net", + "data_size": 102400, + "concurrency": 16, + "operations_per_second": 57.96249652587285, + "bytes_per_second": 5935359.64424938, + "peak_memory_mb": 0, + "memory_efficiency_ratio": 0, + "avg_latency_ms": 17.252535000000005, + "p50_latency_ms": 18.42375, + "p95_latency_ms": 23.280634999999986, + "p99_latency_ms": 31.020590999999996, + "encrypt_latency_ms": 0, + "decrypt_latency_ms": 0, + "timestamp": "2025-09-04 15:00:01", + "dotnet_version": "9.0.1", + "cpu_count": 12, + "total_memory_gb": 36, + "iterations": 80 + }, + { + "test_name": "concurrent", + "language": "net", + "data_size": 512000, + "concurrency": 2, + "operations_per_second": 69.87475648647366, + "bytes_per_second": 35775875.321074516, + "peak_memory_mb": 0, + "memory_efficiency_ratio": 0, + "avg_latency_ms": 14.311319999999998, + "p50_latency_ms": 14.448450000000001, + "p95_latency_ms": 18.333225, + "p99_latency_ms": 19.231605, + "encrypt_latency_ms": 0, + "decrypt_latency_ms": 0, + "timestamp": "2025-09-04 15:00:01", + "dotnet_version": "9.0.1", + "cpu_count": 12, + "total_memory_gb": 36, + "iterations": 10 + }, + { + "test_name": "concurrent", + "language": "net", + "data_size": 512000, + "concurrency": 4, + "operations_per_second": 50.25283457395019, + "bytes_per_second": 25729451.301862497, + "peak_memory_mb": 0, + "memory_efficiency_ratio": 0, + "avg_latency_ms": 19.899375, + "p50_latency_ms": 20.209, + "p95_latency_ms": 23.7863, + "p99_latency_ms": 24.695259999999998, + "encrypt_latency_ms": 0, + "decrypt_latency_ms": 0, + "timestamp": "2025-09-04 15:00:01", + "dotnet_version": "9.0.1", + "cpu_count": 12, + "total_memory_gb": 36, + "iterations": 20 + }, + { + "test_name": "concurrent", + "language": "net", + "data_size": 512000, + "concurrency": 8, + "operations_per_second": 33.51412247417674, + "bytes_per_second": 17159230.70677849, + "peak_memory_mb": 0, + "memory_efficiency_ratio": 0, + "avg_latency_ms": 29.838167500000004, + "p50_latency_ms": 29.38465, + "p95_latency_ms": 38.73364, + "p99_latency_ms": 39.987908000000004, + "encrypt_latency_ms": 0, + "decrypt_latency_ms": 0, + "timestamp": "2025-09-04 15:00:02", + "dotnet_version": "9.0.1", + "cpu_count": 12, + "total_memory_gb": 36, + "iterations": 40 + }, + { + "test_name": "concurrent", + "language": "net", + "data_size": 512000, + "concurrency": 16, + "operations_per_second": 23.99113095870719, + "bytes_per_second": 12283459.05085808, + "peak_memory_mb": 0, + "memory_efficiency_ratio": 0, + "avg_latency_ms": 41.682069999999996, + "p50_latency_ms": 41.307900000000004, + "p95_latency_ms": 68.0154, + "p99_latency_ms": 73.19854199999995, + "encrypt_latency_ms": 0, + "decrypt_latency_ms": 0, + "timestamp": "2025-09-04 15:00:02", + "dotnet_version": "9.0.1", + "cpu_count": 12, + "total_memory_gb": 36, + "iterations": 80 + }, + { + "test_name": "concurrent", + "language": "net", + "data_size": 1048576, + "concurrency": 2, + "operations_per_second": 51.85237421651063, + "bytes_per_second": 54371155.14645185, + "peak_memory_mb": 0, + "memory_efficiency_ratio": 0, + "avg_latency_ms": 19.285519999999998, + "p50_latency_ms": 18.68185, + "p95_latency_ms": 24.263589999999994, + "p99_latency_ms": 25.557358, + "encrypt_latency_ms": 0, + "decrypt_latency_ms": 0, + "timestamp": "2025-09-04 15:00:02", + "dotnet_version": "9.0.1", + "cpu_count": 12, + "total_memory_gb": 36, + "iterations": 10 + }, + { + "test_name": "concurrent", + "language": "net", + "data_size": 1048576, + "concurrency": 4, + "operations_per_second": 36.437432101122624, + "bytes_per_second": 38207416.80286676, + "peak_memory_mb": 0, + "memory_efficiency_ratio": 0, + "avg_latency_ms": 27.444304999999996, + "p50_latency_ms": 27.783099999999997, + "p95_latency_ms": 33.71355, + "p99_latency_ms": 33.80095, + "encrypt_latency_ms": 0, + "decrypt_latency_ms": 0, + "timestamp": "2025-09-04 15:00:02", + "dotnet_version": "9.0.1", + "cpu_count": 12, + "total_memory_gb": 36, + "iterations": 20 + }, + { + "test_name": "concurrent", + "language": "net", + "data_size": 1048576, + "concurrency": 8, + "operations_per_second": 24.254780950828632, + "bytes_per_second": 25432981.190296084, + "peak_memory_mb": 0, + "memory_efficiency_ratio": 0, + "avg_latency_ms": 41.22898500000002, + "p50_latency_ms": 40.928399999999996, + "p95_latency_ms": 52.575745, + "p99_latency_ms": 56.82432, + "encrypt_latency_ms": 0, + "decrypt_latency_ms": 0, + "timestamp": "2025-09-04 15:00:02", + "dotnet_version": "9.0.1", + "cpu_count": 12, + "total_memory_gb": 36, + "iterations": 40 + }, + { + "test_name": "concurrent", + "language": "net", + "data_size": 1048576, + "concurrency": 16, + "operations_per_second": 20.620666067619133, + "bytes_per_second": 21622335.5425198, + "peak_memory_mb": 0, + "memory_efficiency_ratio": 0, + "avg_latency_ms": 48.49503875, + "p50_latency_ms": 51.778400000000005, + "p95_latency_ms": 69.36176999999999, + "p99_latency_ms": 73.26805599999997, + "encrypt_latency_ms": 0, + "decrypt_latency_ms": 0, + "timestamp": "2025-09-04 15:00:03", + "dotnet_version": "9.0.1", + "cpu_count": 12, + "total_memory_gb": 36, + "iterations": 80 + }, + { + "test_name": "concurrent", + "language": "net", + "data_size": 10485760, + "concurrency": 2, + "operations_per_second": 10.74849972440847, + "bytes_per_second": 112706188.47021335, + "peak_memory_mb": 0, + "memory_efficiency_ratio": 0, + "avg_latency_ms": 93.03623999999999, + "p50_latency_ms": 90.85415, + "p95_latency_ms": 99.63023000000001, + "p99_latency_ms": 99.852926, + "encrypt_latency_ms": 0, + "decrypt_latency_ms": 0, + "timestamp": "2025-09-04 15:00:03", + "dotnet_version": "9.0.1", + "cpu_count": 12, + "total_memory_gb": 36, + "iterations": 10 + }, + { + "test_name": "concurrent", + "language": "net", + "data_size": 10485760, + "concurrency": 4, + "operations_per_second": 6.206236721756534, + "bytes_per_second": 65077108.76752579, + "peak_memory_mb": 0, + "memory_efficiency_ratio": 0, + "avg_latency_ms": 161.12823999999998, + "p50_latency_ms": 160.0282, + "p95_latency_ms": 183.73433, + "p99_latency_ms": 187.075746, + "encrypt_latency_ms": 0, + "decrypt_latency_ms": 0, + "timestamp": "2025-09-04 15:00:04", + "dotnet_version": "9.0.1", + "cpu_count": 12, + "total_memory_gb": 36, + "iterations": 20 + }, + { + "test_name": "concurrent", + "language": "net", + "data_size": 10485760, + "concurrency": 8, + "operations_per_second": 3.162044962461191, + "bytes_per_second": 33156444.58557706, + "peak_memory_mb": 0, + "memory_efficiency_ratio": 0, + "avg_latency_ms": 316.25103749999994, + "p50_latency_ms": 318.7346, + "p95_latency_ms": 356.32260499999995, + "p99_latency_ms": 361.82463499999994, + "encrypt_latency_ms": 0, + "decrypt_latency_ms": 0, + "timestamp": "2025-09-04 15:00:06", + "dotnet_version": "9.0.1", + "cpu_count": 12, + "total_memory_gb": 36, + "iterations": 40 + }, + { + "test_name": "concurrent", + "language": "net", + "data_size": 10485760, + "concurrency": 16, + "operations_per_second": 2.3175545653230722, + "bytes_per_second": 24301320.958882056, + "peak_memory_mb": 0, + "memory_efficiency_ratio": 0, + "avg_latency_ms": 431.48930125000004, + "p50_latency_ms": 438.5942, + "p95_latency_ms": 685.451175, + "p99_latency_ms": 745.5463859999998, + "encrypt_latency_ms": 0, + "decrypt_latency_ms": 0, + "timestamp": "2025-09-04 15:00:09", + "dotnet_version": "9.0.1", + "cpu_count": 12, + "total_memory_gb": 36, + "iterations": 80 + }, + { + "test_name": "concurrent", + "language": "net", + "data_size": 52428800, + "concurrency": 2, + "operations_per_second": 1.8094432927438413, + "bytes_per_second": 94866940.5066083, + "peak_memory_mb": 0, + "memory_efficiency_ratio": 0, + "avg_latency_ms": 552.65617, + "p50_latency_ms": 546.17335, + "p95_latency_ms": 592.785525, + "p99_latency_ms": 593.0056649999999, + "encrypt_latency_ms": 0, + "decrypt_latency_ms": 0, + "timestamp": "2025-09-04 15:00:12", + "dotnet_version": "9.0.1", + "cpu_count": 12, + "total_memory_gb": 36, + "iterations": 10 + }, + { + "test_name": "concurrent", + "language": "net", + "data_size": 52428800, + "concurrency": 4, + "operations_per_second": 1.1653736942643278, + "bytes_per_second": 61099144.34184559, + "peak_memory_mb": 0, + "memory_efficiency_ratio": 0, + "avg_latency_ms": 858.09385, + "p50_latency_ms": 867.6488, + "p95_latency_ms": 932.01175, + "p99_latency_ms": 944.68475, + "encrypt_latency_ms": 0, + "decrypt_latency_ms": 0, + "timestamp": "2025-09-04 15:00:16", + "dotnet_version": "9.0.1", + "cpu_count": 12, + "total_memory_gb": 36, + "iterations": 20 + }, + { + "test_name": "concurrent", + "language": "net", + "data_size": 52428800, + "concurrency": 8, + "operations_per_second": 0.6042553191360797, + "bytes_per_second": 31680381.275921695, + "peak_memory_mb": 0, + "memory_efficiency_ratio": 0, + "avg_latency_ms": 1654.9295775000003, + "p50_latency_ms": 1650.32775, + "p95_latency_ms": 1963.07611, + "p99_latency_ms": 1976.301761, + "encrypt_latency_ms": 0, + "decrypt_latency_ms": 0, + "timestamp": "2025-09-04 15:00:25", + "dotnet_version": "9.0.1", + "cpu_count": 12, + "total_memory_gb": 36, + "iterations": 40 + }, + { + "test_name": "concurrent", + "language": "net", + "data_size": 52428800, + "concurrency": 16, + "operations_per_second": 0.24414588097622125, + "bytes_per_second": 12800275.564526109, + "peak_memory_mb": 0, + "memory_efficiency_ratio": 0, + "avg_latency_ms": 4095.91182125, + "p50_latency_ms": 4131.11515, + "p95_latency_ms": 5352.094895, + "p99_latency_ms": 5428.198392, + "encrypt_latency_ms": 0, + "decrypt_latency_ms": 0, + "timestamp": "2025-09-04 15:00:46", + "dotnet_version": "9.0.1", + "cpu_count": 12, + "total_memory_gb": 36, + "iterations": 80 + }, + { + "test_name": "concurrent", + "language": "net", + "data_size": 104857600, + "concurrency": 2, + "operations_per_second": 0.7779149025663529, + "bytes_per_second": 81570289.6873416, + "peak_memory_mb": 0, + "memory_efficiency_ratio": 0, + "avg_latency_ms": 1285.4876500000003, + "p50_latency_ms": 1281.24645, + "p95_latency_ms": 1304.61722, + "p99_latency_ms": 1307.325284, + "encrypt_latency_ms": 0, + "decrypt_latency_ms": 0, + "timestamp": "2025-09-04 15:00:53", + "dotnet_version": "9.0.1", + "cpu_count": 12, + "total_memory_gb": 36, + "iterations": 10 + }, + { + "test_name": "concurrent", + "language": "net", + "data_size": 104857600, + "concurrency": 4, + "operations_per_second": 0.51606548095823, + "bytes_per_second": 54113387.7761257, + "peak_memory_mb": 0, + "memory_efficiency_ratio": 0, + "avg_latency_ms": 1937.738595, + "p50_latency_ms": 1958.83855, + "p95_latency_ms": 2025.5288099999998, + "p99_latency_ms": 2030.2713620000002, + "encrypt_latency_ms": 0, + "decrypt_latency_ms": 0, + "timestamp": "2025-09-04 15:01:03", + "dotnet_version": "9.0.1", + "cpu_count": 12, + "total_memory_gb": 36, + "iterations": 20 + }, + { + "test_name": "concurrent", + "language": "net", + "data_size": 104857600, + "concurrency": 8, + "operations_per_second": 0.29010828717552734, + "bytes_per_second": 30420058.733336575, + "peak_memory_mb": 0, + "memory_efficiency_ratio": 0, + "avg_latency_ms": 3446.9887425, + "p50_latency_ms": 3351.3478, + "p95_latency_ms": 4149.81743, + "p99_latency_ms": 4166.405106, + "encrypt_latency_ms": 0, + "decrypt_latency_ms": 0, + "timestamp": "2025-09-04 15:01:20", + "dotnet_version": "9.0.1", + "cpu_count": 12, + "total_memory_gb": 36, + "iterations": 40 + }, + { + "test_name": "concurrent", + "language": "net", + "data_size": 104857600, + "concurrency": 16, + "operations_per_second": 0.08628624072965597, + "bytes_per_second": 9047768.115933975, + "peak_memory_mb": 0, + "memory_efficiency_ratio": 0, + "avg_latency_ms": 11589.333265000003, + "p50_latency_ms": 10009.5267, + "p95_latency_ms": 23819.583329999998, + "p99_latency_ms": 23974.41349, + "encrypt_latency_ms": 0, + "decrypt_latency_ms": 0, + "timestamp": "2025-09-04 15:02:19", + "dotnet_version": "9.0.1", + "cpu_count": 12, + "total_memory_gb": 36, + "iterations": 80 } ] } \ No newline at end of file From b72f0b326d82419d9bccfeffcf7f9750551fb612 Mon Sep 17 00:00:00 2001 From: Shubham Chaturvedi Date: Thu, 4 Sep 2025 15:28:50 -0700 Subject: [PATCH 4/5] fix: formatting --- esdk-performance-testing/results/raw-data/net_results.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/esdk-performance-testing/results/raw-data/net_results.json b/esdk-performance-testing/results/raw-data/net_results.json index 30ff3ec44..b484f922f 100644 --- a/esdk-performance-testing/results/raw-data/net_results.json +++ b/esdk-performance-testing/results/raw-data/net_results.json @@ -1143,4 +1143,4 @@ "iterations": 80 } ] -} \ No newline at end of file +} From 0a901e0032c9cc5991e54181f76992b12e1776e6 Mon Sep 17 00:00:00 2001 From: Shubham Chaturvedi Date: Tue, 9 Sep 2025 16:45:14 -0700 Subject: [PATCH 5/5] fix: PR Comments --- esdk-performance-testing/benchmarks/net/Benchmark.cs | 1 + esdk-performance-testing/benchmarks/net/Results.cs | 1 + esdk-performance-testing/benchmarks/net/Tests.cs | 1 + 3 files changed, 3 insertions(+) diff --git a/esdk-performance-testing/benchmarks/net/Benchmark.cs b/esdk-performance-testing/benchmarks/net/Benchmark.cs index be6985a22..d19ae20ea 100644 --- a/esdk-performance-testing/benchmarks/net/Benchmark.cs +++ b/esdk-performance-testing/benchmarks/net/Benchmark.cs @@ -105,6 +105,7 @@ public static double GetTotalMemoryGb() try { var gcMemoryInfo = GC.GetGCMemoryInfo(); + // Convert from bytes to GB return gcMemoryInfo.TotalAvailableMemoryBytes / (1024.0 * 1024.0 * 1024.0); } catch diff --git a/esdk-performance-testing/benchmarks/net/Results.cs b/esdk-performance-testing/benchmarks/net/Results.cs index 2e376d748..d63e39160 100644 --- a/esdk-performance-testing/benchmarks/net/Results.cs +++ b/esdk-performance-testing/benchmarks/net/Results.cs @@ -181,6 +181,7 @@ public static BenchmarkResult CreateThroughputResult(List encryptLatenci public static BenchmarkResult CreateMemoryResult(double peakMemoryMb, double avgMemoryMb, int dataSize, int cpuCount, double totalMemoryGb) { + // Convert memory from MB to bytes since data size is in bytes var memoryEfficiency = peakMemoryMb > 0 ? dataSize / (peakMemoryMb * 1024 * 1024) : 0.0; return new BenchmarkResult diff --git a/esdk-performance-testing/benchmarks/net/Tests.cs b/esdk-performance-testing/benchmarks/net/Tests.cs index afbd3657a..1aa76bd1b 100644 --- a/esdk-performance-testing/benchmarks/net/Tests.cs +++ b/esdk-performance-testing/benchmarks/net/Tests.cs @@ -206,6 +206,7 @@ private void SampleMemoryContinuously(long baselineHeap, ulong baselineAllocs, C var currentHeap = GC.GetTotalMemory(false); var currentAllocs = GC.GetTotalAllocatedBytes(false); + // Convert from bytes to MB var heapMB = (currentHeap - baselineHeap) / (1024.0 * 1024.0); var allocsMB = ((long)currentAllocs - (long)baselineAllocs) / (1024.0 * 1024.0);