diff --git a/src/BenchmarkDotNet.TestAdapter/Annotations/BenchmarkCaseConstraintAttribute.cs b/src/BenchmarkDotNet.TestAdapter/Annotations/BenchmarkCaseConstraintAttribute.cs new file mode 100644 index 0000000000..2341180d4a --- /dev/null +++ b/src/BenchmarkDotNet.TestAdapter/Annotations/BenchmarkCaseConstraintAttribute.cs @@ -0,0 +1,18 @@ +using BenchmarkDotNet.Reports; + +namespace BenchmarkDotNet.TestAdapter.Annotations +{ + /// + /// Base for TestAdapter constraints + /// + [System.AttributeUsage(System.AttributeTargets.Method, Inherited = false, AllowMultiple = true)] + public abstract class BenchmarkCaseConstraintAttribute : System.Attribute + { + /// + /// Validate the constraint + /// + /// + /// + protected internal abstract void Validate(BenchmarkReport report, System.Text.StringBuilder builder); + } +} diff --git a/src/BenchmarkDotNet.TestAdapter/Annotations/ComparisonOperators.cs b/src/BenchmarkDotNet.TestAdapter/Annotations/ComparisonOperators.cs new file mode 100644 index 0000000000..3cf7827905 --- /dev/null +++ b/src/BenchmarkDotNet.TestAdapter/Annotations/ComparisonOperators.cs @@ -0,0 +1,37 @@ +namespace BenchmarkDotNet.TestAdapter.Annotations +{ + /// + /// Comparison Operators + /// + public enum ComparisonOperator + { + /// + /// Equal comparison operator + /// + Equal, + /// + /// Not Equal comparison operator + /// + NotEqual, + /// + /// comparison operator + /// + Greater, + /// + /// Greater than comparison operator + /// + GreaterOrEqual, + /// + /// Greater or Equal than comparison operator + /// + Less, + /// + /// Less than comparison operator + /// + LessOrEqual, + /// + /// Less or equal than comparison operator + /// + Between + } +} diff --git a/src/BenchmarkDotNet.TestAdapter/Annotations/GCGenCollectionsConstraintAttribute.cs b/src/BenchmarkDotNet.TestAdapter/Annotations/GCGenCollectionsConstraintAttribute.cs new file mode 100644 index 0000000000..fa10d8bb8a --- /dev/null +++ b/src/BenchmarkDotNet.TestAdapter/Annotations/GCGenCollectionsConstraintAttribute.cs @@ -0,0 +1,95 @@ +using BenchmarkDotNet.Reports; +using System.Text; + +namespace BenchmarkDotNet.TestAdapter.Annotations +{ + /// + /// GC Gen Collections constraints + /// + public sealed class GCGenCollectionsConstraintAttribute : BenchmarkCaseConstraintAttribute + { + /// + /// Instance new + /// + /// + /// + /// + /// + public GCGenCollectionsConstraintAttribute(GCGeneration generation, ComparisonOperator @operator, int from, int? to) + { + Operator = @operator; + From = from; + To = to; + Generation = generation; + } + + /// + /// Instance new + /// + /// + /// + /// + public GCGenCollectionsConstraintAttribute(GCGeneration generation, ComparisonOperator @operator, int from) + : this(generation, @operator, from, null) + { + } + + /// + /// GC Generation + /// + public GCGeneration Generation { get; } + + /// + /// Comparison Operator + /// + public ComparisonOperator Operator { get; } + /// + /// From mean value + /// + public int From { get; } + /// + /// To mean value + /// + public int? To { get; } + + /// inheritdoc + protected internal override void Validate(BenchmarkReport report, StringBuilder builder) + { + var resultRuns = report.GetResultRuns(); + var gcStats = report.GcStats; + var genCollections = Generation switch + { + GCGeneration.Gen0 => gcStats.Gen0Collections, + GCGeneration.Gen1 => gcStats.Gen1Collections, + GCGeneration.Gen2 => gcStats.Gen2Collections, + _ => throw new System.NotSupportedException(), + }; + switch (Operator) + { + case ComparisonOperator.Equal when genCollections != From: + builder.AppendLine($"{Generation} is not equal to expected value {From}"); + break; + case ComparisonOperator.NotEqual when genCollections == From: + builder.AppendLine($"{Generation} is equal to expected value {From}"); + break; + case ComparisonOperator.Less when genCollections >= From: + builder.AppendLine($"{Generation} is greater or equal that expected value {From}"); + break; + case ComparisonOperator.LessOrEqual when genCollections > From: + builder.AppendLine($"{Generation} is greater that expected value {From}"); + break; + case ComparisonOperator.Greater when genCollections <= From: + builder.AppendLine($"{Generation} is less or equal that expected value {From}"); + break; + case ComparisonOperator.GreaterOrEqual when genCollections < From: + builder.AppendLine($"{Generation} is lest that expected value {From}"); + break; + case ComparisonOperator.Between when genCollections < From && genCollections > To: + builder.AppendLine($"{Generation} is not betwenn expected value [{From}-{To}]"); + break; + default: + break; + } + } + } +} diff --git a/src/BenchmarkDotNet.TestAdapter/Annotations/GCGeneration.cs b/src/BenchmarkDotNet.TestAdapter/Annotations/GCGeneration.cs new file mode 100644 index 0000000000..2b730d8581 --- /dev/null +++ b/src/BenchmarkDotNet.TestAdapter/Annotations/GCGeneration.cs @@ -0,0 +1,21 @@ +namespace BenchmarkDotNet.TestAdapter.Annotations +{ + /// + /// GC Generation + /// + public enum GCGeneration: int + { + /// + /// GC Generation 0 + /// + Gen0, + /// + /// GC Generation 1 + /// + Gen1, + /// + /// GC Generation 2 + /// + Gen2 + } +} diff --git a/src/BenchmarkDotNet.TestAdapter/Annotations/MeanConstraintAttribute.cs b/src/BenchmarkDotNet.TestAdapter/Annotations/MeanConstraintAttribute.cs new file mode 100644 index 0000000000..7736002a7f --- /dev/null +++ b/src/BenchmarkDotNet.TestAdapter/Annotations/MeanConstraintAttribute.cs @@ -0,0 +1,83 @@ +using BenchmarkDotNet.Extensions; +using BenchmarkDotNet.Reports; +using System.Text; + +namespace BenchmarkDotNet.TestAdapter.Annotations +{ + /// + /// /// mean constraints + /// + [System.AttributeUsage(System.AttributeTargets.Method, Inherited = false, AllowMultiple = true)] + public sealed class MeanConstraintAttribute : BenchmarkCaseConstraintAttribute + { + /// + /// Instance new + /// + /// + /// + /// + public MeanConstraintAttribute(ComparisonOperator @operator, double from, double? to) + { + Operator = @operator; + From = from; + To = to; + } + + /// + /// Instance new + /// + /// + /// + public MeanConstraintAttribute(ComparisonOperator @operator, double from) : + this(@operator, from, null) + { + } + + /// + /// Comparison Operator + /// + public ComparisonOperator Operator { get; } + /// + /// From mean value + /// + public double From { get; } + /// + /// To mean value + /// + public double? To { get; } + + /// inheritdoc + protected internal override void Validate(BenchmarkReport report, StringBuilder builder) + { + var resultRuns = report.GetResultRuns(); + var statistics = resultRuns.GetStatistics(); + switch (Operator) + { + case ComparisonOperator.Equal when statistics.Mean != From: + builder.AppendLine($"{nameof(statistics.Mean)} is not equal to expected value {From}"); + break; + case ComparisonOperator.NotEqual when statistics.Mean == From: + builder.AppendLine($"{nameof(statistics.Mean)} is equal to expected value {From}"); + break; + case ComparisonOperator.Less when statistics.Mean >= From: + builder.AppendLine($"{nameof(statistics.Mean)} is greater or equal that expected value {From}"); + break; + case ComparisonOperator.LessOrEqual when statistics.Mean > From: + builder.AppendLine($"{nameof(statistics.Mean)} is greater that expected value {From}"); + break; + case ComparisonOperator.Greater when statistics.Mean <= From: + builder.AppendLine($"{nameof(statistics.Mean)} is less or equal that expected value {From}"); + break; + case ComparisonOperator.GreaterOrEqual when statistics.Mean < From: + builder.AppendLine($"{nameof(statistics.Mean)} is lest that expected value {From}"); + break; + case ComparisonOperator.Between when statistics.Mean < From && statistics.Mean > To: + builder.AppendLine($"{nameof(statistics.Mean)} is not between expected value [{From}-{To}]"); + break; + default: + break; + } + + } + } +} diff --git a/src/BenchmarkDotNet.TestAdapter/Annotations/StdDevConstraintAttribute.cs b/src/BenchmarkDotNet.TestAdapter/Annotations/StdDevConstraintAttribute.cs new file mode 100644 index 0000000000..1c80905116 --- /dev/null +++ b/src/BenchmarkDotNet.TestAdapter/Annotations/StdDevConstraintAttribute.cs @@ -0,0 +1,82 @@ +using BenchmarkDotNet.Extensions; +using BenchmarkDotNet.Reports; +using System.Text; + +namespace BenchmarkDotNet.TestAdapter.Annotations +{ + /// + /// standard deviation constraints + /// + public sealed class StdDevConstraintAttribute : BenchmarkCaseConstraintAttribute + { + /// + /// Instance new + /// + /// + /// + /// + public StdDevConstraintAttribute(ComparisonOperator @operator, double from, double? to) + { + Operator = @operator; + From = from; + To = to; + } + + /// + /// Instance new + /// + /// + /// + public StdDevConstraintAttribute(ComparisonOperator @operator, double from) : + this(@operator, from, null) + { + } + + /// + /// Comparison Operator + /// + public ComparisonOperator Operator { get; } + /// + /// From mean value + /// + public double From { get; } + /// + /// To mean value + /// + public double? To { get; } + + /// inheritdoc + protected internal override void Validate(BenchmarkReport report, StringBuilder builder) + { + var resultRuns = report.GetResultRuns(); + var statistics = resultRuns.GetStatistics(); + var stdDev = statistics.StandardDeviation; + switch (Operator) + { + case ComparisonOperator.Equal when stdDev != From: + builder.AppendLine($"{nameof(statistics.StandardDeviation)} is not equal to expected value {From}"); + break; + case ComparisonOperator.NotEqual when stdDev == From: + builder.AppendLine($"{nameof(statistics.StandardDeviation)} is equal to expected value {From}"); + break; + case ComparisonOperator.Less when stdDev >= From: + builder.AppendLine($"{nameof(statistics.StandardDeviation)} is greater or equal that expected value {From}"); + break; + case ComparisonOperator.LessOrEqual when stdDev > From: + builder.AppendLine($"{nameof(statistics.StandardDeviation)} is greater that expected value {From}"); + break; + case ComparisonOperator.Greater when stdDev <= From: + builder.AppendLine($"{nameof(statistics.StandardDeviation)} is less or equal that expected value {From}"); + break; + case ComparisonOperator.GreaterOrEqual when stdDev < From: + builder.AppendLine($"{nameof(statistics.StandardDeviation)} is lest that expected value {From}"); + break; + case ComparisonOperator.Between when stdDev < From && stdDev > To: + builder.AppendLine($"{nameof(statistics.StandardDeviation)} is not between expected value [{From}-{To}]"); + break; + default: + break; + } + } + } +} diff --git a/src/BenchmarkDotNet.TestAdapter/Annotations/StdErrConstraintAttribute.cs b/src/BenchmarkDotNet.TestAdapter/Annotations/StdErrConstraintAttribute.cs new file mode 100644 index 0000000000..ca1464e144 --- /dev/null +++ b/src/BenchmarkDotNet.TestAdapter/Annotations/StdErrConstraintAttribute.cs @@ -0,0 +1,85 @@ +using BenchmarkDotNet.Extensions; +using BenchmarkDotNet.Reports; +using System.Text; + +namespace BenchmarkDotNet.TestAdapter.Annotations +{ + /// + /// standard error constraints + /// + [System.AttributeUsage(System.AttributeTargets.Method, Inherited = false, AllowMultiple = true)] + public sealed class StdErrConstraintAttribute : BenchmarkCaseConstraintAttribute + { + /// + /// Instance new + /// + /// + /// + /// + public StdErrConstraintAttribute(ComparisonOperator @operator, double from, double? to) + { + Operator = @operator; + From = from; + To = to; + } + + /// + /// Instance new + /// + /// + /// + public StdErrConstraintAttribute(ComparisonOperator @operator, double from) : + this(@operator, from, null) + { + } + + /// + /// Comparison Operator + /// + public ComparisonOperator Operator { get; } + /// + /// From mean value + /// + public double From { get; } + /// + /// To mean value + /// + public double? To { get; } + + + /// inheritdoc + protected internal override void Validate(BenchmarkReport report, StringBuilder builder) + { + var resultRuns = report.GetResultRuns(); + var statistics = resultRuns.GetStatistics(); + var stdErr = statistics.StandardError; + switch (Operator) + { + case ComparisonOperator.Equal when stdErr != From: + builder.AppendLine($"{nameof(statistics.StandardError)} is not equal to expected value {From}"); + break; + case ComparisonOperator.NotEqual when stdErr == From: + builder.AppendLine($"{nameof(statistics.StandardError)} is equal to expected value {From}"); + break; + case ComparisonOperator.Less when stdErr >= From: + builder.AppendLine($"{nameof(statistics.StandardError)} is greater or equal that expected value {From}"); + break; + case ComparisonOperator.LessOrEqual when stdErr > From: + builder.AppendLine($"{nameof(statistics.StandardError)} is greater that expected value {From}"); + break; + case ComparisonOperator.Greater when stdErr <= From: + builder.AppendLine($"{nameof(statistics.StandardError)} is less or equal that expected value {From}"); + break; + case ComparisonOperator.GreaterOrEqual when stdErr < From: + builder.AppendLine($"{nameof(statistics.StandardError)} is lest that expected value {From}"); + break; + case ComparisonOperator.Between when stdErr < From && stdErr > To: + builder.AppendLine($"{nameof(statistics.StandardError)} is not between expected value [{From}-{To}]"); + break; + default: + break; + } + + } + } +} diff --git a/src/BenchmarkDotNet.TestAdapter/Annotations/TotalAllocatedBytesConstraintAttribute.cs b/src/BenchmarkDotNet.TestAdapter/Annotations/TotalAllocatedBytesConstraintAttribute.cs new file mode 100644 index 0000000000..70341f14bc --- /dev/null +++ b/src/BenchmarkDotNet.TestAdapter/Annotations/TotalAllocatedBytesConstraintAttribute.cs @@ -0,0 +1,90 @@ +using BenchmarkDotNet.Reports; +using System.Text; + +namespace BenchmarkDotNet.TestAdapter.Annotations +{ + /// + /// GC Total Allocated bytes constraints + /// + public sealed class TotalAllocatedBytesConstraintAttribute : BenchmarkCaseConstraintAttribute + { + private readonly bool excludeAllocationQuantumSideEffects; + + /// + /// Instance new + /// + /// + /// + /// + /// Allocation quantum can affecting some of our nano-benchmarks in non-deterministic way. + /// when this parameter is set to true and the number of all allocated bytes is less or equal AQ, we ignore AQ and put 0 to the results + public TotalAllocatedBytesConstraintAttribute(ComparisonOperator @operator, + long from, + long? to, + bool excludeAllocationQuantumSideEffects = false + ) + { + Operator = @operator; + From = from; + To = to; + this.excludeAllocationQuantumSideEffects = excludeAllocationQuantumSideEffects; + } + + /// + /// Instance new + /// + /// + /// + /// Allocation quantum can affecting some of our nano-benchmarks in non-deterministic way. + /// when this parameter is set to true and the number of all allocated bytes is less or equal AQ, we ignore AQ and put 0 to the results + public TotalAllocatedBytesConstraintAttribute(ComparisonOperator @operator, long from, bool excludeAllocationQuantumSideEffects = false) : + this(@operator, from, null) + { + } + + /// + /// Comparison Operator + /// + public ComparisonOperator Operator { get; } + /// + /// From mean value + /// + public long From { get; } + /// + /// To mean value + /// + public long? To { get; } + + /// inheritdoc + protected internal override void Validate(BenchmarkReport report, StringBuilder builder) + { + var totalAllocatedBytes = report.GcStats.GetTotalAllocatedBytes(excludeAllocationQuantumSideEffects); + switch (Operator) + { + case ComparisonOperator.Equal when totalAllocatedBytes != From: + builder.AppendLine($"GC total allocated bytes is not equal to expected value {From}"); + break; + case ComparisonOperator.NotEqual when totalAllocatedBytes == From: + builder.AppendLine($"GC total allocated bytes is equal to expected value {From}"); + break; + case ComparisonOperator.Less when totalAllocatedBytes >= From: + builder.AppendLine($"GC total allocated bytes is greater or equal that expected value {From}"); + break; + case ComparisonOperator.LessOrEqual when totalAllocatedBytes > From: + builder.AppendLine($"GC total allocated bytes is greater that expected value {From}"); + break; + case ComparisonOperator.Greater when totalAllocatedBytes <= From: + builder.AppendLine($"GC total allocated bytes is less or equal that expected value {From}"); + break; + case ComparisonOperator.GreaterOrEqual when totalAllocatedBytes < From: + builder.AppendLine($"GC total allocated bytes is lest that expected value {From}"); + break; + case ComparisonOperator.Between when totalAllocatedBytes < From && totalAllocatedBytes > To: + builder.AppendLine($"GC total allocated bytes is not between expected value [{From}-{To}]"); + break; + default: + break; + } + } + } +} diff --git a/src/BenchmarkDotNet.TestAdapter/VSTestEventProcessor.cs b/src/BenchmarkDotNet.TestAdapter/VSTestEventProcessor.cs index 35438d4d80..2d4e315593 100644 --- a/src/BenchmarkDotNet.TestAdapter/VSTestEventProcessor.cs +++ b/src/BenchmarkDotNet.TestAdapter/VSTestEventProcessor.cs @@ -130,6 +130,26 @@ public override void OnEndRunBenchmark(BenchmarkCase benchmarkCase, BenchmarkRep var cultureInfo = CultureInfo.InvariantCulture; var formatter = statistics.CreateNanosecondFormatter(cultureInfo); + if (testResult.ErrorMessage is null) + { + // Check thath WorkloadMethod has constraints + var constraints = benchmarkCase.Descriptor.WorkloadMethod.GetCustomAttributes(false).OfType(); + if (constraints.Any()) + { + var constraintsBuilder = new StringBuilder(); + // Validete all constraints + foreach (var constraint in constraints) + { + constraint.Validate(report, constraintsBuilder); + } + if (constraintsBuilder.Length > 0) + { + testResult.Outcome = TestOutcome.Failed; + testResult.ErrorMessage = $"// Constraint Errors: {constraintsBuilder}"; + } + } + } + var builder = new StringBuilder(); var histogram = HistogramBuilder.Adaptive.Build(statistics.Sample.Values); builder.AppendLine("-------------------- Histogram --------------------");