diff --git a/src/BenchmarkDotNet.TestAdapter/Annotations/BenchmarkCaseConstraintAttribute.cs b/src/BenchmarkDotNet.TestAdapter/Annotations/BenchmarkCaseConstraintAttribute.cs
new file mode 100644
index 0000000000..2341180d4a
--- /dev/null
+++ b/src/BenchmarkDotNet.TestAdapter/Annotations/BenchmarkCaseConstraintAttribute.cs
@@ -0,0 +1,18 @@
+using BenchmarkDotNet.Reports;
+
+namespace BenchmarkDotNet.TestAdapter.Annotations
+{
+ ///
+ /// Base for TestAdapter constraints
+ ///
+ [System.AttributeUsage(System.AttributeTargets.Method, Inherited = false, AllowMultiple = true)]
+ public abstract class BenchmarkCaseConstraintAttribute : System.Attribute
+ {
+ ///
+ /// Validate the constraint
+ ///
+ ///
+ ///
+ protected internal abstract void Validate(BenchmarkReport report, System.Text.StringBuilder builder);
+ }
+}
diff --git a/src/BenchmarkDotNet.TestAdapter/Annotations/ComparisonOperators.cs b/src/BenchmarkDotNet.TestAdapter/Annotations/ComparisonOperators.cs
new file mode 100644
index 0000000000..3cf7827905
--- /dev/null
+++ b/src/BenchmarkDotNet.TestAdapter/Annotations/ComparisonOperators.cs
@@ -0,0 +1,37 @@
+namespace BenchmarkDotNet.TestAdapter.Annotations
+{
+ ///
+ /// Comparison Operators
+ ///
+ public enum ComparisonOperator
+ {
+ ///
+ /// Equal comparison operator
+ ///
+ Equal,
+ ///
+ /// Not Equal comparison operator
+ ///
+ NotEqual,
+ ///
+ /// comparison operator
+ ///
+ Greater,
+ ///
+ /// Greater than comparison operator
+ ///
+ GreaterOrEqual,
+ ///
+ /// Greater or Equal than comparison operator
+ ///
+ Less,
+ ///
+ /// Less than comparison operator
+ ///
+ LessOrEqual,
+ ///
+ /// Less or equal than comparison operator
+ ///
+ Between
+ }
+}
diff --git a/src/BenchmarkDotNet.TestAdapter/Annotations/GCGenCollectionsConstraintAttribute.cs b/src/BenchmarkDotNet.TestAdapter/Annotations/GCGenCollectionsConstraintAttribute.cs
new file mode 100644
index 0000000000..fa10d8bb8a
--- /dev/null
+++ b/src/BenchmarkDotNet.TestAdapter/Annotations/GCGenCollectionsConstraintAttribute.cs
@@ -0,0 +1,95 @@
+using BenchmarkDotNet.Reports;
+using System.Text;
+
+namespace BenchmarkDotNet.TestAdapter.Annotations
+{
+ ///
+ /// GC Gen Collections constraints
+ ///
+ public sealed class GCGenCollectionsConstraintAttribute : BenchmarkCaseConstraintAttribute
+ {
+ ///
+ /// Instance new
+ ///
+ ///
+ ///
+ ///
+ ///
+ public GCGenCollectionsConstraintAttribute(GCGeneration generation, ComparisonOperator @operator, int from, int? to)
+ {
+ Operator = @operator;
+ From = from;
+ To = to;
+ Generation = generation;
+ }
+
+ ///
+ /// Instance new
+ ///
+ ///
+ ///
+ ///
+ public GCGenCollectionsConstraintAttribute(GCGeneration generation, ComparisonOperator @operator, int from)
+ : this(generation, @operator, from, null)
+ {
+ }
+
+ ///
+ /// GC Generation
+ ///
+ public GCGeneration Generation { get; }
+
+ ///
+ /// Comparison Operator
+ ///
+ public ComparisonOperator Operator { get; }
+ ///
+ /// From mean value
+ ///
+ public int From { get; }
+ ///
+ /// To mean value
+ ///
+ public int? To { get; }
+
+ /// inheritdoc
+ protected internal override void Validate(BenchmarkReport report, StringBuilder builder)
+ {
+ var resultRuns = report.GetResultRuns();
+ var gcStats = report.GcStats;
+ var genCollections = Generation switch
+ {
+ GCGeneration.Gen0 => gcStats.Gen0Collections,
+ GCGeneration.Gen1 => gcStats.Gen1Collections,
+ GCGeneration.Gen2 => gcStats.Gen2Collections,
+ _ => throw new System.NotSupportedException(),
+ };
+ switch (Operator)
+ {
+ case ComparisonOperator.Equal when genCollections != From:
+ builder.AppendLine($"{Generation} is not equal to expected value {From}");
+ break;
+ case ComparisonOperator.NotEqual when genCollections == From:
+ builder.AppendLine($"{Generation} is equal to expected value {From}");
+ break;
+ case ComparisonOperator.Less when genCollections >= From:
+ builder.AppendLine($"{Generation} is greater or equal that expected value {From}");
+ break;
+ case ComparisonOperator.LessOrEqual when genCollections > From:
+ builder.AppendLine($"{Generation} is greater that expected value {From}");
+ break;
+ case ComparisonOperator.Greater when genCollections <= From:
+ builder.AppendLine($"{Generation} is less or equal that expected value {From}");
+ break;
+ case ComparisonOperator.GreaterOrEqual when genCollections < From:
+ builder.AppendLine($"{Generation} is lest that expected value {From}");
+ break;
+ case ComparisonOperator.Between when genCollections < From && genCollections > To:
+ builder.AppendLine($"{Generation} is not betwenn expected value [{From}-{To}]");
+ break;
+ default:
+ break;
+ }
+ }
+ }
+}
diff --git a/src/BenchmarkDotNet.TestAdapter/Annotations/GCGeneration.cs b/src/BenchmarkDotNet.TestAdapter/Annotations/GCGeneration.cs
new file mode 100644
index 0000000000..2b730d8581
--- /dev/null
+++ b/src/BenchmarkDotNet.TestAdapter/Annotations/GCGeneration.cs
@@ -0,0 +1,21 @@
+namespace BenchmarkDotNet.TestAdapter.Annotations
+{
+ ///
+ /// GC Generation
+ ///
+ public enum GCGeneration: int
+ {
+ ///
+ /// GC Generation 0
+ ///
+ Gen0,
+ ///
+ /// GC Generation 1
+ ///
+ Gen1,
+ ///
+ /// GC Generation 2
+ ///
+ Gen2
+ }
+}
diff --git a/src/BenchmarkDotNet.TestAdapter/Annotations/MeanConstraintAttribute.cs b/src/BenchmarkDotNet.TestAdapter/Annotations/MeanConstraintAttribute.cs
new file mode 100644
index 0000000000..7736002a7f
--- /dev/null
+++ b/src/BenchmarkDotNet.TestAdapter/Annotations/MeanConstraintAttribute.cs
@@ -0,0 +1,83 @@
+using BenchmarkDotNet.Extensions;
+using BenchmarkDotNet.Reports;
+using System.Text;
+
+namespace BenchmarkDotNet.TestAdapter.Annotations
+{
+ ///
+ /// /// mean constraints
+ ///
+ [System.AttributeUsage(System.AttributeTargets.Method, Inherited = false, AllowMultiple = true)]
+ public sealed class MeanConstraintAttribute : BenchmarkCaseConstraintAttribute
+ {
+ ///
+ /// Instance new
+ ///
+ ///
+ ///
+ ///
+ public MeanConstraintAttribute(ComparisonOperator @operator, double from, double? to)
+ {
+ Operator = @operator;
+ From = from;
+ To = to;
+ }
+
+ ///
+ /// Instance new
+ ///
+ ///
+ ///
+ public MeanConstraintAttribute(ComparisonOperator @operator, double from) :
+ this(@operator, from, null)
+ {
+ }
+
+ ///
+ /// Comparison Operator
+ ///
+ public ComparisonOperator Operator { get; }
+ ///
+ /// From mean value
+ ///
+ public double From { get; }
+ ///
+ /// To mean value
+ ///
+ public double? To { get; }
+
+ /// inheritdoc
+ protected internal override void Validate(BenchmarkReport report, StringBuilder builder)
+ {
+ var resultRuns = report.GetResultRuns();
+ var statistics = resultRuns.GetStatistics();
+ switch (Operator)
+ {
+ case ComparisonOperator.Equal when statistics.Mean != From:
+ builder.AppendLine($"{nameof(statistics.Mean)} is not equal to expected value {From}");
+ break;
+ case ComparisonOperator.NotEqual when statistics.Mean == From:
+ builder.AppendLine($"{nameof(statistics.Mean)} is equal to expected value {From}");
+ break;
+ case ComparisonOperator.Less when statistics.Mean >= From:
+ builder.AppendLine($"{nameof(statistics.Mean)} is greater or equal that expected value {From}");
+ break;
+ case ComparisonOperator.LessOrEqual when statistics.Mean > From:
+ builder.AppendLine($"{nameof(statistics.Mean)} is greater that expected value {From}");
+ break;
+ case ComparisonOperator.Greater when statistics.Mean <= From:
+ builder.AppendLine($"{nameof(statistics.Mean)} is less or equal that expected value {From}");
+ break;
+ case ComparisonOperator.GreaterOrEqual when statistics.Mean < From:
+ builder.AppendLine($"{nameof(statistics.Mean)} is lest that expected value {From}");
+ break;
+ case ComparisonOperator.Between when statistics.Mean < From && statistics.Mean > To:
+ builder.AppendLine($"{nameof(statistics.Mean)} is not between expected value [{From}-{To}]");
+ break;
+ default:
+ break;
+ }
+
+ }
+ }
+}
diff --git a/src/BenchmarkDotNet.TestAdapter/Annotations/StdDevConstraintAttribute.cs b/src/BenchmarkDotNet.TestAdapter/Annotations/StdDevConstraintAttribute.cs
new file mode 100644
index 0000000000..1c80905116
--- /dev/null
+++ b/src/BenchmarkDotNet.TestAdapter/Annotations/StdDevConstraintAttribute.cs
@@ -0,0 +1,82 @@
+using BenchmarkDotNet.Extensions;
+using BenchmarkDotNet.Reports;
+using System.Text;
+
+namespace BenchmarkDotNet.TestAdapter.Annotations
+{
+ ///
+ /// standard deviation constraints
+ ///
+ public sealed class StdDevConstraintAttribute : BenchmarkCaseConstraintAttribute
+ {
+ ///
+ /// Instance new
+ ///
+ ///
+ ///
+ ///
+ public StdDevConstraintAttribute(ComparisonOperator @operator, double from, double? to)
+ {
+ Operator = @operator;
+ From = from;
+ To = to;
+ }
+
+ ///
+ /// Instance new
+ ///
+ ///
+ ///
+ public StdDevConstraintAttribute(ComparisonOperator @operator, double from) :
+ this(@operator, from, null)
+ {
+ }
+
+ ///
+ /// Comparison Operator
+ ///
+ public ComparisonOperator Operator { get; }
+ ///
+ /// From mean value
+ ///
+ public double From { get; }
+ ///
+ /// To mean value
+ ///
+ public double? To { get; }
+
+ /// inheritdoc
+ protected internal override void Validate(BenchmarkReport report, StringBuilder builder)
+ {
+ var resultRuns = report.GetResultRuns();
+ var statistics = resultRuns.GetStatistics();
+ var stdDev = statistics.StandardDeviation;
+ switch (Operator)
+ {
+ case ComparisonOperator.Equal when stdDev != From:
+ builder.AppendLine($"{nameof(statistics.StandardDeviation)} is not equal to expected value {From}");
+ break;
+ case ComparisonOperator.NotEqual when stdDev == From:
+ builder.AppendLine($"{nameof(statistics.StandardDeviation)} is equal to expected value {From}");
+ break;
+ case ComparisonOperator.Less when stdDev >= From:
+ builder.AppendLine($"{nameof(statistics.StandardDeviation)} is greater or equal that expected value {From}");
+ break;
+ case ComparisonOperator.LessOrEqual when stdDev > From:
+ builder.AppendLine($"{nameof(statistics.StandardDeviation)} is greater that expected value {From}");
+ break;
+ case ComparisonOperator.Greater when stdDev <= From:
+ builder.AppendLine($"{nameof(statistics.StandardDeviation)} is less or equal that expected value {From}");
+ break;
+ case ComparisonOperator.GreaterOrEqual when stdDev < From:
+ builder.AppendLine($"{nameof(statistics.StandardDeviation)} is lest that expected value {From}");
+ break;
+ case ComparisonOperator.Between when stdDev < From && stdDev > To:
+ builder.AppendLine($"{nameof(statistics.StandardDeviation)} is not between expected value [{From}-{To}]");
+ break;
+ default:
+ break;
+ }
+ }
+ }
+}
diff --git a/src/BenchmarkDotNet.TestAdapter/Annotations/StdErrConstraintAttribute.cs b/src/BenchmarkDotNet.TestAdapter/Annotations/StdErrConstraintAttribute.cs
new file mode 100644
index 0000000000..ca1464e144
--- /dev/null
+++ b/src/BenchmarkDotNet.TestAdapter/Annotations/StdErrConstraintAttribute.cs
@@ -0,0 +1,85 @@
+using BenchmarkDotNet.Extensions;
+using BenchmarkDotNet.Reports;
+using System.Text;
+
+namespace BenchmarkDotNet.TestAdapter.Annotations
+{
+ ///
+ /// standard error constraints
+ ///
+ [System.AttributeUsage(System.AttributeTargets.Method, Inherited = false, AllowMultiple = true)]
+ public sealed class StdErrConstraintAttribute : BenchmarkCaseConstraintAttribute
+ {
+ ///
+ /// Instance new
+ ///
+ ///
+ ///
+ ///
+ public StdErrConstraintAttribute(ComparisonOperator @operator, double from, double? to)
+ {
+ Operator = @operator;
+ From = from;
+ To = to;
+ }
+
+ ///
+ /// Instance new
+ ///
+ ///
+ ///
+ public StdErrConstraintAttribute(ComparisonOperator @operator, double from) :
+ this(@operator, from, null)
+ {
+ }
+
+ ///
+ /// Comparison Operator
+ ///
+ public ComparisonOperator Operator { get; }
+ ///
+ /// From mean value
+ ///
+ public double From { get; }
+ ///
+ /// To mean value
+ ///
+ public double? To { get; }
+
+
+ /// inheritdoc
+ protected internal override void Validate(BenchmarkReport report, StringBuilder builder)
+ {
+ var resultRuns = report.GetResultRuns();
+ var statistics = resultRuns.GetStatistics();
+ var stdErr = statistics.StandardError;
+ switch (Operator)
+ {
+ case ComparisonOperator.Equal when stdErr != From:
+ builder.AppendLine($"{nameof(statistics.StandardError)} is not equal to expected value {From}");
+ break;
+ case ComparisonOperator.NotEqual when stdErr == From:
+ builder.AppendLine($"{nameof(statistics.StandardError)} is equal to expected value {From}");
+ break;
+ case ComparisonOperator.Less when stdErr >= From:
+ builder.AppendLine($"{nameof(statistics.StandardError)} is greater or equal that expected value {From}");
+ break;
+ case ComparisonOperator.LessOrEqual when stdErr > From:
+ builder.AppendLine($"{nameof(statistics.StandardError)} is greater that expected value {From}");
+ break;
+ case ComparisonOperator.Greater when stdErr <= From:
+ builder.AppendLine($"{nameof(statistics.StandardError)} is less or equal that expected value {From}");
+ break;
+ case ComparisonOperator.GreaterOrEqual when stdErr < From:
+ builder.AppendLine($"{nameof(statistics.StandardError)} is lest that expected value {From}");
+ break;
+ case ComparisonOperator.Between when stdErr < From && stdErr > To:
+ builder.AppendLine($"{nameof(statistics.StandardError)} is not between expected value [{From}-{To}]");
+ break;
+ default:
+ break;
+ }
+
+ }
+ }
+}
diff --git a/src/BenchmarkDotNet.TestAdapter/Annotations/TotalAllocatedBytesConstraintAttribute.cs b/src/BenchmarkDotNet.TestAdapter/Annotations/TotalAllocatedBytesConstraintAttribute.cs
new file mode 100644
index 0000000000..70341f14bc
--- /dev/null
+++ b/src/BenchmarkDotNet.TestAdapter/Annotations/TotalAllocatedBytesConstraintAttribute.cs
@@ -0,0 +1,90 @@
+using BenchmarkDotNet.Reports;
+using System.Text;
+
+namespace BenchmarkDotNet.TestAdapter.Annotations
+{
+ ///
+ /// GC Total Allocated bytes constraints
+ ///
+ public sealed class TotalAllocatedBytesConstraintAttribute : BenchmarkCaseConstraintAttribute
+ {
+ private readonly bool excludeAllocationQuantumSideEffects;
+
+ ///
+ /// Instance new
+ ///
+ ///
+ ///
+ ///
+ /// Allocation quantum can affecting some of our nano-benchmarks in non-deterministic way.
+ /// when this parameter is set to true and the number of all allocated bytes is less or equal AQ, we ignore AQ and put 0 to the results
+ public TotalAllocatedBytesConstraintAttribute(ComparisonOperator @operator,
+ long from,
+ long? to,
+ bool excludeAllocationQuantumSideEffects = false
+ )
+ {
+ Operator = @operator;
+ From = from;
+ To = to;
+ this.excludeAllocationQuantumSideEffects = excludeAllocationQuantumSideEffects;
+ }
+
+ ///
+ /// Instance new
+ ///
+ ///
+ ///
+ /// Allocation quantum can affecting some of our nano-benchmarks in non-deterministic way.
+ /// when this parameter is set to true and the number of all allocated bytes is less or equal AQ, we ignore AQ and put 0 to the results
+ public TotalAllocatedBytesConstraintAttribute(ComparisonOperator @operator, long from, bool excludeAllocationQuantumSideEffects = false) :
+ this(@operator, from, null)
+ {
+ }
+
+ ///
+ /// Comparison Operator
+ ///
+ public ComparisonOperator Operator { get; }
+ ///
+ /// From mean value
+ ///
+ public long From { get; }
+ ///
+ /// To mean value
+ ///
+ public long? To { get; }
+
+ /// inheritdoc
+ protected internal override void Validate(BenchmarkReport report, StringBuilder builder)
+ {
+ var totalAllocatedBytes = report.GcStats.GetTotalAllocatedBytes(excludeAllocationQuantumSideEffects);
+ switch (Operator)
+ {
+ case ComparisonOperator.Equal when totalAllocatedBytes != From:
+ builder.AppendLine($"GC total allocated bytes is not equal to expected value {From}");
+ break;
+ case ComparisonOperator.NotEqual when totalAllocatedBytes == From:
+ builder.AppendLine($"GC total allocated bytes is equal to expected value {From}");
+ break;
+ case ComparisonOperator.Less when totalAllocatedBytes >= From:
+ builder.AppendLine($"GC total allocated bytes is greater or equal that expected value {From}");
+ break;
+ case ComparisonOperator.LessOrEqual when totalAllocatedBytes > From:
+ builder.AppendLine($"GC total allocated bytes is greater that expected value {From}");
+ break;
+ case ComparisonOperator.Greater when totalAllocatedBytes <= From:
+ builder.AppendLine($"GC total allocated bytes is less or equal that expected value {From}");
+ break;
+ case ComparisonOperator.GreaterOrEqual when totalAllocatedBytes < From:
+ builder.AppendLine($"GC total allocated bytes is lest that expected value {From}");
+ break;
+ case ComparisonOperator.Between when totalAllocatedBytes < From && totalAllocatedBytes > To:
+ builder.AppendLine($"GC total allocated bytes is not between expected value [{From}-{To}]");
+ break;
+ default:
+ break;
+ }
+ }
+ }
+}
diff --git a/src/BenchmarkDotNet.TestAdapter/VSTestEventProcessor.cs b/src/BenchmarkDotNet.TestAdapter/VSTestEventProcessor.cs
index 35438d4d80..2d4e315593 100644
--- a/src/BenchmarkDotNet.TestAdapter/VSTestEventProcessor.cs
+++ b/src/BenchmarkDotNet.TestAdapter/VSTestEventProcessor.cs
@@ -130,6 +130,26 @@ public override void OnEndRunBenchmark(BenchmarkCase benchmarkCase, BenchmarkRep
var cultureInfo = CultureInfo.InvariantCulture;
var formatter = statistics.CreateNanosecondFormatter(cultureInfo);
+ if (testResult.ErrorMessage is null)
+ {
+ // Check thath WorkloadMethod has constraints
+ var constraints = benchmarkCase.Descriptor.WorkloadMethod.GetCustomAttributes(false).OfType();
+ if (constraints.Any())
+ {
+ var constraintsBuilder = new StringBuilder();
+ // Validete all constraints
+ foreach (var constraint in constraints)
+ {
+ constraint.Validate(report, constraintsBuilder);
+ }
+ if (constraintsBuilder.Length > 0)
+ {
+ testResult.Outcome = TestOutcome.Failed;
+ testResult.ErrorMessage = $"// Constraint Errors: {constraintsBuilder}";
+ }
+ }
+ }
+
var builder = new StringBuilder();
var histogram = HistogramBuilder.Adaptive.Build(statistics.Sample.Values);
builder.AppendLine("-------------------- Histogram --------------------");