Failure thresholds mtp

Documentation/Coverlet.MTP.Integration.md

@@ -87,6 +87,9 @@ dotnet exec <test-assembly.dll> --help
 | `--coverlet-skip-auto-props` | Skip auto-implemented properties. (default: `false`) |
 | `--coverlet-does-not-return-attribute <attribute>` | Attributes that mark methods as not returning. Can be specified multiple times. (default: `none`) |
 | `--coverlet-exclude-assemblies-without-sources <value>` | Exclude assemblies without source code. Values: `MissingAll`, `MissingAny`, `None`. (default: `None`) |
+| `--threshold <threshold>` | Exits with error if the coverage % is below value |
+| `--threshold-type <branch\line\|method>` | Coverage type to apply the threshold to. [default: `line`, `branch`, `method`] |
+| `--threshold-stat <Average\|Minimum\|Total>` | Coverage statistic used to enforce the threshold value. [default: Minimum] |
 
 > [!TIP]
 > If you encounter instrumentation failures like "The process cannot access the file ... because it is being used by another process", try setting `--coverlet-exclude-assemblies-without-sources MissingAll` (or in a config file: `"ExcludeAssembliesWithoutSources": "MissingAll"`) to skip assemblies without sources and reduce access conflicts.
@@ -167,21 +170,21 @@ The `testconfig.json` format is the standard configuration file for Microsoft Te
 
 | Key | Type | Description |
 | :--- | :--- | :---------- |
-| `include` | string | Comma-separated include filters (e.g., `[MyApp.*]*`) |
-| `includeDirectory` | string | Comma-separated additional directories for sources |
-| `exclude` | string | Comma-separated exclude filters (e.g., `[*.Tests]*`) |
-| `excludeByFile` | string | Comma-separated glob patterns for source file exclusion |
-| `excludeByAttribute` | string | Comma-separated attributes to exclude |
-| `format` | string | Comma-separated output formats (default: `cobertura`) |
-| `useSourceLink` | bool | Enable SourceLink support |
-| `singleHit` | bool | Limit hits to one per location |
-| `includeTestAssembly` | bool | Include test assembly in coverage |
-| `skipAutoProps` | bool | Skip auto-implemented properties |
-| `doesNotReturnAttribute` | string | Comma-separated attributes marking non-returning methods |
-| `deterministicReport` | bool | Generate deterministic reports |
-| `excludeAssembliesWithoutSources` | string | Values: `MissingAll`, `MissingAny`, `None` |
-| `disableManagedInstrumentationRestore` | bool | Disable managed instrumentation restore |
-| `mergeWith` | string | Path to existing coverage file to merge with |
+| `Include` | string | Comma-separated include filters (e.g., `[MyApp.*]*`) |
+| `IncludeDirectory` | string | Comma-separated additional directories for sources |
+| `Exclude` | string | Comma-separated exclude filters (e.g., `[*.Tests]*`) |
+| `ExcludeByFile` | string | Comma-separated glob patterns for source file exclusion |
+| `ExcludeByAttribute` | string | Comma-separated attributes to exclude |
+| `Format` | string | Comma-separated output formats (default: `cobertura`) |
+| `UseSourceLink` | bool | Enable SourceLink support |
+| `SingleHit` | bool | Limit hits to one per location |
+| `IncludeTestAssembly` | bool | Include test assembly in coverage |
+| `SkipAutoProps` | bool | Skip auto-implemented properties |
+| `DoesNotReturnAttribute` | string | Comma-separated attributes marking non-returning methods |
+| `DeterministicReport` | bool | Generate deterministic reports |
+| `ExcludeAssembliesWithoutSources` | string | Values: `MissingAll`, `MissingAny`, `None` |
+| `DisableManagedInstrumentationRestore` | bool | Disable managed instrumentation restore |
+| `MergeWith` | string | Path to existing coverage file to merge with |
 
 > [!NOTE]
 > Keys in `testconfig.json` use **camelCase** (e.g., `excludeByAttribute`), following the Microsoft Testing Platform convention.
@@ -236,6 +239,9 @@ The legacy `coverlet.mtp.appsettings.json` format is still supported for backwar
 | `DoesNotReturnAttribute` | string | Comma-separated attributes marking non-returning methods |
 | `DeterministicReport` | bool | Generate deterministic reports |
 | `ExcludeAssembliesWithoutSources` | string | Values: `MissingAll`, `MissingAny`, `None` (default: `MissingAll`) |
+| `Threshold` | int | Exits with error if the code coverage [0..100%]  is below value |
+| `ThresholdType` | string | Comma-separated coverage type to apply the Threshold to. [default: `line`, `branch`, `method`] |
+| `ThresholdStat` | string | Coverage statistic used to enforce the threshold value. [default: `Minimum`, `Average`, `Total`] |
 
 **Example `coverlet.mtp.appsettings.json`:**
 

Documentation/DriversFeatures.md

@@ -9,11 +9,11 @@ In the table below we keep track of main differences:
 | Feature                            | MSBuild       | .NET Tool    | VS DataCollector  | MTP Extension |
 |:-----------------------------------|:--------------|:-------------|:------------------|:--------------|
 | .NET Core support(>= 8.0)          | Yes           | Yes          | Yes               | Yes           |
-| .NET Framework support(>= 4.7.2)   | Yes           | Yes          | Yes(since 3.0.0)  | No            |
+| .NET Framework support(>= 4.7.2)   | Yes           | Yes          | Yes(since 3.0.0)  | Yes           |
 | Show result on console             | Yes           | Yes          | No                | Yes           |
 | Deterministic reports output folder| Yes           | Yes          | No                | No            |
 | Merge reports                      | Yes           | Yes          | No                | No            |
-| Coverage threshold validation      | Yes           | Yes          | No                | No            |
+| Coverage threshold validation      | Yes           | Yes          | No                | Yes           |
 | Deterministic build support        | Yes           | No           | Yes               | No            |
 
 > [!TIP]

global.json

@@ -1,6 +1,6 @@
 {
   "sdk": {
-    "version": "10.0.302",
+    "version": "10.0.400",
     "rollForward": "latestFeature"
   },
   "test": {

src/coverlet.MTP/Collector/CollectorExtension.cs

@@ -4,9 +4,11 @@
 #if NETSTANDARD2_0
 using System.Diagnostics;
 #endif
+using System.Globalization;
 using System.Text;
 using Coverlet.Core;
 using Coverlet.Core.Abstractions;
+using Coverlet.Core.Enums;
 using Coverlet.Core.Helpers;
 using Coverlet.Core.Symbols;
 using Coverlet.MTP.CommandLine;
@@ -137,6 +139,9 @@ Task ITestHostProcessLifetimeHandler.BeforeTestHostProcessStartAsync(Cancellatio
       _configuration.SkipAutoProps = config.SkipAutoProps;
       _configuration.formats = config.GetOutputFormats();
       _configuration.FilePrefix = config.GetFilePrefix();
+      _configuration.Threshold = config.GetThreshold();
+      _configuration.ThresholdStat = config.GetThresholdStatistic();
+      _configuration.ThresholdType = config.GetThresholdTypes();
       _configuration.UseSourceLink = false;
 
       _logger.LogVerbose($"Test module path: {_testModulePath}");
@@ -487,6 +492,117 @@ await _outputDisplay.DisplayAsync(
       cancellation).ConfigureAwait(false);
   }
 
+  /// <summary>
+  /// Displays the configured coverage thresholds and their evaluation results.
+  /// </summary>
+  private async Task DisplayThresholdSummaryAsync(CoverageResult result, CancellationToken cancellation)
+  {
+    if (!_configuration.Threshold.HasValue)
+    {
+      return;
+    }
+
+    ThresholdStatistic thresholdStat = _configuration.ThresholdStat;
+    Dictionary<ThresholdTypeFlags, double> thresholdValues = BuildThresholdValues(
+      _configuration.ThresholdType,
+      _configuration.Threshold.Value);
+    ThresholdTypeFlags belowThreshold = result.GetThresholdTypesBelowThreshold(thresholdValues, thresholdStat);
+
+    var summary = new StringBuilder();
+    summary.AppendLine();
+    summary.AppendLine("  Coverage Threshold Results:");
+    foreach (KeyValuePair<ThresholdTypeFlags, double> thresholdValue in thresholdValues)
+    {
+      ThresholdTypeFlags type = thresholdValue.Key;
+      double threshold = thresholdValue.Value;
+      double coverage = GetThresholdCoverage(result, type, thresholdStat);
+      string comparison = (belowThreshold & type) == ThresholdTypeFlags.None ? ">=" : "<";
+      summary.AppendLine(
+        $"    {thresholdStat} - {type} ({GetThresholdScopeDescription(thresholdStat)}): " +
+        $"{coverage.ToString("F1", CultureInfo.InvariantCulture)}% {comparison} " +
+        $"{threshold.ToString("F1", CultureInfo.InvariantCulture)}% threshold");
+    }
+
+    if (belowThreshold != ThresholdTypeFlags.None)
+    {
+      summary.AppendLine();
+      foreach (KeyValuePair<ThresholdTypeFlags, double> thresholdValue in thresholdValues.Where(pair => (belowThreshold & pair.Key) != ThresholdTypeFlags.None))
+      {
+        ThresholdTypeFlags type = thresholdValue.Key;
+        double threshold = thresholdValue.Value;
+        summary.AppendLine(
+          $"    The {thresholdStat.ToString().ToLowerInvariant()} {type.ToString().ToLowerInvariant()} coverage is below the specified " +
+          $"{threshold.ToString("F1", CultureInfo.InvariantCulture)}% threshold.");
+      }
+    }
+
+    await _outputDisplay.DisplayAsync(
+      this,
+      new TextOutputDeviceData(summary.ToString()),
+      cancellation).ConfigureAwait(false);
+  }
+
+  private static Dictionary<ThresholdTypeFlags, double> BuildThresholdValues(IEnumerable<string> thresholdTypes, double threshold)
+  {
+    var values = new Dictionary<ThresholdTypeFlags, double>();
+    foreach (string thresholdType in thresholdTypes)
+    {
+      values[ParseThresholdType(thresholdType)] = threshold;
+    }
+
+    return values;
+  }
+
+  private static ThresholdTypeFlags ParseThresholdType(string thresholdType) =>
+    thresholdType.Trim().ToLowerInvariant() switch
+    {
+      "line" => ThresholdTypeFlags.Line,
+      "branch" => ThresholdTypeFlags.Branch,
+      "method" => ThresholdTypeFlags.Method,
+      _ => throw new InvalidOperationException($"Invalid threshold type '{thresholdType}'. Valid values are line, branch, and method.")
+    };
+
+  private static double GetThresholdCoverage(CoverageResult result, ThresholdTypeFlags thresholdType, ThresholdStatistic thresholdStat)
+  {
+    if (thresholdStat == ThresholdStatistic.Minimum)
+    {
+      return result.Modules.Values
+        .Select(module => GetCoveragePercent(module, thresholdType))
+        .DefaultIfEmpty(0)
+        .Min();
+    }
+
+    CoverageDetails coverage = thresholdType switch
+    {
+      ThresholdTypeFlags.Line => CoverageSummary.CalculateLineCoverage(result.Modules),
+      ThresholdTypeFlags.Branch => CoverageSummary.CalculateBranchCoverage(result.Modules),
+      ThresholdTypeFlags.Method => CoverageSummary.CalculateMethodCoverage(result.Modules),
+      _ => throw new ArgumentOutOfRangeException(nameof(thresholdType), thresholdType, "A single coverage threshold type is required.")
+    };
+
+    return thresholdStat == ThresholdStatistic.Average
+      ? coverage.AverageModulePercent
+      : coverage.Percent;
+  }
+
+  private static double GetCoveragePercent(Documents module, ThresholdTypeFlags thresholdType) =>
+    thresholdType switch
+    {
+      ThresholdTypeFlags.Line => CoverageSummary.CalculateLineCoverage(module).Percent,
+      ThresholdTypeFlags.Branch => CoverageSummary.CalculateBranchCoverage(module).Percent,
+      ThresholdTypeFlags.Method => CoverageSummary.CalculateMethodCoverage(module).Percent,
+      _ => throw new ArgumentOutOfRangeException(nameof(thresholdType), thresholdType, "A single coverage threshold type is required.")
+    };
+
+  private static string GetThresholdScopeDescription(ThresholdStatistic thresholdStat) =>
+    thresholdStat switch
+    {
+      ThresholdStatistic.Total => "Total over Module",
+      ThresholdStatistic.Average => "Average per Module",
+      ThresholdStatistic.Minimum => "Minimum per Module",
+      _ => throw new ArgumentOutOfRangeException(nameof(thresholdStat), thresholdStat, null)
+    };
+
   /// <summary>
   /// Displays generated report paths to output device.
   /// </summary>
@@ -546,13 +662,30 @@ private async Task GenerateReportsAsync(CoverageResult result, CancellationToken
     // Display results
     await DisplayGeneratedReportsAsync(generatedReports, cancellation);
 
-    // Display coverage summary table after the file artifacts list
+    // Display code coverage summary table after the file artifacts list
     await DisplayCoverageSummaryAsync(result, cancellation);
 
     // Display console-type report output (e.g. teamcity) directly to the output device
     await DisplayConsoleReportOutputsAsync(consoleOutputs, cancellation);
-  }
 
+    // Display threshold metric summary table after the file artifacts list
+    await DisplayThresholdSummaryAsync(result, cancellation);
+
+    // Exitcode `CoverageThresholdFailed = 14` shall be set if any threshold is not met
+    if (_configuration.Threshold.HasValue)
+    {
+      ThresholdStatistic thresholdStat = _configuration.ThresholdStat;
+      Dictionary<ThresholdTypeFlags, double> thresholdValues = BuildThresholdValues(
+        _configuration.ThresholdType,
+        _configuration.Threshold.Value);
+      ThresholdTypeFlags belowThreshold = result.GetThresholdTypesBelowThreshold(thresholdValues, thresholdStat);
+      if (belowThreshold != ThresholdTypeFlags.None)
+      {
+        _logger.LogError("Coverage thresholds not met. Setting exit code for Microsoft Testing Framework to 14.");
+        Environment.ExitCode = 14;
+      }
+    }
+  }
   private string GetHitsFilePath()
   {
     // The hits file is in the same directory as the instrumented module

src/coverlet.MTP/CommandLine/CoverletCommandLineOptionDefinitions.cs

@@ -30,6 +30,9 @@ public static IReadOnlyCollection<CommandLineOption> GetAllOptions()
       new CommandLineOption(CoverletOptionNames.SkipAutoProps, "Skip auto-implemented properties.", ArgumentArity.Zero, isHidden: false),
       new CommandLineOption(CoverletOptionNames.DoesNotReturnAttribute, "Attributes that mark methods as not returning.", ArgumentArity.ZeroOrMore, isHidden: false),
       new CommandLineOption(CoverletOptionNames.ExcludeAssembliesWithoutSources, "Exclude assemblies without source code.", ArgumentArity.ZeroOrOne, isHidden: false),
+      new CommandLineOption(CoverletOptionNames.Threshold, "Coverage threshold percentage.", ArgumentArity.ExactlyOne, isHidden: false),
+      new CommandLineOption(CoverletOptionNames.ThresholdType, "Type of coverage threshold (line, branch, method).", ArgumentArity.OneOrMore, isHidden: false),
+      new CommandLineOption(CoverletOptionNames.ThresholdStat, "Statistic for coverage threshold (total, average, minimum).", ArgumentArity.ExactlyOne, isHidden: false),
     ];
   }
 }

src/coverlet.MTP/CommandLine/CoverletExtensionCommandLineProvider.cs

@@ -25,6 +25,26 @@ public CoverletExtensionCommandLineProvider(IExtension extension)
   public IReadOnlyCollection<CommandLineOption> GetCommandLineOptions()
     => CoverletCommandLineOptionDefinitions.GetAllOptions();
 
+#if NETSTANDARD2_0
+  private static bool TryParseEnum(Type enumType, string value, bool ignoreCase, out object? result)
+  {
+    try
+    {
+      result = Enum.Parse(enumType, value, ignoreCase);
+      return Enum.IsDefined(enumType, result);
+    }
+    catch (ArgumentException)
+    {
+      result = null;
+      return false;
+    }
+  }
+#else
+  private static bool TryParseEnum(Type enumType, string value, bool ignoreCase, out object? result)
+  {
+    return Enum.TryParse(enumType, value, ignoreCase, out result) && Enum.IsDefined(enumType, result);
+  }
+#endif
   public Task<ValidationResult> ValidateOptionArgumentsAsync(CommandLineOption commandOption, string[] arguments)
   {
     if (commandOption.Name == CoverletOptionNames.Formats)
@@ -72,9 +92,57 @@ public Task<ValidationResult> ValidateOptionArgumentsAsync(CommandLineOption com
         return Task.FromResult(ValidationResult.Invalid($"The value '{arguments[0]}' is not a valid option for '{commandOption.Name}'."));
       }
     }
+
+    if (commandOption.Name == CoverletOptionNames.Threshold)
+    {
+      if (arguments.Length == 0)
+      {
+        return Task.FromResult(ValidationResult.Invalid($"At least one value must be specified for '{commandOption.Name}'."));
+      }
+      if (arguments.Length > 1)
+      {
+        return Task.FromResult(ValidationResult.Invalid($"Only one value is allowed for '{commandOption.Name}'."));
+      }
+      if (!int.TryParse(arguments[0], out int thresholdValue) || thresholdValue < 0 || thresholdValue > 100)
+      {
+        return Task.FromResult(ValidationResult.Invalid($"The value '{arguments[0]}' is not a valid option for '{commandOption.Name}'. It must be an integer between 0 and 100."));
+      }
+    }
+
+    // Validate ThresholdType option to ensure it has a valid value.
+    if (commandOption.Name == CoverletOptionNames.ThresholdType)
+    {
+      if (arguments.Length == 0 || arguments.SelectMany(value => value.Split(',')).Any(value => !IsThresholdType(value)))
+      {
+        return Task.FromResult(ValidationResult.Invalid($"The value for '{commandOption.Name}' must be line, branch, or method."));
+      }
+    }
+
+    // Validate the ThresholdStat option to ensure it has a valid value
+    if (commandOption.Name == CoverletOptionNames.ThresholdStat)
+    {
+      if (arguments.Length == 0)
+      {
+        return Task.FromResult(ValidationResult.Invalid($"At least one value must be specified for '{commandOption.Name}'."));
+      }
+      if (arguments.Length > 1)
+      {
+        return Task.FromResult(ValidationResult.Invalid($"Only one value is allowed for '{commandOption.Name}'."));
+      }
+
+      if (!TryParseEnum(typeof(Coverlet.Core.Enums.ThresholdStatistic), arguments[0], ignoreCase: true, out object? thresholdStatistic))
+      {
+        return Task.FromResult(ValidationResult.Invalid($"The value '{arguments[0]}' is not a valid option for '{commandOption.Name}'(total, average, minimum)."));
+      }
+    }
     return ValidationResult.ValidTask;
   }
 
+  private static bool IsThresholdType(string value) =>
+    value.Trim().Equals("line", StringComparison.OrdinalIgnoreCase) ||
+    value.Trim().Equals("branch", StringComparison.OrdinalIgnoreCase) ||
+    value.Trim().Equals("method", StringComparison.OrdinalIgnoreCase);
+
   /// <summary>
   /// Validates that the file prefix is a safe filename segment without path traversal risks.
   /// </summary>
@@ -116,4 +184,3 @@ public Task<ValidationResult> ValidateCommandLineOptionsAsync(Microsoft.Testing.
     return ValidationResult.ValidTask;
   }
 }
-

src/coverlet.MTP/CommandLine/CoverletOptionNames.cs

@@ -18,4 +18,7 @@ internal static class CoverletOptionNames
   public const string SkipAutoProps = "coverlet-skip-auto-props";
   public const string DoesNotReturnAttribute = "coverlet-does-not-return-attribute";
   public const string ExcludeAssembliesWithoutSources = "coverlet-exclude-assemblies-without-sources";
+  public const string Threshold = "coverlet-threshold";
+  public const string ThresholdType = "coverlet-threshold-type";
+  public const string ThresholdStat = "coverlet-threshold-stat";
 }