Improve Benchmark Accuracy

BenchmarkDotNet.sln

@@ -59,6 +59,8 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BenchmarkDotNet.Exporters.P
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BenchmarkDotNet.Exporters.Plotting.Tests", "tests\BenchmarkDotNet.Exporters.Plotting.Tests\BenchmarkDotNet.Exporters.Plotting.Tests.csproj", "{199AC83E-30BD-40CD-87CE-0C838AC0320D}"
 EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BenchmarkDotNet.Weaver", "src\BenchmarkDotNet.Weaver\BenchmarkDotNet.Weaver.csproj", "{5731DE42-16FE-430E-BA90-0EBE714CB221}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -161,6 +163,10 @@ Global
 		{199AC83E-30BD-40CD-87CE-0C838AC0320D}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{199AC83E-30BD-40CD-87CE-0C838AC0320D}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{199AC83E-30BD-40CD-87CE-0C838AC0320D}.Release|Any CPU.Build.0 = Release|Any CPU
+		{5731DE42-16FE-430E-BA90-0EBE714CB221}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{5731DE42-16FE-430E-BA90-0EBE714CB221}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{5731DE42-16FE-430E-BA90-0EBE714CB221}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{5731DE42-16FE-430E-BA90-0EBE714CB221}.Release|Any CPU.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
@@ -190,6 +196,7 @@ Global
 		{2E2283A3-6DA6-4482-8518-99D6D9F689AB} = {D6597E3A-6892-4A68-8E14-042FC941FDA2}
 		{B92ECCEF-7C27-4012-9E19-679F3C40A6A6} = {D6597E3A-6892-4A68-8E14-042FC941FDA2}
 		{199AC83E-30BD-40CD-87CE-0C838AC0320D} = {14195214-591A-45B7-851A-19D3BA2413F9}
+		{5731DE42-16FE-430E-BA90-0EBE714CB221} = {D6597E3A-6892-4A68-8E14-042FC941FDA2}
 	EndGlobalSection
 	GlobalSection(ExtensibilityGlobals) = postSolution
 		SolutionGuid = {4D9AF12B-1F7F-45A7-9E8C-E4E46ADCBD1F}

NuGet.Config

@@ -8,11 +8,13 @@
     <clear />
 
     <add key="api.nuget.org" value="https://api.nuget.org/v3/index.json" />
-    <!-- reuquired to run Mono AOT benchmarks -->
+    <!-- required to run Mono AOT benchmarks -->
     <add key="dotnet6" value="https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet6/nuget/v3/index.json" />
     <add key="dotnet7" value="https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet7/nuget/v3/index.json" />
     <add key="dotnet8" value="https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet8/nuget/v3/index.json" />
     <add key="dotnet9" value="https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet9/nuget/v3/index.json" />
     <add key="dotnet10" value="https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet10/nuget/v3/index.json" />
+
+    <add key="benchmarkdotnet.weaver" value="src/BenchmarkDotNet.Weaver/packages" />
   </packageSources>
 </configuration>

build/BenchmarkDotNet.Build/Program.cs

@@ -16,8 +16,29 @@ public static int Main(string[] args)
     }
 }
 
+[TaskName(Name)]
+[TaskDescription("Pack Weaver")]
+public class PackWeaverTask : FrostingTask<BuildContext>, IHelpProvider
+{
+    private const string Name = "pack-weaver";
+
+    public override void Run(BuildContext context) => context.BuildRunner.PackWeaver();
+
+    public HelpInfo GetHelp()
+    {
+        return new HelpInfo
+        {
+            Examples = new[]
+            {
+                new Example(Name)
+            }
+        };
+    }
+}
+
 [TaskName(Name)]
 [TaskDescription("Restore NuGet packages")]
+[IsDependentOn(typeof(PackWeaverTask))]
 public class RestoreTask : FrostingTask<BuildContext>, IHelpProvider
 {
     private const string Name = "restore";

build/BenchmarkDotNet.Build/Runners/BuildRunner.cs

@@ -8,6 +8,7 @@
 using Cake.Common.Tools.DotNet.Workload.Install;
 using Cake.Core;
 using Cake.Core.IO;
+using System.Linq;
 
 namespace BenchmarkDotNet.Build.Runners;
 
@@ -20,6 +21,34 @@ public BuildRunner(BuildContext context)
         this.context = context;
     }
 
+    public void PackWeaver()
+    {
+        var weaverPath = context.AllPackableSrcProjects.Single(p => p.GetFilename() == "BenchmarkDotNet.Weaver.csproj");
+
+        context.DotNetRestore(weaverPath.GetDirectory().FullPath,
+            new DotNetRestoreSettings
+            {
+                MSBuildSettings = context.MsBuildSettingsRestore
+            });
+
+        context.Information("BuildSystemProvider: " + context.BuildSystem().Provider);
+        context.DotNetBuild(weaverPath.FullPath, new DotNetBuildSettings
+        {
+            NoRestore = true,
+            DiagnosticOutput = true,
+            MSBuildSettings = context.MsBuildSettingsBuild,
+            Configuration = context.BuildConfiguration,
+            Verbosity = context.BuildVerbosity
+        });
+
+        context.DotNetPack(weaverPath.FullPath, new DotNetPackSettings
+        {
+            OutputDirectory = weaverPath.GetDirectory().Combine("packages"),
+            MSBuildSettings = context.MsBuildSettingsPack,
+            Configuration = context.BuildConfiguration
+        });
+    }
+
     public void Restore()
     {
         context.DotNetRestore(context.SolutionFile.FullPath,
@@ -71,7 +100,7 @@ public void Pack()
         var settingsSrc = new DotNetPackSettings
         {
             OutputDirectory = context.ArtifactsDirectory,
-            ArgumentCustomization = args => args.Append("--include-symbols").Append("-p:SymbolPackageFormat=snupkg"),
+            ArgumentCustomization = args => args.Append("--include-symbols").Append("-p:SymbolPackageFormat=snupkg").Append("-p:IsFullPack=true"),
             MSBuildSettings = context.MsBuildSettingsPack,
             Configuration = context.BuildConfiguration
         };

build/common.props

@@ -90,4 +90,9 @@
       </Code>
     </Task>
   </UsingTask>
+
+  <PropertyGroup>
+	<!-- Increment this when the BenchmarkDotNet.Weaver package needs to be re-packed. -->
+	<WeaverVersionSuffix>-1</WeaverVersionSuffix>
+  </PropertyGroup>
 </Project>

samples/BenchmarkDotNet.Samples.FSharp/BenchmarkDotNet.Samples.FSharp.fsproj

@@ -3,6 +3,7 @@
     <!-- MSBuild was complaing about InformationalVersion from common.props file, so I excluded them in conditional way -->
     <IsFsharp>true</IsFsharp>
   </PropertyGroup>
+  <Import Project="..\..\build\common.props" />
   <PropertyGroup>
     <OutputType>Exe</OutputType>
     <TargetFrameworks>net462;net8.0</TargetFrameworks>

samples/BenchmarkDotNet.Samples/IntroSmokeStringBuilder.cs

@@ -0,0 +1,34 @@
+using BenchmarkDotNet.Attributes;
+using System.Text;
+
+namespace BenchmarkDotNet.Samples
+{
+    [MemoryDiagnoser(false)]
+    public class IntroSmokeStringBuilder
+    {
+        [Benchmark]
+        [Arguments(1)]
+        [Arguments(1_000)]
+        public StringBuilder Append_Strings(int repeat)
+        {
+            StringBuilder builder = new StringBuilder();
+
+            // strings are not sorted by length to mimic real input
+            for (int i = 0; i < repeat; i++)
+            {
+                builder.Append("12345");
+                builder.Append("1234567890abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMN");
+                builder.Append("1234567890abcdefghijklmnopqrstuvwxy");
+                builder.Append("1234567890");
+                builder.Append("1234567890abcdefghijklmnopqrstuvwxyzABCDEFGHI");
+                builder.Append("1234567890abcde");
+                builder.Append("1234567890abcdefghijklmnopqrstuvwxyzABCD");
+                builder.Append("1234567890abcdefghijklmnopqrst");
+                builder.Append("1234567890abcdefghij");
+                builder.Append("1234567890abcdefghijklmno");
+            }
+
+            return builder;
+        }
+    }
+}

src/BenchmarkDotNet.Annotations/BenchmarkDotNet.Annotations.csproj

@@ -14,4 +14,19 @@
   <ItemGroup>
     <Compile Include="..\BenchmarkDotNet\Helpers\CodeAnnotations.cs" Link="Attributes\CodeAnnotations.cs" />
   </ItemGroup>
+
+  <Choose>
+	<When Condition="'$(IsFullPack)' == 'true'">
+	  <!-- ProjectReference to ensure the nuget package has a dependency on BenchmarkDotNet.Weaver using the proper version. -->
+	  <ItemGroup>
+		<ProjectReference Include="..\BenchmarkDotNet.Weaver\BenchmarkDotNet.Weaver.csproj" />
+	  </ItemGroup>
+	</When>
+	<Otherwise>
+	  <!-- PackageReference for transitive weaver dependency for ProjectReferences to this. -->
+	  <ItemGroup>
+		<PackageReference Include="BenchmarkDotNet.Weaver" Version="$(Version)$(WeaverVersionSuffix)" />
+	  </ItemGroup>
+	</Otherwise>
+  </Choose>
 </Project>

src/BenchmarkDotNet.Weaver/.gitignore

@@ -0,0 +1 @@
+!packages/

src/BenchmarkDotNet.Weaver/BenchmarkDotNet.Weaver.csproj

@@ -0,0 +1,27 @@
+<!-- When any changes are made to this project, increment the WeaverVersionSuffix in the common.props file,
+     then re-pack with output to \packages, and delete the old nupkg. -->
+
+<Project Sdk="Microsoft.NET.Sdk">
+  <Import Project="..\..\build\common.props" />
+  <PropertyGroup>
+	<TargetFrameworks>netstandard2.0</TargetFrameworks>
+	<VersionSuffix Condition="'$(IsFullPack)' != 'true'">$(VersionSuffix)$(WeaverVersionSuffix)</VersionSuffix>
+	<OutputPath>$(MSBuildThisFileDirectory)bin\$(Configuration)</OutputPath>
+	<CopyLocalLockFileAssemblies>true</CopyLocalLockFileAssemblies>
+	<IncludeBuildOutput>false</IncludeBuildOutput>
+	<NoWarn>$(NoWarn);NU5100;NU5128</NoWarn>
+  </PropertyGroup>
+
+  <ItemGroup>
+	<PackageReference Include="Microsoft.Build.Framework" Version="17.12.6" />
+	<PackageReference Include="Microsoft.Build.Utilities.Core" Version="17.12.6" />
+	<PackageReference Include="Mono.Cecil" Version="0.11.6" />
+  </ItemGroup>
+
+  <ItemGroup>
+	<!-- Include .targets file and all DLLs in the output directory in the NuGet package -->
+	<Content Include="$(MSBuildThisFileDirectory)buildTransitive\**\*.targets" Pack="true" PackagePath="buildTransitive" />
+	<Content Include="$(OutputPath)**\*.dll" Pack="true" PackagePath="tasks/$(TargetFramework)" />
+	<None Remove="packages\**" />
+  </ItemGroup>
+</Project>

src/BenchmarkDotNet.Weaver/buildTransitive/netstandard2.0/BenchmarkDotNet.Weaver.targets

@@ -0,0 +1,8 @@
+<?xml version="1.0" encoding="utf-8"?>
+<Project xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+  <UsingTask TaskName="BenchmarkDotNet.Weaver.WeaveAssemblyTask" AssemblyFile="$(MSBuildThisFileDirectory)..\..\tasks\netstandard2.0\BenchmarkDotNet.Weaver.dll" />
+
+  <Target Name="WeaveAssemblies" AfterTargets="AfterBuild" >
+	<WeaveAssemblyTask TargetDir="$(TargetDir)" TargetAssembly="$(TargetDir)$(TargetFileName)" />
+  </Target>
+</Project>

src/BenchmarkDotNet.Weaver/src/WeaveAssemblyTask.cs

@@ -0,0 +1,122 @@
+using System;
+using System.IO;
+using System.Linq;
+using Microsoft.Build.Framework;
+using Microsoft.Build.Utilities;
+using Mono.Cecil;
+
+namespace BenchmarkDotNet.Weaver;
+
+internal class CustomAssemblyResolver : DefaultAssemblyResolver
+{
+    public override AssemblyDefinition Resolve(AssemblyNameReference name, ReaderParameters parameters)
+        // NetStandard causes StackOverflow. https://github.com/jbevain/cecil/issues/573
+        // Mscorlib fails to resolve in Visual Studio. https://github.com/jbevain/cecil/issues/966
+        // We don't care about any types from runtime assemblies anyway, so just skip resolving them.
+        => name.Name is "netstandard" or "mscorlib" or "System.Runtime" or "System.Private.CoreLib"
+            ? null
+            : base.Resolve(name, parameters);
+}
+
+/// <summary>
+/// The Task used by MSBuild to weave the assembly.
+/// </summary>
+public sealed class WeaveAssemblyTask : Task
+{
+    /// <summary>
+    /// The directory of the output.
+    /// </summary>
+    [Required]
+    public string TargetDir { get; set; }
+
+    /// <summary>
+    /// The path of the target assembly.
+    /// </summary>
+    [Required]
+    public string TargetAssembly { get; set; }
+
+    /// <summary>
+    /// Runs the weave assembly task.
+    /// </summary>
+    /// <returns><see langword="true"/> if successful; <see langword="false"/> otherwise.</returns>
+    public override bool Execute()
+    {
+        if (!File.Exists(TargetAssembly))
+        {
+            Log.LogError($"Assembly not found: {TargetAssembly}");
+            return false;
+        }
+
+        var resolver = new CustomAssemblyResolver();
+        resolver.AddSearchDirectory(TargetDir);
+
+        // ReaderParameters { ReadWrite = true } is necessary to later write the file.
+        // https://stackoverflow.com/questions/41840455/locked-target-assembly-with-mono-cecil-and-pcl-code-injection
+        var readerParameters = new ReaderParameters
+        {
+            ReadWrite = true,
+            AssemblyResolver = resolver
+        };
+
+        bool benchmarkMethodsImplAdjusted = false;
+        try
+        {
+            using var module = ModuleDefinition.ReadModule(TargetAssembly, readerParameters);
+
+            foreach (var type in module.Types)
+            {
+                ProcessType(type, ref benchmarkMethodsImplAdjusted);
+            }
+
+            // Write the modified assembly to file.
+            module.Write();
+        }
+        catch (Exception e)
+        {
+            if (benchmarkMethodsImplAdjusted)
+            {
+                Log.LogWarning($"Benchmark methods were found that require NoInlining, and assembly weaving failed.{Environment.NewLine}{e}");
+            }
+        }
+        return true;
+    }
+
+    private static void ProcessType(TypeDefinition type, ref bool benchmarkMethodsImplAdjusted)
+    {
+        // We can skip non-public types as they are not valid for benchmarks.
+        if (type.IsNotPublic)
+        {
+            return;
+        }
+
+        // Remove AggressiveInlining and add NoInlining to all [Benchmark] methods.
+        foreach (var method in type.Methods)
+        {
+            if (method.CustomAttributes.Any(IsBenchmarkAttribute))
+            {
+                var oldImpl = method.ImplAttributes;
+                method.ImplAttributes = (oldImpl & ~MethodImplAttributes.AggressiveInlining) | MethodImplAttributes.NoInlining;
+                benchmarkMethodsImplAdjusted |= (oldImpl & MethodImplAttributes.NoInlining) == 0;
+            }
+        }
+
+        // Recursively process nested types
+        foreach (var nestedType in type.NestedTypes)
+        {
+            ProcessType(nestedType, ref benchmarkMethodsImplAdjusted);
+        }
+    }
+
+    private static bool IsBenchmarkAttribute(CustomAttribute attribute)
+    {
+        // BenchmarkAttribute is unsealed, so we need to walk its hierarchy.
+        for (var attr = attribute.AttributeType; attr != null; attr = attr.Resolve()?.BaseType)
+        {
+            if (attr.FullName == "BenchmarkDotNet.Attributes.BenchmarkAttribute")
+            {
+                return true;
+            }
+        }
+        return false;
+    }
+}