Call benchmark method directly

src/BenchmarkDotNet/Code/CodeGenerator.cs

@@ -45,7 +45,6 @@ internal static string Generate(BuildPartition buildPartition)
                     .Replace("$ID$", buildInfo.Id.ToString())
                     .Replace("$OperationsPerInvoke$", provider.OperationsPerInvoke)
                     .Replace("$WorkloadTypeName$", provider.WorkloadTypeName)
-                    .Replace("$WorkloadMethodDelegate$", provider.WorkloadMethodDelegate(passArguments))
                     .Replace("$WorkloadMethodReturnType$", provider.WorkloadMethodReturnTypeName)
                     .Replace("$WorkloadMethodReturnTypeModifiers$", provider.WorkloadMethodReturnTypeModifiers)
                     .Replace("$OverheadMethodReturnTypeName$", provider.OverheadMethodReturnTypeName)

src/BenchmarkDotNet/Code/DeclarationsProvider.cs

@@ -36,8 +36,6 @@ internal abstract class DeclarationsProvider
 
         public virtual string WorkloadMethodReturnTypeName => WorkloadMethodReturnType.GetCorrectCSharpTypeName();
 
-        public virtual string WorkloadMethodDelegate(string passArguments) => Descriptor.WorkloadMethod.Name;
-
         public virtual string WorkloadMethodReturnTypeModifiers => null;
 
         public virtual string GetWorkloadMethodCall(string passArguments) => $"{Descriptor.WorkloadMethod.Name}({passArguments})";
@@ -149,9 +147,6 @@ internal class TaskDeclarationsProvider : VoidDeclarationsProvider
     {
         public TaskDeclarationsProvider(Descriptor descriptor) : base(descriptor) { }
 
-        public override string WorkloadMethodDelegate(string passArguments)
-            => $"({passArguments}) => {{ BenchmarkDotNet.Helpers.AwaitHelper.GetResult({Descriptor.WorkloadMethod.Name}({passArguments})); }}";
-
         public override string GetWorkloadMethodCall(string passArguments) => $"BenchmarkDotNet.Helpers.AwaitHelper.GetResult({Descriptor.WorkloadMethod.Name}({passArguments}))";
 
         protected override Type WorkloadMethodReturnType => typeof(void);
@@ -166,9 +161,6 @@ public GenericTaskDeclarationsProvider(Descriptor descriptor) : base(descriptor)
 
         protected override Type WorkloadMethodReturnType => Descriptor.WorkloadMethod.ReturnType.GetTypeInfo().GetGenericArguments().Single();
 
-        public override string WorkloadMethodDelegate(string passArguments)
-            => $"({passArguments}) => {{ return BenchmarkDotNet.Helpers.AwaitHelper.GetResult({Descriptor.WorkloadMethod.Name}({passArguments})); }}";
-
         public override string GetWorkloadMethodCall(string passArguments) => $"BenchmarkDotNet.Helpers.AwaitHelper.GetResult({Descriptor.WorkloadMethod.Name}({passArguments}))";
     }
 }

src/BenchmarkDotNet/Helpers/Reflection.Emit/IlGeneratorEmitOpExtensions.cs

@@ -116,6 +116,22 @@ public static void EmitLdarg(this ILGenerator ilBuilder, ParameterInfo argument)
             }
         }
 
+        public static void EmitStarg(this ILGenerator ilBuilder, ParameterInfo argument)
+        {
+            var position = argument.Position;
+            if (!((MethodBase) argument.Member).IsStatic)
+                position++;
+
+            if (position < 255)
+            {
+                ilBuilder.Emit(OpCodes.Starg_S, (byte) position);
+            }
+            else
+            {
+                ilBuilder.Emit(OpCodes.Starg, checked((short) position));
+            }
+        }
+
         public static void EmitLdindStind(this ILGenerator ilBuilder, Type resultType)
         {
             if (!resultType.IsByRef)

src/BenchmarkDotNet/Helpers/Reflection.Emit/IlGeneratorStatementExtensions.cs

@@ -84,64 +84,55 @@ public static void EmitSetDelegateToThisField(
             }
         }
 
-        public static void EmitLoopBeginFromLocToArg(
+        public static void EmitLoopBeginFromArgToZero(
             this ILGenerator ilBuilder,
             Label loopStartLabel,
-            Label loopHeadLabel,
-            LocalBuilder indexLocal,
-            ParameterInfo toArg)
+            Label loopHeadLabel)
         {
-            // loop counter stored as loc0, loop max passed as arg1
+            // invokeCount passed as arg
             /*
-                // for (long i = 0L; i < invokeCount; i++)
-                IL_0000: ldc.i4.0
-                IL_0001: conv.i8
-                IL_0002: stloc.0
+                // while (--invokeCount >= 0)
             */
-            ilBuilder.Emit(OpCodes.Ldc_I4_0);
-            ilBuilder.Emit(OpCodes.Conv_I8);
-            ilBuilder.EmitStloc(indexLocal);
 
-            // IL_0003: br.s IL_0036 // loop head: IL_0036 // we use long jump
+            // IL_0000: br.s IL_000e // loop head: IL_000e // we use long jump
             ilBuilder.Emit(OpCodes.Br, loopHeadLabel);
 
-            // loop start (head: IL_0036)
+            // loop start (head: IL_000e)
             ilBuilder.MarkLabel(loopStartLabel);
         }
 
-        public static void EmitLoopEndFromLocToArg(
+        public static void EmitLoopEndFromArgToZero(
             this ILGenerator ilBuilder,
             Label loopStartLabel,
             Label loopHeadLabel,
-            LocalBuilder indexLocal,
-            ParameterInfo toArg)
+            ParameterInfo arg)
         {
-            // loop counter stored as loc0, loop max passed as arg1
+            // invokeCount passed as arg
             /*
-                // for (long i = 0L; i < invokeCount; i++)
-                IL_0031: ldloc.0
-                IL_0032: ldc.i4.1
-                IL_0033: conv.i8
-                IL_0034: add
-                IL_0035: stloc.0
-             */
-            ilBuilder.EmitLdloc(indexLocal);
-            ilBuilder.Emit(OpCodes.Ldc_I4_1);
-            ilBuilder.Emit(OpCodes.Conv_I8);
-            ilBuilder.Emit(OpCodes.Add);
-            ilBuilder.EmitStloc(indexLocal);
-
-            /*
-                // for (long i = 0L; i < invokeCount; i++)
-                IL_0036: ldloc.0 // loop head: IL_0036
-                IL_0037: ldarg.1
-                IL_0038: blt.s IL_0005 // we use long jump
+                // while (--invokeCount >= 0)
+                IL_0008: ldarg.1
+                IL_0009: ldc.i4.1
+                IL_000a: conv.i8
+                IL_000b: sub
+                IL_000c: dup
+                IL_000d: starg.s invokeCount
+                IL_000f: ldc.i4.0
+                IL_0010: conv.i8
+                IL_0011: bge.s IL_0002
                 // end loop
              */
+
+            // loop head
             ilBuilder.MarkLabel(loopHeadLabel);
-            ilBuilder.EmitLdloc(indexLocal);
-            ilBuilder.EmitLdarg(toArg);
-            ilBuilder.Emit(OpCodes.Blt, loopStartLabel);
+            ilBuilder.EmitLdarg(arg);
+            ilBuilder.Emit(OpCodes.Ldc_I4_1);
+            ilBuilder.Emit(OpCodes.Conv_I8);
+            ilBuilder.Emit(OpCodes.Sub);
+            ilBuilder.Emit(OpCodes.Dup);
+            ilBuilder.EmitStarg(arg);
+            ilBuilder.Emit(OpCodes.Ldc_I4_0);
+            ilBuilder.Emit(OpCodes.Conv_I8);
+            ilBuilder.Emit(OpCodes.Bge, loopStartLabel);
         }
     }
 }

src/BenchmarkDotNet/Helpers/Reflection.Emit/MethodBuilderExtensions.cs

@@ -38,13 +38,5 @@ public static MethodBuilder SetNoOptimizationImplementationFlag(this MethodBuild
 
             return methodBuilder;
         }
-
-        public static MethodBuilder SetAggressiveOptimizationImplementationFlag(this MethodBuilder methodBuilder)
-        {
-            methodBuilder.SetImplementationFlags(
-                methodBuilder.GetMethodImplementationFlags() | CodeGenHelper.AggressiveOptimizationOptionForEmit);
-
-            return methodBuilder;
-        }
     }
 }

src/BenchmarkDotNet/Portability/CodeGen.cs

@@ -7,6 +7,5 @@ public static class CodeGenHelper
     {
         // AggressiveOptimization is not available in netstandard2.0, so just use the value casted to enum.
         public const MethodImplOptions AggressiveOptimizationOption = (MethodImplOptions) 512;
-        public const MethodImplAttributes AggressiveOptimizationOptionForEmit = (MethodImplAttributes) 512;
     }
 }

src/BenchmarkDotNet/Toolchains/InProcess/Emit/Implementation/Runnable/RunnableConstants.cs

@@ -20,8 +20,6 @@ public class RunnableConstants
         public const string GlobalCleanupActionFieldName = "globalCleanupAction";
         public const string IterationSetupActionFieldName = "iterationSetupAction";
         public const string IterationCleanupActionFieldName = "iterationCleanupAction";
-        public const string WorkloadDelegateFieldName = "workloadDelegate";
-        public const string OverheadDelegateFieldName = "overheadDelegate";
         public const string NotElevenFieldName = "NotEleven";
         public const string DummyVarFieldName = "dummyVar";
 

tests/BenchmarkDotNet.IntegrationTests.ManualRunning/ExpectedBenchmarkResultsTests.cs

@@ -13,25 +13,26 @@
 using BenchmarkDotNet.Reports;
 using BenchmarkDotNet.Tests.XUnit;
 using BenchmarkDotNet.Toolchains.InProcess.Emit;
+using Perfolizer;
 using Perfolizer.Horology;
+using Perfolizer.Mathematics.Common;
+using Perfolizer.Mathematics.SignificanceTesting;
+using Perfolizer.Mathematics.SignificanceTesting.MannWhitney;
 using Perfolizer.Metrology;
 using Xunit;
 using Xunit.Abstractions;
 
 namespace BenchmarkDotNet.IntegrationTests.ManualRunning
 {
-    public class ExpectedBenchmarkResultsTests : BenchmarkTestExecutor
+    public class ExpectedBenchmarkResultsTests(ITestOutputHelper output) : BenchmarkTestExecutor(output)
     {
         // NativeAot takes a long time to build, so not including it in these tests.
         // We also don't test InProcessNoEmitToolchain because it is known to be less accurate than code-gen toolchains.
 
         private static readonly TimeInterval FallbackCpuResolutionValue = TimeInterval.FromNanoseconds(0.2d);
 
-        public ExpectedBenchmarkResultsTests(ITestOutputHelper output) : base(output) { }
-
         private static IEnumerable<Type> EmptyBenchmarkTypes() =>
-            new[]
-            {
+            [
                 typeof(EmptyVoid),
                 typeof(EmptyByte),
                 typeof(EmptySByte),
@@ -46,7 +47,7 @@ private static IEnumerable<Type> EmptyBenchmarkTypes() =>
                 typeof(EmptyUIntPtr),
                 typeof(EmptyVoidPointer),
                 typeof(EmptyClass)
-            };
+            ];
 
         public static IEnumerable<object[]> InProcessData()
         {
@@ -60,8 +61,8 @@ public static IEnumerable<object[]> CoreData()
         {
             foreach (var type in EmptyBenchmarkTypes())
             {
-                yield return new object[] { type, RuntimeMoniker.Net70 };
-                yield return new object[] { type, RuntimeMoniker.Mono70 };
+                yield return new object[] { type, RuntimeMoniker.Net80 };
+                yield return new object[] { type, RuntimeMoniker.Mono80 };
             }
         }
 
@@ -76,17 +77,19 @@ public static IEnumerable<object[]> FrameworkData()
 
         [Theory]
         [MemberData(nameof(InProcessData))]
-        public void EmptyBenchmarksReportZeroTimeAndAllocated_InProcess(Type benchmarkType)
+        public void EmptyBenchmarkReportsZeroTimeAndAllocated_InProcess(Type benchmarkType)
         {
             AssertZeroResults(benchmarkType, ManualConfig.CreateEmpty()
                 .AddJob(Job.Default
                     .WithToolchain(InProcessEmitToolchain.Instance)
-                ));
+                // IL Emit has incorrect overhead measurement. https://github.com/dotnet/runtime/issues/89685
+                // We multiply the threshold to account for it.
+                ), multiplyThresholdBy: RuntimeInformation.IsNetCore ? 3 : 1);
         }
 
         [TheoryEnvSpecific("To not repeat tests in both Full .NET Framework and Core", EnvRequirement.DotNetCoreOnly)]
         [MemberData(nameof(CoreData))]
-        public void EmptyBenchmarksReportZeroTimeAndAllocated_Core(Type benchmarkType, RuntimeMoniker runtimeMoniker)
+        public void EmptyBenchmarkReportsZeroTimeAndAllocated_Core(Type benchmarkType, RuntimeMoniker runtimeMoniker)
         {
             AssertZeroResults(benchmarkType, ManualConfig.CreateEmpty()
                 .AddJob(Job.Default
@@ -96,23 +99,23 @@ public void EmptyBenchmarksReportZeroTimeAndAllocated_Core(Type benchmarkType, R
 
         [TheoryEnvSpecific("Can only run Full .NET Framework and Mono tests from Framework host", EnvRequirement.FullFrameworkOnly)]
         [MemberData(nameof(FrameworkData))]
-        public void EmptyBenchmarksReportZeroTimeAndAllocated_Framework(Type benchmarkType, RuntimeMoniker runtimeMoniker)
+        public void EmptyBenchmarkReportsZeroTimeAndAllocated_Framework(Type benchmarkType, RuntimeMoniker runtimeMoniker)
         {
             AssertZeroResults(benchmarkType, ManualConfig.CreateEmpty()
                 .AddJob(Job.Default
                     .WithRuntime(runtimeMoniker.GetRuntime())
                 ));
         }
 
-        private void AssertZeroResults(Type benchmarkType, IConfig config)
+        private void AssertZeroResults(Type benchmarkType, IConfig config, int multiplyThresholdBy = 1)
         {
             var summary = CanExecute(benchmarkType, config
                 .WithSummaryStyle(SummaryStyle.Default.WithTimeUnit(TimeUnit.Nanosecond))
                 .AddDiagnoser(new MemoryDiagnoser(new MemoryDiagnoserConfig(false)))
             );
 
             var cpuResolution = CpuDetector.Cpu?.MaxFrequency()?.ToResolution() ?? FallbackCpuResolutionValue;
-            var threshold = new NumberValue(cpuResolution.Nanoseconds).ToThreshold();
+            var threshold = new NumberValue(cpuResolution.Nanoseconds * multiplyThresholdBy).ToThreshold();
 
             foreach (var report in summary.Reports)
             {
@@ -126,87 +129,137 @@ private void AssertZeroResults(Type benchmarkType, IConfig config)
             }
         }
 
-        [Fact]
-        public void DifferentSizedStructsBenchmarksReportsNonZeroTimeAndZeroAllocated_InProcess()
+        private static IEnumerable<Type> NonEmptyBenchmarkTypes() =>
+            [
+                typeof(DifferentSizedStructs),
+                typeof(ActualWork)
+            ];
+
+        public static IEnumerable<object[]> NonEmptyInProcessData()
+        {
+            foreach (var type in NonEmptyBenchmarkTypes())
+            {
+                yield return new object[] { type };
+            }
+        }
+
+        public static IEnumerable<object[]> NonEmptyCoreData()
+        {
+            foreach (var type in NonEmptyBenchmarkTypes())
+            {
+                yield return new object[] { type, RuntimeMoniker.Net80 };
+                yield return new object[] { type, RuntimeMoniker.Mono80 };
+            }
+        }
+
+        public static IEnumerable<object[]> NonEmptyFrameworkData()
         {
-            AssertDifferentSizedStructsResults(ManualConfig.CreateEmpty()
+            foreach (var type in NonEmptyBenchmarkTypes())
+            {
+                yield return new object[] { type, RuntimeMoniker.Net462 };
+                yield return new object[] { type, RuntimeMoniker.Mono };
+            }
+        }
+
+        [Theory]
+        [MemberData(nameof(NonEmptyInProcessData))]
+        public void NonEmptyBenchmarkReportsNonZeroTimeAndZeroAllocated_InProcess(Type benchmarkType)
+        {
+            AssertNonZeroResults(benchmarkType, ManualConfig.CreateEmpty()
                 .AddJob(Job.Default
                     .WithToolchain(InProcessEmitToolchain.Instance)
-                ));
+                // InProcess overhead measurements are incorrect, so we adjust the results to account for it. https://github.com/dotnet/runtime/issues/89685
+                ), subtractOverheadByClocks: RuntimeInformation.IsNetCore ? 3 : 1);
         }
 
         [TheoryEnvSpecific("To not repeat tests in both Full .NET Framework and Core", EnvRequirement.DotNetCoreOnly)]
-        [InlineData(RuntimeMoniker.Net70)]
-        [InlineData(RuntimeMoniker.Mono70)]
-        public void DifferentSizedStructsBenchmarksReportsNonZeroTimeAndZeroAllocated_Core(RuntimeMoniker runtimeMoniker)
+        [MemberData(nameof(NonEmptyCoreData))]
+        public void NonEmptyBenchmarkReportsNonZeroTimeAndZeroAllocated_Core(Type benchmarkType, RuntimeMoniker runtimeMoniker)
         {
-            AssertDifferentSizedStructsResults(ManualConfig.CreateEmpty()
+            AssertNonZeroResults(benchmarkType, ManualConfig.CreateEmpty()
                 .AddJob(Job.Default
                     .WithRuntime(runtimeMoniker.GetRuntime())
                 ));
         }
 
-        [TheoryEnvSpecific("Can only run Full .NET Framework and Mono tests from Framework host", EnvRequirement.FullFrameworkOnly)]
-        [InlineData(RuntimeMoniker.Net462)]
-        [InlineData(RuntimeMoniker.Mono)]
-        public void DifferentSizedStructsBenchmarksReportsNonZeroTimeAndZeroAllocated_Framework(RuntimeMoniker runtimeMoniker)
+        [TheoryEnvSpecific("Can only run Mono tests from Framework host", EnvRequirement.FullFrameworkOnly)]
+        [MemberData(nameof(NonEmptyFrameworkData))]
+        public void NonEmptyBenchmarkReportsNonZeroTimeAndZeroAllocated_Framework(Type benchmarkType, RuntimeMoniker runtimeMoniker)
         {
-            AssertDifferentSizedStructsResults(ManualConfig.CreateEmpty()
+            AssertNonZeroResults(benchmarkType, ManualConfig.CreateEmpty()
                 .AddJob(Job.Default
                     .WithRuntime(runtimeMoniker.GetRuntime())
                 ));
         }
 
-        private void AssertDifferentSizedStructsResults(IConfig config)
+        private void AssertNonZeroResults(Type benchmarkType, IConfig config, int subtractOverheadByClocks = 0)
         {
-            var summary = CanExecute<DifferentSizedStructs>(config
+            var summary = CanExecute(benchmarkType, config
                 .WithSummaryStyle(SummaryStyle.Default.WithTimeUnit(TimeUnit.Nanosecond))
                 .AddDiagnoser(new MemoryDiagnoser(new MemoryDiagnoserConfig(false)))
             );
 
             var cpuResolution = CpuDetector.Cpu?.MaxFrequency()?.ToResolution() ?? FallbackCpuResolutionValue;
-            var threshold = (cpuResolution / 2).ToThreshold();
+            // Modern cpus can execute multiple instructions per clock cycle,
+            // resulting in measurements greater than 0 but less than 1 clock cycle.
+            // (example: Intel Core i9-9880H CPU 2.30GHz reports 0.2852 ns for `_field++;`)
+            var threshold = new NumberValue(cpuResolution.Nanoseconds / 4).ToThreshold();
+            // InProcess overhead measurements are incorrect, so we adjust the results to account for it. https://github.com/dotnet/runtime/issues/89685
+            var overheadSubtraction = cpuResolution.Nanoseconds * subtractOverheadByClocks;
 
             foreach (var report in summary.Reports)
             {
                 var workloadMeasurements = report.AllMeasurements.Where(m => m.Is(IterationMode.Workload, IterationStage.Actual)).GetStatistics().Sample;
-                var overheadMeasurements = report.AllMeasurements.Where(m => m.Is(IterationMode.Overhead, IterationStage.Actual)).GetStatistics().Sample;
+                var overheadMeasurements = new Sample(report.AllMeasurements
+                    .Where(m => m.Is(IterationMode.Overhead, IterationStage.Actual))
+                    .GetStatistics().OriginalValues
+                    .Select(x => x - overheadSubtraction).ToArray());
 
-                bool isZero = ZeroMeasurementHelper.AreIndistinguishable(workloadMeasurements, overheadMeasurements, threshold);
-                Assert.False(isZero, $"Actual time was 0.");
+                var comparisonResult = new SimpleEquivalenceTest(MannWhitneyTest.Instance).Perform(workloadMeasurements, overheadMeasurements, threshold, SignificanceLevel.P1E5);
+                Assert.True(comparisonResult == ComparisonResult.Greater, "Workload measurements are not greater than overhead.");
 
                 Assert.True((report.GcStats.GetBytesAllocatedPerOperation(report.BenchmarkCase) ?? 0L) == 0L, "Memory allocations measured above 0.");
             }
         }
     }
+}
 
-    public struct Struct16
-    {
-        public long l1, l2;
-    }
+// Types outside of namespace so it's easier to read in the test explorer.
+#pragma warning disable CA1050 // Declare types in namespaces
+public struct Struct16
+{
+    public long l1, l2;
+}
 
-    public struct Struct32
-    {
-        public long l1, l2, l3, l4;
-    }
+public struct Struct32
+{
+    public long l1, l2, l3, l4;
+}
 
-    public struct Struct64
-    {
-        public long l1, l2, l3, l4, l5, l6, l7, l8;
-    }
+public struct Struct64
+{
+    public long l1, l2, l3, l4, l5, l6, l7, l8;
+}
 
-    public struct Struct128
-    {
-        public long l1, l2, l3, l4, l5, l6, l7, l8, l9, l10, l11, l12, l13, l14, l15, l16;
-    }
+public struct Struct128
+{
+    public long l1, l2, l3, l4, l5, l6, l7, l8, l9, l10, l11, l12, l13, l14, l15, l16;
+}
 
-    public class DifferentSizedStructs
-    {
-        [Benchmark] public Struct16 Struct16() => default;
-        [Benchmark] public Struct32 Struct32() => default;
-        [Benchmark] public Struct64 Struct64() => default;
-        [Benchmark] public Struct128 Struct128() => default;
-    }
+public class DifferentSizedStructs
+{
+    [Benchmark] public Struct16 Struct16() => default;
+    [Benchmark] public Struct32 Struct32() => default;
+    [Benchmark] public Struct64 Struct64() => default;
+    [Benchmark] public Struct128 Struct128() => default;
+}
+
+public class ActualWork
+{
+    public int _field;
+
+    [Benchmark]
+    public void IncrementField() => _field++;
 }
 
 public class EmptyVoid

tests/BenchmarkDotNet.IntegrationTests/InProcess.EmitTests/NaiveRunnableEmitDiff.cs

@@ -31,7 +31,8 @@ public class NaiveRunnableEmitDiff
         {
             { OpCodes.Br_S, OpCodes.Br },
             { OpCodes.Blt_S, OpCodes.Blt },
-            { OpCodes.Bne_Un_S, OpCodes.Bne_Un }
+            { OpCodes.Bne_Un_S, OpCodes.Bne_Un },
+            { OpCodes.Bge_S, OpCodes.Bge }
         };
 
         public static void RunDiff(string roslynAssemblyPath, string emittedAssemblyPath, ILogger logger)