Skip to content

Commit e055116

Browse files
JosephTremouletJosephTremoulet
committed
Modify benchmarks to run in perf test harness
 - Add result validation - Add [Benchmark] attributes and appropriate iteration counts - Minor edits here and there to target .NET Standard 1.4 - Exception: pi-digits rewritten to use managed BitInteger instead of p/invoke out to GMP.
1 parent d0099ff commit e055116

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

42 files changed

+6015
-221
lines changed

‎.gitattributes

+2
Original file line numberDiff line numberDiff line change
@@ -69,6 +69,8 @@ src/pal/tests/palsuite/paltestlist_to_be_reviewed.txt text eol=lf
6969

7070
tests/src/JIT/Performance/CodeQuality/BenchmarksGame/regexdna/regexdna-input25.txt text eol=lf
7171
tests/src/JIT/Performance/CodeQuality/BenchmarksGame/regexdna/regexdna-input25000.txt text eol=lf
72+
tests/src/JIT/Performance/CodeQuality/BenchmarksGame/regex-redux/regexdna-input25.txt text eol=lf
73+
tests/src/JIT/Performance/CodeQuality/BenchmarksGame/regex-redux/regexdna-input25000.txt text eol=lf
7274
tests/src/JIT/Performance/CodeQuality/BenchmarksGame/reverse-complement/revcomp-input25.txt text eol=lf
7375
tests/src/JIT/Performance/CodeQuality/BenchmarksGame/reverse-complement/revcomp-input25000.txt text eol=lf
7476
tests/src/JIT/Performance/CodeQuality/BenchmarksGame/k-nucleotide/knucleotide-input.txt text eol=lf

‎tests/src/JIT/Performance/CodeQuality/BenchmarksGame/binarytrees/binarytrees-best.cs

+46-12
Original file line numberDiff line numberDiff line change
@@ -18,16 +18,36 @@ minor improvements by Alex Yakunin
1818
using System;
1919
using System.Runtime.CompilerServices;
2020
using System.Threading.Tasks;
21+
using Microsoft.Xunit.Performance;
22+
23+
[assembly: OptimizeForBenchmarks]
24+
[assembly: MeasureGCCounts]
2125

2226
namespace BenchmarksGame
2327
{
24-
public sealed class BinaryTrees
28+
public sealed class BinaryTrees_5
2529
{
2630
public const int MinDepth = 4;
2731

28-
public static void Main(string[] args)
32+
public static int Main(string[] args)
2933
{
3034
var n = args.Length == 0 ? 0 : int.Parse(args[0]);
35+
36+
int check = Bench(n, true);
37+
int expected = 4398;
38+
39+
// Return 100 on success, anything else on failure.
40+
return check - expected + 100;
41+
}
42+
43+
[Benchmark(InnerIterationCount = 7)]
44+
public static void RunBench()
45+
{
46+
Benchmark.Iterate(() => Bench(16, false));
47+
}
48+
49+
static int Bench(int n, bool verbose)
50+
{
3151
var maxDepth = n < (MinDepth + 2) ? MinDepth + 2 : n;
3252
var stretchDepth = maxDepth + 1;
3353

@@ -44,8 +64,8 @@ public static void Main(string[] args)
4464
var count = 0;
4565
if (depthCopy >= 17)
4666
{
47-
// Parallelized computation for relatively large tasks
48-
var miniTasks = new Task<int>[iterationCount];
67+
// Parallelized computation for relatively large tasks
68+
var miniTasks = new Task<int>[iterationCount];
4969
for (var i = 0; i < iterationCount; i++)
5070
miniTasks[i] = Task.Run(() => TreeNode.CreateTree(depthCopy).CountNodes());
5171
Task.WaitAll(miniTasks);
@@ -54,21 +74,35 @@ public static void Main(string[] args)
5474
}
5575
else
5676
{
57-
// Sequential computation for smaller tasks
58-
for (var i = 0; i < iterationCount; i++)
77+
// Sequential computation for smaller tasks
78+
for (var i = 0; i < iterationCount; i++)
5979
count += TreeNode.CreateTree(depthCopy).CountNodes();
6080
}
6181
return $"{iterationCount}\t trees of depth {depthCopy}\t check: {count}";
6282
});
6383
}
6484
Task.WaitAll(tasks);
6585

66-
Console.WriteLine("stretch tree of depth {0}\t check: {1}",
67-
stretchDepth, stretchDepthTask.Result);
68-
foreach (var task in tasks)
69-
Console.WriteLine(task.Result);
70-
Console.WriteLine("long lived tree of depth {0}\t check: {1}",
71-
maxDepth, maxDepthTask.Result);
86+
if (verbose)
87+
{
88+
int count = 0;
89+
Action<string> printAndSum = (string s) =>
90+
{
91+
Console.WriteLine(s);
92+
count += int.Parse(s.Substring(s.LastIndexOf(':') + 1).TrimStart());
93+
};
94+
95+
printAndSum(String.Format("stretch tree of depth {0}\t check: {1}",
96+
stretchDepth, stretchDepthTask.Result));
97+
foreach (var task in tasks)
98+
printAndSum(task.Result);
99+
printAndSum(String.Format("long lived tree of depth {0}\t check: {1}",
100+
maxDepth, maxDepthTask.Result));
101+
102+
return count;
103+
}
104+
105+
return 0;
72106
}
73107
}
74108

‎tests/src/JIT/Performance/CodeQuality/BenchmarksGame/binarytrees/binarytrees-best.csproj

+41
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,41 @@
1+
<?xml version="1.0" encoding="utf-8"?>
2+
<Project ToolsVersion="12.0" DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
3+
<Import Project="$([MSBuild]::GetDirectoryNameOfFileAbove($(MSBuildThisFileDirectory), dir.props))\dir.props" />
4+
<PropertyGroup>
5+
<Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration>
6+
<Platform Condition=" '$(Platform)' == '' ">AnyCPU</Platform>
7+
<SchemaVersion>2.0</SchemaVersion>
8+
<ProjectGuid>{95DFC527-4DC1-495E-97D7-E94EE1F7140D}</ProjectGuid>
9+
<OutputType>Exe</OutputType>
10+
<ProjectTypeGuids>{786C830F-07A1-408B-BD7F-6EE04809D6DB};{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}</ProjectTypeGuids>
11+
<SolutionDir Condition="$(SolutionDir) == '' Or $(SolutionDir) == '*Undefined*'">..\..\</SolutionDir>
12+
<NuGetTargetMoniker>.NETStandard,Version=v1.4</NuGetTargetMoniker>
13+
<NuGetTargetMonikerShort>netstandard1.4</NuGetTargetMonikerShort>
14+
<GCStressIncompatible>true</GCStressIncompatible>
15+
</PropertyGroup>
16+
<!-- Default configurations to help VS understand the configurations -->
17+
<PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|AnyCPU' ">
18+
</PropertyGroup>
19+
<PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Release|AnyCPU' ">
20+
</PropertyGroup>
21+
<PropertyGroup>
22+
<DebugType>pdbonly</DebugType>
23+
<Optimize>true</Optimize>
24+
</PropertyGroup>
25+
<ItemGroup>
26+
<CodeAnalysisDependentAssemblyPaths Condition=" '$(VS100COMNTOOLS)' != '' " Include="$(VS100COMNTOOLS)..\IDE\PrivateAssemblies">
27+
<Visible>False</Visible>
28+
</CodeAnalysisDependentAssemblyPaths>
29+
</ItemGroup>
30+
<ItemGroup>
31+
<Service Include="{82A7F48D-3B50-4B1E-B82E-3ADA8210C358}" />
32+
</ItemGroup>
33+
<ItemGroup>
34+
<Compile Include="$(MSBuildProjectName).cs" />
35+
</ItemGroup>
36+
<Import Project="$([MSBuild]::GetDirectoryNameOfFileAbove($(MSBuildThisFileDirectory), dir.targets))\dir.targets" />
37+
<PropertyGroup Condition=" '$(MsBuildProjectDirOverride)' != '' "></PropertyGroup>
38+
<PropertyGroup>
39+
<ProjectAssetsFile>$(JitPackagesConfigFileDirectory)benchmark\obj\project.assets.json</ProjectAssetsFile>
40+
</PropertyGroup>
41+
</Project>

‎tests/src/JIT/Performance/CodeQuality/BenchmarksGame/binarytrees/binarytrees-serial.cs

+33-7
Original file line numberDiff line numberDiff line change
@@ -14,23 +14,43 @@ contributed by Marek Safar
1414
*/
1515

1616
using System;
17+
using Microsoft.Xunit.Performance;
18+
19+
[assembly: OptimizeForBenchmarks]
20+
[assembly: MeasureGCCounts]
1721

1822
namespace BenchmarksGame
1923
{
20-
class BinaryTrees
24+
public class BinaryTrees_2
2125
{
2226
const int minDepth = 4;
2327

24-
public static void Main(String[] args)
28+
public static int Main(String[] args)
2529
{
2630
int n = 0;
2731
if (args.Length > 0) n = Int32.Parse(args[0]);
2832

33+
int check = Bench(n, true);
34+
int expected = 4398;
35+
36+
// Return 100 on success, anything else on failure.
37+
return check - expected + 100;
38+
}
39+
40+
[Benchmark(InnerIterationCount = 7)]
41+
public static void RunBench()
42+
{
43+
Benchmark.Iterate(() => Bench(16, false));
44+
}
45+
46+
static int Bench(int n, bool verbose)
47+
{
2948
int maxDepth = Math.Max(minDepth + 2, n);
3049
int stretchDepth = maxDepth + 1;
3150

3251
int check = (TreeNode.bottomUpTree(stretchDepth)).itemCheck();
33-
Console.WriteLine("stretch tree of depth {0}\t check: {1}", stretchDepth, check);
52+
int checkSum = check;
53+
if (verbose) Console.WriteLine("stretch tree of depth {0}\t check: {1}", stretchDepth, check);
3454

3555
TreeNode longLivedTree = TreeNode.bottomUpTree(maxDepth);
3656

@@ -43,13 +63,19 @@ public static void Main(String[] args)
4363
{
4464
check += (TreeNode.bottomUpTree(depth)).itemCheck();
4565
}
66+
checkSum += check;
4667

47-
Console.WriteLine("{0}\t trees of depth {1}\t check: {2}",
48-
iterations, depth, check);
68+
if (verbose)
69+
Console.WriteLine("{0}\t trees of depth {1}\t check: {2}", iterations, depth, check);
4970
}
5071

51-
Console.WriteLine("long lived tree of depth {0}\t check: {1}",
52-
maxDepth, longLivedTree.itemCheck());
72+
check = longLivedTree.itemCheck();
73+
checkSum += check;
74+
75+
if (verbose)
76+
Console.WriteLine("long lived tree of depth {0}\t check: {1}", maxDepth, check);
77+
78+
return checkSum;
5379
}
5480

5581

‎tests/src/JIT/Performance/CodeQuality/BenchmarksGame/binarytrees/binarytrees-serial.csproj

+41
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,41 @@
1+
<?xml version="1.0" encoding="utf-8"?>
2+
<Project ToolsVersion="12.0" DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
3+
<Import Project="$([MSBuild]::GetDirectoryNameOfFileAbove($(MSBuildThisFileDirectory), dir.props))\dir.props" />
4+
<PropertyGroup>
5+
<Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration>
6+
<Platform Condition=" '$(Platform)' == '' ">AnyCPU</Platform>
7+
<SchemaVersion>2.0</SchemaVersion>
8+
<ProjectGuid>{95DFC527-4DC1-495E-97D7-E94EE1F7140D}</ProjectGuid>
9+
<OutputType>Exe</OutputType>
10+
<ProjectTypeGuids>{786C830F-07A1-408B-BD7F-6EE04809D6DB};{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}</ProjectTypeGuids>
11+
<SolutionDir Condition="$(SolutionDir) == '' Or $(SolutionDir) == '*Undefined*'">..\..\</SolutionDir>
12+
<NuGetTargetMoniker>.NETStandard,Version=v1.4</NuGetTargetMoniker>
13+
<NuGetTargetMonikerShort>netstandard1.4</NuGetTargetMonikerShort>
14+
<GCStressIncompatible>true</GCStressIncompatible>
15+
</PropertyGroup>
16+
<!-- Default configurations to help VS understand the configurations -->
17+
<PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|AnyCPU' ">
18+
</PropertyGroup>
19+
<PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Release|AnyCPU' ">
20+
</PropertyGroup>
21+
<PropertyGroup>
22+
<DebugType>pdbonly</DebugType>
23+
<Optimize>true</Optimize>
24+
</PropertyGroup>
25+
<ItemGroup>
26+
<CodeAnalysisDependentAssemblyPaths Condition=" '$(VS100COMNTOOLS)' != '' " Include="$(VS100COMNTOOLS)..\IDE\PrivateAssemblies">
27+
<Visible>False</Visible>
28+
</CodeAnalysisDependentAssemblyPaths>
29+
</ItemGroup>
30+
<ItemGroup>
31+
<Service Include="{82A7F48D-3B50-4B1E-B82E-3ADA8210C358}" />
32+
</ItemGroup>
33+
<ItemGroup>
34+
<Compile Include="$(MSBuildProjectName).cs" />
35+
</ItemGroup>
36+
<Import Project="$([MSBuild]::GetDirectoryNameOfFileAbove($(MSBuildThisFileDirectory), dir.targets))\dir.targets" />
37+
<PropertyGroup Condition=" '$(MsBuildProjectDirOverride)' != '' "></PropertyGroup>
38+
<PropertyGroup>
39+
<ProjectAssetsFile>$(JitPackagesConfigFileDirectory)benchmark\obj\project.assets.json</ProjectAssetsFile>
40+
</PropertyGroup>
41+
</Project>

‎tests/src/JIT/Performance/CodeQuality/BenchmarksGame/fannkuch-redux/fannkuch-redux-best.cs

+30-3
Original file line numberDiff line numberDiff line change
@@ -17,10 +17,14 @@ parallel and small optimisations by Anthony Lloyd
1717
using System;
1818
using System.Threading;
1919
using System.Runtime.CompilerServices;
20+
using Microsoft.Xunit.Performance;
21+
using Xunit;
22+
23+
[assembly: OptimizeForBenchmarks]
2024

2125
namespace BenchmarksGame
2226
{
23-
public static class FannkuchRedux
27+
public static class FannkuchRedux_5
2428
{
2529
static int[] fact, chkSums, maxFlips;
2630
[MethodImpl(MethodImplOptions.AggressiveInlining)]
@@ -98,9 +102,30 @@ static void run(int n, int taskId, int taskSize)
98102
maxFlips[taskId] = maxflips;
99103
}
100104

101-
public static void Main(string[] args)
105+
public static int Main(string[] args)
102106
{
103107
int n = args.Length > 0 ? int.Parse(args[0]) : 7;
108+
int sum = Bench(n, true);
109+
110+
int expected = 228;
111+
112+
// Return 100 on success, anything else on failure.
113+
return sum - expected + 100;
114+
}
115+
116+
[Benchmark(InnerIterationCount = 20)]
117+
[InlineData(10, 73196)]
118+
public static void RunBench(int n, int expectedSum)
119+
{
120+
Benchmark.Iterate(() =>
121+
{
122+
int sum = Bench(n, false);
123+
Assert.Equal(expectedSum, sum);
124+
});
125+
}
126+
127+
static int Bench(int n, bool verbose)
128+
{
104129
fact = new int[n + 1];
105130
fact[0] = 1;
106131
var factn = 1;
@@ -124,7 +149,9 @@ public static void Main(string[] args)
124149
chksum += chkSums[i];
125150
if (maxFlips[i] > maxflips) maxflips = maxFlips[i];
126151
}
127-
Console.Out.WriteLineAsync(chksum + "\nPfannkuchen(" + n + ") = " + maxflips);
152+
if (verbose) Console.Out.WriteLineAsync(chksum + "\nPfannkuchen(" + n + ") = " + maxflips);
153+
154+
return chksum;
128155
}
129156
}
130157
}

‎tests/src/JIT/Performance/CodeQuality/BenchmarksGame/fannkuch-redux/fannkuch-redux-best.csproj

+38
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
<?xml version="1.0" encoding="utf-8"?>
2+
<Project ToolsVersion="12.0" DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
3+
<Import Project="$([MSBuild]::GetDirectoryNameOfFileAbove($(MSBuildThisFileDirectory), dir.props))\dir.props" />
4+
<PropertyGroup>
5+
<Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration>
6+
<Platform Condition=" '$(Platform)' == '' ">AnyCPU</Platform>
7+
<SchemaVersion>2.0</SchemaVersion>
8+
<ProjectGuid>{F49D82D3-9D13-47B5-83F8-52B1FE3FF452}</ProjectGuid>
9+
<OutputType>Exe</OutputType>
10+
<ProjectTypeGuids>{786C830F-07A1-408B-BD7F-6EE04809D6DB};{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}</ProjectTypeGuids>
11+
<SolutionDir Condition="$(SolutionDir) == '' Or $(SolutionDir) == '*Undefined*'">..\..\</SolutionDir>
12+
<NuGetTargetMoniker>.NETStandard,Version=v1.4</NuGetTargetMoniker>
13+
<NuGetTargetMonikerShort>netstandard1.4</NuGetTargetMonikerShort>
14+
</PropertyGroup>
15+
<!-- Default configurations to help VS understand the configurations -->
16+
<PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|AnyCPU' "></PropertyGroup>
17+
<PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Release|AnyCPU' " />
18+
<PropertyGroup>
19+
<DebugType>pdbonly</DebugType>
20+
<Optimize>true</Optimize>
21+
</PropertyGroup>
22+
<ItemGroup>
23+
<CodeAnalysisDependentAssemblyPaths Condition=" '$(VS100COMNTOOLS)' != '' " Include="$(VS100COMNTOOLS)..\IDE\PrivateAssemblies">
24+
<Visible>False</Visible>
25+
</CodeAnalysisDependentAssemblyPaths>
26+
</ItemGroup>
27+
<ItemGroup>
28+
<Service Include="{82A7F48D-3B50-4B1E-B82E-3ADA8210C358}" />
29+
</ItemGroup>
30+
<ItemGroup>
31+
<Compile Include="$(MSBuildProjectName).cs" />
32+
</ItemGroup>
33+
<Import Project="$([MSBuild]::GetDirectoryNameOfFileAbove($(MSBuildThisFileDirectory), dir.targets))\dir.targets" />
34+
<PropertyGroup Condition=" '$(MsBuildProjectDirOverride)' != '' "></PropertyGroup>
35+
<PropertyGroup>
36+
<ProjectAssetsFile>$(JitPackagesConfigFileDirectory)benchmark\obj\project.assets.json</ProjectAssetsFile>
37+
</PropertyGroup>
38+
</Project>

0 commit comments

Comments
 (0)