diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml new file mode 100644 index 000000000..45ce3c915 --- /dev/null +++ b/.github/workflows/benchmark.yml @@ -0,0 +1,49 @@ +name: Build and run benchmarks + +# Description: +# This workflow automates performance benchmarking for the Coverlet project. +# +# Triggers: +# - Manual trigger via workflow_dispatch +# +# What it does: +# 1. Sets up Ubuntu environment +# 2. Installs .NET SDK based on global.json +# 3. Builds benchmark tests in Release mode +# 4. Runs performance benchmarks using BenchmarkDotNet +# 5. Uploads benchmark results as artifacts +# +# Results: +# - Benchmark artifacts are stored under: artifacts/bin/coverlet.core.benchmark.tests/release/BenchmarkDotNet.Artifacts/results/ +# - Results can be downloaded from GitHub Actions artifacts +# +# Usage: +# - Can be manually triggered from Actions tab using workflow_dispatch +# - Use results to monitor performance changes over time + +on: + workflow_dispatch: + +jobs: + build: + + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4.2.2 + - name: Setup .NET + uses: actions/setup-dotnet@v3.4.0 + with: + global-json-file: global.json + source-url: https://pkgs.dev.azure.com/bertk0374/_packaging/intern/nuget/v3/index.json + - name: Restore dependencies + run: dotnet restore + - name: Build release + run: dotnet build --no-restore -c Release ./test/coverlet.core.benchmark.tests/coverlet.core.benchmark.tests.csproj + - name: Run Benchmarks + run: cd ./artifacts/bin/coverlet.core.benchmark.tests/release && ./coverlet.core.benchmark.tests + - name: Upload benchmark results + uses: actions/upload-artifact@v2 + with: + name: Benchmark_Results + path: ./artifacts/bin/coverlet.core.benchmark.tests/release/BenchmarkDotNet.Artifacts/results/* diff --git a/.gitignore b/.gitignore index 514880624..d53610b92 100644 --- a/.gitignore +++ b/.gitignore @@ -315,6 +315,8 @@ coverage.*.cobertura.xml coverage.*.opencover.xml FolderProfile.pubxml +BenchmarkDotNet.Artifacts/ /NuGet.config nuget.config *.dmp +test/coverlet.core.benchmark.tests/InstrumentationHelperBenchmarks.cs diff --git a/Directory.Packages.props b/Directory.Packages.props index eba94e254..51dafc0a9 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -17,6 +17,10 @@ 3.0.2 + + + + diff --git a/coverlet.sln b/coverlet.sln index a449bd072..6dd67136f 100644 --- a/coverlet.sln +++ b/coverlet.sln @@ -86,6 +86,8 @@ Project("{778DAE3C-4631-46EA-AA77-85C1314464D9}") = "coverlet.tests.projectsampl EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "coverlet.tests.utils", "test\coverlet.tests.utils\coverlet.tests.utils.csproj", "{0B109210-03CB-413F-888C-3023994AA384}" EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "coverlet.core.benchmark.tests", "test\coverlet.core.benchmark.tests\coverlet.core.benchmark.tests.csproj", "{8E8C4799-6F9D-49D8-96EA-E9BD1D187DAD}" +EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "coverlet.tests.projectsample.wpf8.selfcontained", "test\coverlet.tests.projectsample.wpf8.selfcontained\coverlet.tests.projectsample.wpf8.selfcontained.csproj", "{71004336-9896-4AE5-8367-B29BB1680542}" EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "coverlet.core.coverage.tests", "test\coverlet.core.coverage.tests\coverlet.core.coverage.tests.csproj", "{F74AD549-EFE0-4CD9-AD10-B2189E3FD5BB}" @@ -188,6 +190,10 @@ Global {0B109210-03CB-413F-888C-3023994AA384}.Debug|Any CPU.Build.0 = Debug|Any CPU {0B109210-03CB-413F-888C-3023994AA384}.Release|Any CPU.ActiveCfg = Release|Any CPU {0B109210-03CB-413F-888C-3023994AA384}.Release|Any CPU.Build.0 = Release|Any CPU + {8E8C4799-6F9D-49D8-96EA-E9BD1D187DAD}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {8E8C4799-6F9D-49D8-96EA-E9BD1D187DAD}.Debug|Any CPU.Build.0 = Debug|Any CPU + {8E8C4799-6F9D-49D8-96EA-E9BD1D187DAD}.Release|Any CPU.ActiveCfg = Release|Any CPU + {8E8C4799-6F9D-49D8-96EA-E9BD1D187DAD}.Release|Any CPU.Build.0 = Release|Any CPU {71004336-9896-4AE5-8367-B29BB1680542}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {71004336-9896-4AE5-8367-B29BB1680542}.Debug|Any CPU.Build.0 = Debug|Any CPU {71004336-9896-4AE5-8367-B29BB1680542}.Release|Any CPU.ActiveCfg = Release|Any CPU @@ -225,6 +231,7 @@ Global {351A034E-E642-4DB9-A21D-F71C8151C243} = {2FEBDE1B-83E3-445B-B9F8-5644B0E0E134} {03400776-1F9A-4326-B927-1CA9B64B42A1} = {2FEBDE1B-83E3-445B-B9F8-5644B0E0E134} {0B109210-03CB-413F-888C-3023994AA384} = {2FEBDE1B-83E3-445B-B9F8-5644B0E0E134} + {8E8C4799-6F9D-49D8-96EA-E9BD1D187DAD} = {2FEBDE1B-83E3-445B-B9F8-5644B0E0E134} {71004336-9896-4AE5-8367-B29BB1680542} = {2FEBDE1B-83E3-445B-B9F8-5644B0E0E134} {F74AD549-EFE0-4CD9-AD10-B2189E3FD5BB} = {2FEBDE1B-83E3-445B-B9F8-5644B0E0E134} EndGlobalSection diff --git a/src/coverlet.core/Properties/AssemblyInfo.cs b/src/coverlet.core/Properties/AssemblyInfo.cs index 0a6d02544..3e03b0a2f 100644 --- a/src/coverlet.core/Properties/AssemblyInfo.cs +++ b/src/coverlet.core/Properties/AssemblyInfo.cs @@ -12,6 +12,7 @@ [assembly: InternalsVisibleTo("coverlet.core.tests, PublicKey=0024000004800000940000000602000000240000525341310004000001000100757cf9291d78a82e5bb58a827a3c46c2f959318327ad30d1b52e918321ffbd847fb21565b8576d2a3a24562a93e86c77a298b564a0f1b98f63d7a1441a3a8bcc206da3ed09d5dacc76e122a109a9d3ac608e21a054d667a2bae98510a1f0f653c0e6f58f42b4b3934f6012f5ec4a09b3dfd3e14d437ede1424bdb722aead64ad")] [assembly: InternalsVisibleTo("coverlet.core.coverage.tests, PublicKey=0024000004800000940000000602000000240000525341310004000001000100094aad8eb75c06c9f2443dda84573b8db55cd6678452a60010db2643467ac28928db3a06b0b1ac3016645b448937d5e671b36504bcfc0fda27e996c5e1b0ee49747145cda6d47508d1e3c60b144634d95e33d4efe49536372df8139f48d3d897ae6931c2876d4f5d00215fd991cbcecde2705e53e19309e21c8b59d19eb925b1")] +[assembly: InternalsVisibleTo("coverlet.core.benchmark.tests, PublicKey=002400000480000094000000060200000024000052534131000400000100010061d9d48f9cd6a4733ea1d88bc8a09c53a3040c3446c41858781df135170e8fe4e82a6cc6d9836f070ae0a28ebd7cd6e30dc1a853b350ae08ae77f437bc9f9f3b0ef23eb9b05eea38f97edb26a2dd2d0d8b32c6335c47b32f5277621118267f1a5717233eae25a3fe126d89d14b85a7a8e07657bf681a8a82100762a42ec477aa")] [assembly: InternalsVisibleTo("coverlet.collector.tests, PublicKey=0024000004800000940000000602000000240000525341310004000001000100ed0ed6af9693182615b8dcadc83c918b8d36312f86cefc69539d67d4189cd1b89420e7c3871802ffef7f5ca7816c68ad856c77bf7c230cc07824d96aa5d1237eebd30e246b9a14e22695fb26b40c800f74ea96619092cbd3a5d430d6c003fc7a82e8ccd1e315b935105d9232fe9e99e8d7ff54bba6f191959338d4a3169df9b3")] [assembly: InternalsVisibleTo("coverlet.msbuild.tasks.tests, PublicKey=002400000480000094000000060200000024000052534131000400000100010071b1583d63637a225f3f640252fee7130f0f3f2127d75025c1c3ee2d6dfc79a4950919268e0784d7ff54b0eadd8e4762e3e150da422e20e091eb0811d9d84e1779d5b95e349d5428aebb16e82e081bdf805926c5a9eb2094aaed9d36442de024264976a8835c7d6923047cf2f745e8f0ded2332f8980acd390f725224d976ed8")] [assembly: InternalsVisibleTo("coverlet.integration.tests, PublicKey=002400000480000094000000060200000024000052534131000400000100010001d24efbe9cbc2dc49b7a3d2ae34ca37cfb69b4f450acd768a22ce5cd021c8a38ae7dc68b2809a1ac606ad531b578f192a5690b2986990cbda4dd84ec65a3a4c1c36f6d7bb18f08592b93091535eaee2f0c8e48763ed7f190db2008e1f9e0facd5c0df5aaab74febd3430e09a428a72e5e6b88357f92d78e47512d46ebdc3cbb")] diff --git a/test/coverlet.core.benchmark.tests/CoverageBenchmarks.cs b/test/coverlet.core.benchmark.tests/CoverageBenchmarks.cs new file mode 100644 index 000000000..4018da93e --- /dev/null +++ b/test/coverlet.core.benchmark.tests/CoverageBenchmarks.cs @@ -0,0 +1,68 @@ +// Copyright (c) Toni Solarin-Sodara +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using System; +using System.IO; +using BenchmarkDotNet.Attributes; +using Coverlet.Core; +using Coverlet.Core.Abstractions; +using Coverlet.Core.Helpers; +using Coverlet.Core.Symbols; +using Moq; + +namespace coverlet.core.benchmark.tests +{ + [MemoryDiagnoser] + public class CoverageBenchmarks + { + private Coverage _coverage; + private readonly Mock _mockLogger = new(); + private DirectoryInfo _directory; + + [GlobalSetup(Target = nameof(GetCoverageBenchmark))] + public void GetCoverageBenchmarkSetup() + { + string module = GetType().Assembly.Location; + string pdb = Path.Combine(Path.GetDirectoryName(module), Path.GetFileNameWithoutExtension(module) + ".pdb"); + + _directory = Directory.CreateDirectory(Path.Combine(Path.GetTempPath(), Guid.NewGuid().ToString())); + + File.Copy(module, Path.Combine(_directory.FullName, Path.GetFileName(module)), true); + File.Copy(pdb, Path.Combine(_directory.FullName, Path.GetFileName(pdb)), true); + + // TODO: Find a way to mimick hits + var instrumentationHelper = + new InstrumentationHelper(new ProcessExitHandler(), new RetryHelper(), new FileSystem(), new Mock().Object, + new SourceRootTranslator(module, new Mock().Object, new FileSystem(), new AssemblyAdapter())); + + var parameters = new CoverageParameters + { + IncludeFilters = new string[] { "[coverlet.tests.projectsample.excludedbyattribute*]*" }, + IncludeDirectories = Array.Empty(), + ExcludeFilters = Array.Empty(), + ExcludedSourceFiles = Array.Empty(), + ExcludeAttributes = Array.Empty(), + IncludeTestAssembly = false, + SingleHit = false, + MergeWith = string.Empty, + UseSourceLink = false + }; + + _coverage = new Coverage(Path.Combine(_directory.FullName, Path.GetFileName(module)), parameters, _mockLogger.Object, instrumentationHelper, new FileSystem(), new SourceRootTranslator(_mockLogger.Object, new FileSystem()), new CecilSymbolHelper()); + _coverage.PrepareModules(); + + } + + [GlobalCleanup] + public void IterationCleanup() + { + _directory.Delete(true); + } + + [Benchmark] + public void GetCoverageBenchmark() + { + CoverageResult result = _coverage.GetCoverageResult(); + } + } +} diff --git a/test/coverlet.core.benchmark.tests/HowTo.md b/test/coverlet.core.benchmark.tests/HowTo.md new file mode 100644 index 000000000..e8cccc989 --- /dev/null +++ b/test/coverlet.core.benchmark.tests/HowTo.md @@ -0,0 +1,134 @@ +# How to benchmark coverlet.core + +Coverlet.core.benchmark uses [BenchmarkDotNet](https://github.com/dotnet/BenchmarkDotNet) which has some runtime requirements + +- Build the project in `Release` mode +- Make sure you have the latest version of the .NET SDK installed +- Make sure you have the latest version of the BenchmarkDotNet package installed + +Use a terminal and run the following commands: + +```bash +dotnet build test/coverlet.core.benchmark.tests -c release +cd artifacts/bin/coverlet.core.benchmark.tests/release +coverlet.core.benchmark.tests.exe +``` +> [!TIP] +> If error occurred missing `TestAssets\System.Private.CoreLib.dll` or `TestAssets\System.Private.CoreLib.pdb`. +> Just copy the files from `artifacts\bin\coverlet.core.tests\debug\TestAssets`. + +The benchmark will automatically create reports in folder `BenchmarkDotNet.Artifacts` eg. find these files: + +``` +BenchmarkRun-20250411-083105.log +results\BenchmarkRun-joined-2025-04-11-08-38-13-report-github.md +results\BenchmarkRun-joined-2025-04-11-08-38-13-report.csv +results\BenchmarkRun-joined-2025-04-11-08-38-13-report.html +results\BenchmarkRun-joined-2025-04-11-08-55-34-report-github.md +results\BenchmarkRun-joined-2025-04-11-08-55-34-report.csv +results\BenchmarkRun-joined-2025-04-11-08-55-34-report.html +``` + +> [!NOTE] +> This should be done for every coverlet release to avoid performance degradations. + + +## Additional information +- [BenchmarkDotNet](https://benchmarkdotnet.org) +- [Analyze BenchmarkDotNet data in Visual Studio](https://learn.microsoft.com/en-us/visualstudio/profiling/profiling-with-benchmark-dotnet) +- [.NET benchmarking and profiling for beginners](https://medium.com/ingeniouslysimple/net-benchmarking-and-profiling-for-beginners-62462e1e9a19) + +
+ +## Coverlet 6.0.0 + +``` + +BenchmarkDotNet v0.14.0, Windows 11 (10.0.26120.3671) +AMD Ryzen 7 Microsoft Surface Edition, 1 CPU, 16 logical and 8 physical cores +.NET SDK 9.0.203 + [Host] : .NET 6.0.36 (6.0.3624.51421), X64 RyuJIT AVX2 + +Job=ShortRun Toolchain=InProcessNoEmitToolchain IterationCount=3 +LaunchCount=1 WarmupCount=3 + +``` +| Type | Method | Mean | Error | StdDev | Gen0 | Gen1 | Gen2 | Allocated | +|----------------------- |---------------------- |--------------------:|-------------------:|------------------:|------------:|------------:|----------:|-------------:| +| CoverageBenchmarks | GetCoverageBenchmark | 46.42 ns | 1.670 ns | 0.092 ns | 0.0612 | - | - | 128 B | +| InstrumenterBenchmarks | InstrumenterBenchmark | 4,938,713,766.67 ns | 767,760,955.034 ns | 42,083,568.809 ns | 857000.0000 | 109000.0000 | 2000.0000 | 2879633880 B | + +## Coverlet 6.0.1 + +``` + +BenchmarkDotNet v0.14.0, Windows 11 (10.0.26120.3671) +AMD Ryzen 7 Microsoft Surface Edition, 1 CPU, 16 logical and 8 physical cores +.NET SDK 8.0.408 + [Host] : .NET 8.0.15 (8.0.1525.16413), X64 RyuJIT AVX2 + +Job=ShortRun Toolchain=InProcessNoEmitToolchain IterationCount=3 +LaunchCount=1 WarmupCount=3 + +``` +| Type | Method | Mean | Error | StdDev | Gen0 | Gen1 | Gen2 | Allocated | +|----------------------- |---------------------- |--------------------:|-------------------:|------------------:|------------:|-----------:|----------:|-------------:| +| CoverageBenchmarks | GetCoverageBenchmark | 48.14 ns | 8.681 ns | 0.476 ns | 0.0612 | - | - | 128 B | +| InstrumenterBenchmarks | InstrumenterBenchmark | 3,675,771,933.33 ns | 874,256,026.013 ns | 47,920,923.025 ns | 789000.0000 | 97000.0000 | 2000.0000 | 2864466608 B | + +## Coverlet 6.0.2 + +``` + +BenchmarkDotNet v0.14.0, Windows 11 (10.0.26120.3671) +AMD Ryzen 7 Microsoft Surface Edition, 1 CPU, 16 logical and 8 physical cores +.NET SDK 8.0.408 + [Host] : .NET 8.0.15 (8.0.1525.16413), X64 RyuJIT AVX2 + +Job=ShortRun Toolchain=InProcessNoEmitToolchain IterationCount=3 +LaunchCount=1 WarmupCount=3 + +``` +| Type | Method | Mean | Error | StdDev | Gen0 | Gen1 | Gen2 | Allocated | +|----------------------- |---------------------- |---------------------:|--------------------:|-------------------:|------------:|------------:|----------:|-------------:| +| CoverageBenchmarks | GetCoverageBenchmark | 46.20 ns | 12.15 ns | 0.666 ns | 0.0612 | - | - | 128 B | +| InstrumenterBenchmarks | InstrumenterBenchmark | 19,105,224,033.33 ns | 4,450,103,671.99 ns | 243,925,199.451 ns | 867000.0000 | 130000.0000 | 2000.0000 | 3170097400 B | + +## Coverlet 6.0.3 + +``` + +BenchmarkDotNet v0.14.0, Windows 11 (10.0.26120.3671) +AMD Ryzen 7 Microsoft Surface Edition, 1 CPU, 16 logical and 8 physical cores +.NET SDK 8.0.408 + [Host] : .NET 8.0.15 (8.0.1525.16413), X64 RyuJIT AVX2 + +Job=ShortRun Toolchain=InProcessNoEmitToolchain IterationCount=3 +LaunchCount=1 WarmupCount=3 + +``` +| Type | Method | Mean | Error | StdDev | Gen0 | Gen1 | Gen2 | Allocated | +|----------------------- |---------------------- |--------------------:|-------------------:|------------------:|------------:|-----------:|----------:|-------------:| +| CoverageBenchmarks | GetCoverageBenchmark | 47.32 ns | 4.246 ns | 0.233 ns | 0.0612 | - | - | 128 B | +| InstrumenterBenchmarks | InstrumenterBenchmark | 3,620,665,600.00 ns | 580,611,812.738 ns | 31,825,292.772 ns | 775000.0000 | 91000.0000 | 2000.0000 | 2798558288 B | + +## Coverlet 6.0.4 + +``` + +BenchmarkDotNet v0.14.0, Windows 11 (10.0.26120.3671) +AMD Ryzen 7 Microsoft Surface Edition, 1 CPU, 16 logical and 8 physical cores +.NET SDK 8.0.408 + [Host] : .NET 8.0.15 (8.0.1525.16413), X64 RyuJIT AVX2 + +Job=ShortRun Toolchain=InProcessNoEmitToolchain IterationCount=3 +LaunchCount=1 WarmupCount=3 + +``` +| Type | Method | Mean | Error | StdDev | Gen0 | Gen1 | Gen2 | Allocated | +|----------------------- |---------------------- |--------------------:|-------------------:|------------------:|------------:|-----------:|----------:|-------------:| +| CoverageBenchmarks | GetCoverageBenchmark | 43.59 ns | 9.890 ns | 0.542 ns | 0.0612 | - | - | 128 B | +| InstrumenterBenchmarks | InstrumenterBenchmark | 3,594,263,533.33 ns | 193,126,202.387 ns | 10,585,898.871 ns | 776000.0000 | 97000.0000 | 2000.0000 | 2798557560 B | + + +
diff --git a/test/coverlet.core.benchmark.tests/InstrumenterBenchmarks.cs b/test/coverlet.core.benchmark.tests/InstrumenterBenchmarks.cs new file mode 100644 index 000000000..58682129c --- /dev/null +++ b/test/coverlet.core.benchmark.tests/InstrumenterBenchmarks.cs @@ -0,0 +1,108 @@ +// Copyright (c) Toni Solarin-Sodara +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using System.IO; +using BenchmarkDotNet.Attributes; +using Coverlet.Core; +using Coverlet.Core.Abstractions; +using Coverlet.Core.Helpers; +using Coverlet.Core.Instrumentation; +using Coverlet.Core.Symbols; +using Moq; + +namespace coverlet.core.benchmark.tests +{ + public class InstrumenterBenchmarks + { + Mock _mockLogger; + Mock _partialMockFileSystem; + readonly string[] _files = new[] + { + "System.Private.CoreLib.dll", + "System.Private.CoreLib.pdb" + }; + Instrumenter _instrumenter; + DirectoryInfo _directory; + SourceRootTranslator _sourceRootTranslator; + CoverageParameters _parameters; + InstrumentationHelper _instrumentationHelper; + + [GlobalCleanup] + public void IterationCleanup() + { + _directory.Delete(true); + } + + [Benchmark] + public void InstrumenterBenchmark() + { + _mockLogger = new Mock(); + _directory = Directory.CreateDirectory(Path.Combine(Directory.GetCurrentDirectory(), nameof(InstrumenterBenchmark))); + + foreach (string file in _files) + { + File.Copy(Path.Combine(Directory.GetCurrentDirectory(), "TestAssets", file), Path.Combine(_directory.FullName, file), overwrite: true); + } + + _partialMockFileSystem = new Mock(); + _partialMockFileSystem.CallBase = true; + _partialMockFileSystem.Setup(fs => fs.OpenRead(It.IsAny())).Returns((string path) => + { + if (Path.GetFileName(path.Replace(@"\", @"/")) == _files[1]) + { + return File.OpenRead(Path.Combine(Path.Combine(Directory.GetCurrentDirectory(), "TestAssets"), _files[1])); + } + else + { + return File.OpenRead(path); + } + }); + _partialMockFileSystem.Setup(fs => fs.Exists(It.IsAny())).Returns((string path) => + { + if (Path.GetFileName(path.Replace(@"\", @"/")) == _files[1]) + { + return File.Exists(Path.Combine(Path.Combine(Directory.GetCurrentDirectory(), "TestAssets"), _files[1])); + } + else + { + if (path.Contains(@":\git\runtime")) + { + return true; + } + else + { + return File.Exists(path); + } + } + }); + _sourceRootTranslator = new SourceRootTranslator(_mockLogger.Object, new FileSystem()); + _parameters = new CoverageParameters(); + _instrumentationHelper = + new InstrumentationHelper(new ProcessExitHandler(), new RetryHelper(), _partialMockFileSystem.Object, _mockLogger.Object, _sourceRootTranslator); + _instrumenter = new Instrumenter(Path.Combine(_directory.FullName, _files[0]), "_coverlet_instrumented", _parameters, _mockLogger.Object, _instrumentationHelper, _partialMockFileSystem.Object, _sourceRootTranslator, new CecilSymbolHelper()); + + // implement your benchmark here + InstrumenterResult result = _instrumenter.Instrument(); + } + + [Benchmark] + public void InstrumenterBigClassBenchmark() + { + _mockLogger = new Mock(); + + string bigClassFilePath = Path.Combine(Directory.GetCurrentDirectory(), "coverlet.testsubject.dll"); + + _partialMockFileSystem = new Mock(); + _partialMockFileSystem.CallBase = true; + + _sourceRootTranslator = new SourceRootTranslator(_mockLogger.Object, new FileSystem()); + _parameters = new CoverageParameters(); + _instrumentationHelper = + new InstrumentationHelper(new ProcessExitHandler(), new RetryHelper(), _partialMockFileSystem.Object, _mockLogger.Object, _sourceRootTranslator); + _instrumenter = new Instrumenter(bigClassFilePath, "_coverlet_instrumented", _parameters, _mockLogger.Object, _instrumentationHelper, _partialMockFileSystem.Object, _sourceRootTranslator, new CecilSymbolHelper()); + + // Benchmark the instrumentation of BigClass + InstrumenterResult result = _instrumenter.Instrument(); + } + } +} diff --git a/test/coverlet.core.benchmark.tests/Program.cs b/test/coverlet.core.benchmark.tests/Program.cs new file mode 100644 index 000000000..a90a9cdc6 --- /dev/null +++ b/test/coverlet.core.benchmark.tests/Program.cs @@ -0,0 +1,46 @@ +// Copyright (c) Toni Solarin-Sodara +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +//using System.Diagnostics.Tracing; +using BenchmarkDotNet.Configs; +using BenchmarkDotNet.Diagnosers; +//using BenchmarkDotNet.Diagnostics.Windows; +using BenchmarkDotNet.Exporters; +using BenchmarkDotNet.Exporters.Csv; +using BenchmarkDotNet.Exporters.Json; +using BenchmarkDotNet.Exporters.Xml; +using BenchmarkDotNet.Jobs; +using BenchmarkDotNet.Running; +using BenchmarkDotNet.Toolchains.InProcess.NoEmit; +//using Microsoft.Diagnostics.NETCore.Client; +//using Microsoft.Diagnostics.Tracing.Parsers; + +namespace coverlet.core.benchmark.tests +{ + public class Program + { + + public static void Main(string[] args) + { + var config = DefaultConfig.Instance + .WithOptions(ConfigOptions.JoinSummary) + .AddJob(Job + .ShortRun + .WithLaunchCount(1) + .WithToolchain(InProcessNoEmitToolchain.Instance)) + .AddExporter(CsvExporter.Default, CsvMeasurementsExporter.Default, RPlotExporter.Default, HtmlExporter.Default, JsonExporter.Default, MarkdownExporter.GitHub, XmlExporter.Default) + .AddDiagnoser(MemoryDiagnoser.Default, ThreadingDiagnoser.Default, ExceptionDiagnoser.Default) + //.AddDiagnoser(new InliningDiagnoser(), new EtwProfiler()) // only windows platform, requires elevated privileges + //.AddDiagnoser(new EventPipeProfiler(EventPipeProfile.CpuSampling)) // stops collecting results ??? + ; + var summary = BenchmarkRunner.Run(new[]{ + BenchmarkConverter.TypeToBenchmarks( typeof(CoverageBenchmarks), config), + BenchmarkConverter.TypeToBenchmarks( typeof(InstrumenterBenchmarks), config), + BenchmarkConverter.TypeToBenchmarks( typeof(InstrumentationHelperBenchmarks), config), + }); + + // Use this to select benchmarks from the console and execute with additional options e.g. 'coverlet.core.benchmark.tests.exe --profiler EP' + //var summaries = BenchmarkSwitcher.FromAssembly(typeof(Program).Assembly).Run(args); + } + } +} diff --git a/test/coverlet.core.benchmark.tests/Properties/AssemblyInfo.cs b/test/coverlet.core.benchmark.tests/Properties/AssemblyInfo.cs new file mode 100644 index 000000000..0bd3d9a20 --- /dev/null +++ b/test/coverlet.core.benchmark.tests/Properties/AssemblyInfo.cs @@ -0,0 +1,6 @@ +// Copyright (c) Toni Solarin-Sodara +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using System.Reflection; + +[assembly: AssemblyKeyFile("coverlet.core.benchmark.tests.snk")] diff --git a/test/coverlet.core.benchmark.tests/coverlet.core.benchmark.tests.csproj b/test/coverlet.core.benchmark.tests/coverlet.core.benchmark.tests.csproj new file mode 100644 index 000000000..51f19275c --- /dev/null +++ b/test/coverlet.core.benchmark.tests/coverlet.core.benchmark.tests.csproj @@ -0,0 +1,46 @@ + + + net8.0 + Exe + AnyCPU + pdbonly + true + true + true + Release + false + $(NoWarn);CS0162 + + false + + + + + + + + + + + + + + + + + + + + Always + + + Always + + + Always + + + Always + + + \ No newline at end of file diff --git a/test/coverlet.core.benchmark.tests/coverlet.core.benchmark.tests.snk b/test/coverlet.core.benchmark.tests/coverlet.core.benchmark.tests.snk new file mode 100644 index 000000000..b65a0a48d Binary files /dev/null and b/test/coverlet.core.benchmark.tests/coverlet.core.benchmark.tests.snk differ diff --git a/test/coverlet.core.tests/TestAssets/coverlet.testsubject.dll b/test/coverlet.core.tests/TestAssets/coverlet.testsubject.dll new file mode 100644 index 000000000..85b377f99 Binary files /dev/null and b/test/coverlet.core.tests/TestAssets/coverlet.testsubject.dll differ diff --git a/test/coverlet.core.tests/TestAssets/coverlet.testsubject.pdb b/test/coverlet.core.tests/TestAssets/coverlet.testsubject.pdb new file mode 100644 index 000000000..07e4aa44d Binary files /dev/null and b/test/coverlet.core.tests/TestAssets/coverlet.testsubject.pdb differ