Add GenerateAndBuildOnly Option

Author: lihaMSFT

docs/articles/configs/configoptions.md

@@ -10,6 +10,7 @@ The config options let you customize some behavior of BenchmarkDotNet - mainly r
 Available config options are:
 
 * `ConfigOptions.Default` - No configuration option is set - this is the default.
+* `ConfigOptions.GenerateAndBuildOnly` - Only run toolchain generator and builder without executor. Then performs normal cleanup.
 * `ConfigOptions.KeepBenchmarkFiles` - All auto-generated files should be kept after running the benchmarks (by default they are removed).
 * `ConfigOptions.JoinSummary` - All benchmarks results should be joined into a single summary (by default we have a summary per type).
 * `ConfigOptions.StopOnFirstError` - Benchmarking should be stopped after the first error (by default it's not).

src/BenchmarkDotNet/Analysers/EnvironmentAnalyser.cs

@@ -1,4 +1,5 @@
 using System.Collections.Generic;
+using System.Collections.Immutable;
 using System.Linq;
 using System.Reflection;
 using System.Text;
@@ -8,6 +9,7 @@
 using BenchmarkDotNet.Reports;
 using BenchmarkDotNet.Toolchains.InProcess.Emit;
 using BenchmarkDotNet.Toolchains.InProcess.NoEmit;
+using BenchmarkDotNet.Toolchains.Results;
 
 namespace BenchmarkDotNet.Analysers
 {

src/BenchmarkDotNet/Attributes/GenerateAndBuildOnlyAttribute.cs

@@ -0,0 +1,21 @@
+using System;
+using BenchmarkDotNet.Configs;
+using JetBrains.Annotations;
+
+namespace BenchmarkDotNet.Attributes
+{
+    /// <summary>
+    /// determines if benchmark should be run after being generated and built
+    /// </summary>
+    [PublicAPI]
+    [AttributeUsage(AttributeTargets.Class)]
+    public class GenerateAndBuildOnlyAttribute : Attribute, IConfigSource
+    {
+        public IConfig Config { get; }
+
+        public GenerateAndBuildOnlyAttribute(bool value = true)
+        {
+            Config = ManualConfig.CreateEmpty().WithOption(ConfigOptions.GenerateAndBuildOnly, value);
+        }
+    }
+}

src/BenchmarkDotNet/Configs/ConfigExtensions.cs

@@ -80,6 +80,11 @@ public static class ConfigExtensions
         /// </summary>
         [PublicAPI] public static IConfig KeepBenchmarkFiles(this IConfig config, bool value = true) => config.WithOption(ConfigOptions.KeepBenchmarkFiles, value);
 
+        /// <summary>
+        /// determines if we skip executing after running generator and builder
+        /// </summary>
+        [PublicAPI] public static IConfig GenerateAndBuildOnly(this IConfig config, bool value = true) => config.WithOption(ConfigOptions.GenerateAndBuildOnly, value);
+
         /// <summary>
         /// determines if the exported result files should not be overwritten (be default they are overwritten)
         /// </summary>

src/BenchmarkDotNet/Configs/ConfigOptions.cs

@@ -48,7 +48,11 @@ public enum ConfigOptions
         /// <summary>
         /// Continue the execution if the last run was stopped.
         /// </summary>
-        Resume = 1 << 9
+        Resume = 1 << 9,
+        /// <summary>
+        /// Don't run the benchmark after it's generated and built.
+        /// </summary>
+        GenerateAndBuildOnly = 1 << 10
     }
 
     internal static class ConfigOptionsExtensions

src/BenchmarkDotNet/ConsoleArguments/CommandLineOptions.cs

@@ -86,6 +86,9 @@ public bool UseDisassemblyDiagnoser
         [Option("keepFiles", Required = false, Default = false, HelpText = "Determines if all auto-generated files should be kept or removed after running the benchmarks.")]
         public bool KeepBenchmarkFiles { get; set; }
 
+        [Option("generateAndBuildOnly", Required = false, Default = false, HelpText = "Determines if we only run generator and builder without executor.")]
+        public bool GenerateAndBuildOnly { get; set; }
+
         [Option("noOverwrite", Required = false, Default = false, HelpText = "Determines if the exported result files should not be overwritten (be default they are overwritten).")]
         public bool DontOverwriteResults { get; set; }
 

src/BenchmarkDotNet/ConsoleArguments/ConfigParser.cs

@@ -242,6 +242,7 @@ private static IConfig CreateConfig(CommandLineOptions options, IConfig globalCo
 
             config.WithOption(ConfigOptions.JoinSummary, options.Join);
             config.WithOption(ConfigOptions.KeepBenchmarkFiles, options.KeepBenchmarkFiles);
+            config.WithOption(ConfigOptions.GenerateAndBuildOnly, options.GenerateAndBuildOnly);
             config.WithOption(ConfigOptions.DontOverwriteResults, options.DontOverwriteResults);
             config.WithOption(ConfigOptions.StopOnFirstError, options.StopOnFirstError);
             config.WithOption(ConfigOptions.DisableLogFile, options.DisableLogFile);

src/BenchmarkDotNet/Reports/BenchmarkReport.cs

@@ -47,6 +47,16 @@ public BenchmarkReport(
                 ?? (IReadOnlyDictionary<string, Metric>)ImmutableDictionary<string, Metric>.Empty;
         }
 
+        public BenchmarkReport(BenchmarkCase benchmarkCase, GenerateResult generateResult, BuildResult buildResult)
+        {
+            BenchmarkCase = benchmarkCase;
+            GenerateResult = generateResult;
+            BuildResult = buildResult;
+            ExecuteResults = Array.Empty<ExecuteResult>();
+            AllMeasurements = ImmutableList<Measurement>.Empty;
+            Metrics = ImmutableDictionary<string, Metric>.Empty;
+        }
+
         public override string ToString() => $"{BenchmarkCase.DisplayInfo}, {AllMeasurements.Count} runs";
 
         public IReadOnlyList<Measurement> GetResultRuns() => AllMeasurements.Where(r => r.Is(IterationMode.Workload, IterationStage.Result)).ToList();

src/BenchmarkDotNet/Running/BenchmarkRunnerClean.cs

@@ -196,7 +196,15 @@ private static Summary Run(BenchmarkRunInfo benchmarkRunInfo,
                         if (!config.Options.IsSet(ConfigOptions.KeepBenchmarkFiles))
                             artifactsToCleanup.AddRange(buildResult.ArtifactsToCleanup);
 
-                        var report = RunCore(benchmark, info.benchmarkId, logger, resolver, buildResult);
+                        BenchmarkReport report;
+                        if (config.Options.IsSet(ConfigOptions.GenerateAndBuildOnly))
+                        {
+                            report = new BenchmarkReport(benchmark, buildResult, buildResult);
+                            reports.Add(report);
+                            continue;
+                        }
+
+                        report = RunCore(benchmark, info.benchmarkId, logger, resolver, buildResult);
                         if (report.AllMeasurements.Any(m => m.Operations == 0))
                             throw new InvalidOperationException("An iteration with 'Operations == 0' detected");
                         reports.Add(report);