Allow filters to filter out every benchmark from a type (Fixes #2860) (#2879)

* Allow filters to filter out every benchmark from a type

* Add test for assembly with filtered benchmarks case

* Enable assembly with filtered benchmarks test on .NET Framework 4

Author: User0332

src/BenchmarkDotNet/Running/BenchmarkConverter.cs

@@ -59,6 +59,8 @@ private static BenchmarkRunInfo MethodsToBenchmarksWithFullConfig(Type type, Met
 
             var benchmarks = new List<BenchmarkCase>();
 
+            bool containsBenchmarkDeclarations = false;
+
             foreach (var target in targets)
             {
                 var argumentsDefinitions = GetArgumentsDefinitions(target.WorkloadMethod, target.Type, configPerType.SummaryStyle).ToArray();
@@ -75,12 +77,14 @@ from job in configPerMethod.GetJobs()
                     from parameterInstance in parameterInstances
                     select BenchmarkCase.Create(target, job, parameterInstance, configPerMethod);
 
+                if (benchmarksForTarget.Any() && !containsBenchmarkDeclarations) containsBenchmarkDeclarations = true;
+
                 benchmarks.AddRange(GetFilteredBenchmarks(benchmarksForTarget, configPerMethod.GetFilters()));
             }
 
             var orderedBenchmarks = configPerType.Orderer.GetExecutionOrder(benchmarks.ToImmutableArray()).ToArray();
 
-            return new BenchmarkRunInfo(orderedBenchmarks, type, configPerType);
+            return new BenchmarkRunInfo(orderedBenchmarks, type, configPerType, containsBenchmarkDeclarations);
         }
 
         private static ImmutableConfig GetFullTypeConfig(Type type, IConfig? config)

src/BenchmarkDotNet/Running/BenchmarkRunInfo.cs

@@ -3,14 +3,9 @@
 
 namespace BenchmarkDotNet.Running
 {
-    public class BenchmarkRunInfo : IDisposable
+    public class BenchmarkRunInfo(BenchmarkCase[] benchmarksCase, Type type, ImmutableConfig config, bool containsBenchmarkDeclarations) : IDisposable
     {
-        public BenchmarkRunInfo(BenchmarkCase[] benchmarksCase, Type type, ImmutableConfig config)
-        {
-            BenchmarksCases = benchmarksCase;
-            Type = type;
-            Config = config;
-        }
+        public BenchmarkRunInfo(BenchmarkCase[] benchmarksCase, Type type, ImmutableConfig config) : this(benchmarksCase, type, config, benchmarksCase.Length > 0) { }
 
         public void Dispose()
         {
@@ -20,8 +15,9 @@ public void Dispose()
             }
         }
 
-        public BenchmarkCase[] BenchmarksCases { get; }
-        public Type Type { get; }
-        public ImmutableConfig Config { get; }
+        public BenchmarkCase[] BenchmarksCases { get; } = benchmarksCase;
+        public Type Type { get; } = type;
+        public ImmutableConfig Config { get; } = config;
+        public bool ContainsBenchmarkDeclarations { get; } = containsBenchmarkDeclarations;
     }
 }

src/BenchmarkDotNet/Running/BenchmarkRunnerClean.cs

@@ -633,7 +633,7 @@ private static (BenchmarkRunInfo[], List<ValidationError>) GetSupportedBenchmark
 
             foreach (var benchmarkRunInfo in benchmarkRunInfos)
             {
-                if (benchmarkRunInfo.BenchmarksCases.Length == 0)
+                if (!benchmarkRunInfo.ContainsBenchmarkDeclarations)
                 {
                     validationErrors.Add(new ValidationError(true, $"No [Benchmark] attribute found on '{benchmarkRunInfo.Type.Name}' benchmark case."));
                     continue;

tests/BenchmarkDotNet.Tests/Running/RunningEmptyBenchmarkTests.cs

@@ -9,6 +9,7 @@
 using BenchmarkDotNet.Loggers;
 using BenchmarkDotNet.Reports;
 using System.Runtime.InteropServices;
+using BenchmarkDotNet.Filters;
 
 namespace BenchmarkDotNet.Tests.Running
 {
@@ -232,8 +233,8 @@ public void MixedTypes_ThrowsValidationError_WhenNoBenchmarkAttribute(string[]?
         #region Assembly Tests
         // In this tests there is no config and logger because the logger is initiated at CreateCompositeLogger when the BenchmarkRunInfo[] is empty
         // those cannot be inserted using config
-        [Theory]
 
+        [Theory]
         [InlineData(null)]
         [InlineData(new object[] { new string[] { " " } })]
         public void AssemblyWithoutBenchmarks_ThrowsValidationError_WhenNoBenchmarksFound(string[]? args)
@@ -308,6 +309,42 @@ public void AssemblyWithBenchmarks_RunsSuccessfully_WhenBenchmarkAttributePresen
                 Assert.DoesNotContain(summary.ValidationErrors, validationError => validationError.Message == GetGeneralValidationError());
             }
         }
+
+        [Fact]
+        public void AssemblyWithBenchmarks_RunsNothing_WhenAllBenchmarksFilteredOutFromOneTypeWithBenchmarkAttributePresent()
+        {
+            // Create a mock assembly with benchmark types
+            var assemblyName = new AssemblyName("MockAssemblyWithBenchmarks");
+            var assemblyBuilder = AssemblyBuilder.DefineDynamicAssembly(assemblyName, AssemblyBuilderAccess.Run);
+            var moduleBuilder = assemblyBuilder.DefineDynamicModule("MockModule");
+
+            // Create a benchmark type
+            var benchmarkTypeBuilder = moduleBuilder.DefineType("MockBenchmark", TypeAttributes.Public);
+            var benchmarkMethod = benchmarkTypeBuilder.DefineMethod("Benchmark", MethodAttributes.Public, typeof(void), Type.EmptyTypes);
+
+            // Generate method body
+            var ilGenerator = benchmarkMethod.GetILGenerator();
+            ilGenerator.Emit(OpCodes.Ret); // Just return from the method
+
+            var benchmarkAttributeCtor = typeof(BenchmarkAttribute).GetConstructor(new[] { typeof(int), typeof(string) });
+            if (benchmarkAttributeCtor == null)
+                throw new InvalidOperationException("Could not find BenchmarkAttribute constructor");
+            benchmarkMethod.SetCustomAttribute(new CustomAttributeBuilder(
+                benchmarkAttributeCtor,
+                new object[] { 0, "" }));
+            benchmarkTypeBuilder.CreateType();
+
+            Summary[] summaries = null;
+
+            GetConfigWithLogger(out var logger, out var config);
+
+            config.AddFilter(new NameFilter(name => name != "Benchmark")); // Filter out only benchmark method on MockBenchmark
+
+            summaries = BenchmarkRunner.Run(assemblyBuilder, config);
+            Assert.DoesNotContain(GetValidationErrorForType(benchmarkTypeBuilder), logger.GetLog());
+            Assert.Contains(GetExporterNoBenchmarksError(), logger.GetLog());
+        }
+
         #endregion
         #region Helper Methods
         private string GetValidationErrorForType(Type type)
@@ -330,6 +367,11 @@ private string GetGeneralValidationError()
             return $"No benchmarks were found.";
         }
 
+        private string GetExporterNoBenchmarksError()
+        {
+            return "There are no benchmarks found";
+        }
+
         private void GetConfigWithLogger(out AccumulationLogger logger, out ManualConfig manualConfig)
         {
             logger = new AccumulationLogger();