fix: Fix comparisons of absolute thresholds, add helpers for defining them (#32)

Fixes #31

Author: hassila

Plugins/BenchmarkTool/BenchmarkTool+Baselines.swift

@@ -234,10 +234,10 @@ extension BenchmarkBaseline: Equatable {
         var betterOrEqualForIdentifier = true
 
         for (lhsBenchmarkIdentifier, lhsBenchmarkResults) in lhs.results {
-/*            if printOutput {
-                print("Checking for threshold violations for `\(lhsBenchmarkIdentifier.target):\(lhsBenchmarkIdentifier.name)`.")
-            }
-*/
+            /*            if printOutput {
+                 print("Checking for threshold violations for `\(lhsBenchmarkIdentifier.target):\(lhsBenchmarkIdentifier.name)`.")
+             }
+             */
             for lhsBenchmarkResult in lhsBenchmarkResults {
                 if let rhsResults = rhs.results.first(where: { $0.key == lhsBenchmarkIdentifier }) {
                     if let rhsBenchmarkResult = rhsResults.value.first(where: { $0.metric == lhsBenchmarkResult.metric }) {

Sources/Benchmark/BenchmarkResult+Defaults.swift

@@ -16,8 +16,10 @@
 // swiftlint:disable identifier_name
 
 public extension BenchmarkResult {
-    typealias PercentileRelativeThresholds = [BenchmarkResult.Percentile: Double]
-    typealias PercentileAbsoluteThresholds = [BenchmarkResult.Percentile: Int]
+    typealias PercentileRelativeThreshold = Double
+    typealias PercentileAbsoluteThreshold = Int
+    typealias PercentileRelativeThresholds = [BenchmarkResult.Percentile: PercentileRelativeThreshold]
+    typealias PercentileAbsoluteThresholds = [BenchmarkResult.Percentile: PercentileAbsoluteThreshold]
 
     struct PercentileThresholds: Codable {
         public init(relative: BenchmarkResult.PercentileRelativeThresholds = .none,
@@ -117,3 +119,54 @@ public extension BenchmarkResult.PercentileThresholds {
         BenchmarkResult.PercentileThresholds()
     }
 }
+
+// Convenience functions for defining absolute thresholds
+public extension BenchmarkResult.PercentileAbsoluteThreshold {
+    static func hours(_ hours: Int) -> BenchmarkResult.PercentileAbsoluteThreshold {
+        hours * 1_000_000_000 * 60 * 60
+    }
+
+    static func minutes(_ minutes: Int) -> BenchmarkResult.PercentileAbsoluteThreshold {
+        minutes * 1_000_000_000 * 60
+    }
+
+    static func seconds(_ seconds: Int) -> BenchmarkResult.PercentileAbsoluteThreshold {
+        seconds * 1_000_000_000
+    }
+
+    static func milliseconds(_ milliseconds: Int) -> BenchmarkResult.PercentileAbsoluteThreshold {
+        milliseconds * 1_000_000
+    }
+
+    static func microseconds(_ microseconds: Int) -> BenchmarkResult.PercentileAbsoluteThreshold {
+        microseconds * 1_000
+    }
+
+    static func nanoseconds(_ value: Int) -> BenchmarkResult.PercentileAbsoluteThreshold {
+        value
+    }
+
+    static func nanoseconds(_ value: UInt) -> BenchmarkResult.PercentileAbsoluteThreshold {
+        Int(value)
+    }
+
+    static func giga(_ value: Int) -> BenchmarkResult.PercentileAbsoluteThreshold {
+        value * 1_000_000_000
+    }
+
+    static func mega(_ value: Int) -> BenchmarkResult.PercentileAbsoluteThreshold {
+        value * 1_000_000
+    }
+
+    static func kilo(_ value: Int) -> BenchmarkResult.PercentileAbsoluteThreshold {
+        value * 1_000
+    }
+
+    static func count(_ value: Int) -> BenchmarkResult.PercentileAbsoluteThreshold {
+        value
+    }
+
+    static func count(_ value: UInt) -> BenchmarkResult.PercentileAbsoluteThreshold {
+        Int(value)
+    }
+}

Sources/Benchmark/BenchmarkResult.swift

@@ -131,7 +131,7 @@ public struct BenchmarkResult: Codable, Comparable, Equatable {
 
         return allIsLess
     }
-
+// swiftlint:disable function_body_length
     public func betterResultsOrEqual(than otherResult: BenchmarkResult,
                                      thresholds: BenchmarkResult.PercentileThresholds = .default,
                                      printOutput: Bool = false) -> Bool {
@@ -146,11 +146,12 @@ public struct BenchmarkResult: Codable, Comparable, Equatable {
         }
 
         rhs.scaleResults(to: lhs)
-
+        // swiftlint:disable function_parameter_count
         func worseResult(_ lhs: Int,
                          _ rhs: Int,
                          _ percentile: BenchmarkResult.Percentile,
                          _ thresholds: BenchmarkResult.PercentileThresholds,
+                         _ scalingFactor: Int,
                          _ printOutput: Bool) -> Bool {
             let relativeDifference = (100 - (100.0 * Double(lhs) / Double(rhs)))
             let absoluteDifference = lhs - rhs
@@ -161,17 +162,18 @@ public struct BenchmarkResult: Codable, Comparable, Equatable {
             if let threshold = thresholds.relative[percentile] {
                 if reverseComparison ? relativeDifference > threshold : -relativeDifference > threshold {
                     if printOutput {
-                        print("`\(metric.description)` failed relative threshold check, [\(percentile)] result" +
+                        print("`\(metric.description)` relative threshold violated, [\(percentile)] result" +
                             " (\(roundToDecimalplaces(abs(relativeDifference), 1))) > threshold (\(threshold))")
                     }
                     thresholdViolated = true
                 }
             }
 
-            if let threshold = thresholds.absolute[percentile] {
+            if var threshold = thresholds.absolute[percentile] {
+                threshold = threshold / (1_000_000_000 / scalingFactor)
                 if reverseComparison ? -absoluteDifference > threshold : absoluteDifference > threshold {
                     if printOutput {
-                        print("`\(metric.description)` failed absolute threshold check, [\(percentile)] result" +
+                        print("`\(metric.description)` absolute threshold violated, [\(percentile)] result" +
                             " (\(abs(absoluteDifference))) > threshold (\(threshold))")
                     }
                     thresholdViolated = true
@@ -184,7 +186,12 @@ public struct BenchmarkResult: Codable, Comparable, Equatable {
 
         lhs.percentiles.forEach { percentile, lhsPercentile in
             if let rhsPercentile = rhs.percentiles[percentile] {
-                worse = worseResult(lhsPercentile, rhsPercentile, percentile, thresholds, printOutput) || worse
+                worse = worseResult(lhsPercentile,
+                                    rhsPercentile,
+                                    percentile,
+                                    thresholds,
+                                    lhs.timeUnits.rawValue,
+                                    printOutput) || worse
             } else {
                 print("\(rhs.metric) missing value for percentile \(percentile), skipping it.")
             }

Tests/BenchmarkTests/BenchmarkResultTests.swift

@@ -11,7 +11,7 @@
 @testable import BenchmarkSupport
 import XCTest
 
-// swiftlint: disable function_body_length
+// swiftlint:disable function_body_length
 final class BenchmarkResultTests: XCTestCase {
     func testBenchmarkResultEquality() throws {
         let measurementCount = 100

Tests/BenchmarkTests/BenchmarkRunnerTests.swift

@@ -28,7 +28,7 @@ final class BenchmarkRunnerTests: XCTestCase, BenchmarkRunnerReadWrite {
     private var readMessage: Int = 0
     private var writeCount: Int = 0
 
-    // swiftlint: disable test_case_accessibility
+    // swiftlint:disable test_case_accessibility
     internal func write(_: BenchmarkCommandReply) throws {
         writeCount += 1
 //        print("write \(reply)")