From b96bfcf620b0453497b9521e241c74ac63739a0b Mon Sep 17 00:00:00 2001 From: nikhitha malkapuram Date: Mon, 20 Jan 2025 21:29:16 +0530 Subject: [PATCH] not emitting N/A values for now.Adding a metric that shows Metric is N/A explicitly. Actual resolution is fixing specjbb process for not emitting N/A values --- .../FIO/FioMetricsParserTests.cs | 24 +++++++++---------- .../SPEC/SpecJbbMetricsParserTests.cs | 4 ++-- .../3DMark/ThreeDMarkExecutor.cs | 12 +++++----- .../SPECjbb/SpecJbbMetricsParser.cs | 9 ++++++- .../VirtualClient.Contracts/Metric.cs | 12 +++++----- .../MetricAggregate.cs | 6 ++--- .../VirtualClientLoggingExtensions.cs | 2 +- .../MetricAssert.cs | 2 +- 8 files changed, 39 insertions(+), 32 deletions(-) diff --git a/src/VirtualClient/VirtualClient.Actions.UnitTests/FIO/FioMetricsParserTests.cs b/src/VirtualClient/VirtualClient.Actions.UnitTests/FIO/FioMetricsParserTests.cs index 5f4b639bc6..1070c3b1a3 100644 --- a/src/VirtualClient/VirtualClient.Actions.UnitTests/FIO/FioMetricsParserTests.cs +++ b/src/VirtualClient/VirtualClient.Actions.UnitTests/FIO/FioMetricsParserTests.cs @@ -245,8 +245,8 @@ public void FioResultsParserReadsTheExpectedMeasurementValuesFromTheResults_Read Assert.IsNotNull(metrics); Assert.IsNotEmpty(metrics); - IEnumerable> expectedValues = FioMetricsParserTests.ExpectedReadMetrics.Select(entry => new Tuple(entry.Key, entry.Value)); - IEnumerable> actualValues = metrics.Select(m => new Tuple(m.Name, m.Value)); + IEnumerable> expectedValues = FioMetricsParserTests.ExpectedReadMetrics.Select(entry => new Tuple(entry.Key, entry.Value)); + IEnumerable> actualValues = metrics.Select(m => new Tuple(m.Name, m.Value)); CollectionAssert.AreEquivalent(expectedValues, actualValues); } @@ -262,8 +262,8 @@ public void FioResultsParserReadsTheExpectedMeasurementValuesFromTheResults_Writ Assert.IsNotNull(metrics); Assert.IsNotEmpty(metrics); - IEnumerable> expectedValues = FioMetricsParserTests.ExpectedWriteMetrics.Select(entry => new Tuple(entry.Key, entry.Value)); - IEnumerable> actualValues = metrics.Select(m => new Tuple(m.Name, m.Value)); + IEnumerable> expectedValues = FioMetricsParserTests.ExpectedWriteMetrics.Select(entry => new Tuple(entry.Key, entry.Value)); + IEnumerable> actualValues = metrics.Select(m => new Tuple(m.Name, m.Value)); CollectionAssert.AreEquivalent(expectedValues, actualValues); } @@ -282,11 +282,11 @@ public void FioResultsParserAppliesTheExpectedDefaultConversionFactorToMeasureme // All latency metrics are emitted in milliseconds form (vs. nanoseconds). double expectedConversionFactor = 0.000001; - IEnumerable> expectedValues = FioMetricsParserTests.ExpectedReadMetrics.Where(entry => entry.Key.Contains("latency")) - .Select(entry => new Tuple(entry.Key, entry.Value * expectedConversionFactor)); + IEnumerable> expectedValues = FioMetricsParserTests.ExpectedReadMetrics.Where(entry => entry.Key.Contains("latency")) + .Select(entry => new Tuple(entry.Key, entry.Value * expectedConversionFactor)); - IEnumerable> actualValues = metrics.Where(m => m.Name.Contains("latency")) - .Select(m => new Tuple(m.Name, m.Value)); + IEnumerable> actualValues = metrics.Where(m => m.Name.Contains("latency")) + .Select(m => new Tuple(m.Name, m.Value)); Assert.IsTrue(expectedValues.Count() == actualValues.Count()); CollectionAssert.AreEquivalent(expectedValues, actualValues); @@ -306,11 +306,11 @@ public void FioResultsParserAppliesTheExpectedDefaultConversionFactorToMeasureme // All latency metrics are emitted in milliseconds form (vs. nanoseconds). double expectedConversionFactor = 0.000001; - IEnumerable> expectedValues = FioMetricsParserTests.ExpectedWriteMetrics.Where(entry => entry.Key.Contains("latency")) - .Select(entry => new Tuple(entry.Key, entry.Value * expectedConversionFactor)); + IEnumerable> expectedValues = FioMetricsParserTests.ExpectedWriteMetrics.Where(entry => entry.Key.Contains("latency")) + .Select(entry => new Tuple(entry.Key, entry.Value * expectedConversionFactor)); - IEnumerable> actualValues = metrics.Where(m => m.Name.Contains("latency")) - .Select(m => new Tuple(m.Name, m.Value)); + IEnumerable> actualValues = metrics.Where(m => m.Name.Contains("latency")) + .Select(m => new Tuple(m.Name, m.Value)); Assert.IsTrue(expectedValues.Count() == actualValues.Count()); CollectionAssert.AreEquivalent(expectedValues, actualValues); diff --git a/src/VirtualClient/VirtualClient.Actions.UnitTests/SPEC/SpecJbbMetricsParserTests.cs b/src/VirtualClient/VirtualClient.Actions.UnitTests/SPEC/SpecJbbMetricsParserTests.cs index 04126857fb..428c43a615 100644 --- a/src/VirtualClient/VirtualClient.Actions.UnitTests/SPEC/SpecJbbMetricsParserTests.cs +++ b/src/VirtualClient/VirtualClient.Actions.UnitTests/SPEC/SpecJbbMetricsParserTests.cs @@ -53,9 +53,9 @@ public void SpecJbbParserVerifyNanMetricsFpRate() Assert.AreEqual(4, metrics.Count); MetricAssert.Exists(metrics, "hbIR (max attempted)", 304872, "jOPS"); - MetricAssert.Exists(metrics, "hbIR (settled)", null, "jOPS"); + MetricAssert.Exists(metrics, "hbIR (settled)_N/A", 1, "jOPS"); MetricAssert.Exists(metrics, "max-jOPS", 234751, "jOPS"); - MetricAssert.Exists(metrics, "critical-jOPS", null , "jOPS"); + MetricAssert.Exists(metrics, "critical-jOPS_N/A", 1 , "jOPS"); } } } \ No newline at end of file diff --git a/src/VirtualClient/VirtualClient.Actions/3DMark/ThreeDMarkExecutor.cs b/src/VirtualClient/VirtualClient.Actions/3DMark/ThreeDMarkExecutor.cs index 6ad04db23f..c0dce1ed56 100644 --- a/src/VirtualClient/VirtualClient.Actions/3DMark/ThreeDMarkExecutor.cs +++ b/src/VirtualClient/VirtualClient.Actions/3DMark/ThreeDMarkExecutor.cs @@ -324,9 +324,9 @@ private Task ExecuteWorkloadAsync(EventContext telemetryContext, CancellationTok private IList CalculateTimeSpyAggregates(IList metrics) { IList aggregates = new List(); - double? tsgt1 = 0; - double? tsgt2 = 0; - double? tsct = 0; + double tsgt1 = 0; + double tsgt2 = 0; + double tsct = 0; foreach (Metric metric in metrics) { if (metric.Name == "timespy.graphics.1") @@ -346,9 +346,9 @@ private IList CalculateTimeSpyAggregates(IList metrics) // Weighted Harmonic Mean of Individual Scores if (tsgt1 != 0 && tsgt2 != 0 && tsct != 0) { - double? graphicsScore = 165 * (2 / ((1 / tsgt1) + (1 / tsgt2))); - double? cpuScore = 298 * tsct; - double? aggScore = 1 / ((0.85 / graphicsScore) + (0.15 / cpuScore)); + double graphicsScore = 165 * (2 / ((1 / tsgt1) + (1 / tsgt2))); + double cpuScore = 298 * tsct; + double aggScore = 1 / ((0.85 / graphicsScore) + (0.15 / cpuScore)); aggregates.Add(new Metric("timespy.graphics.agg", graphicsScore, "score", MetricRelativity.HigherIsBetter)); aggregates.Add(new Metric("timespy.cpu.agg", cpuScore, "score", MetricRelativity.HigherIsBetter)); aggregates.Add(new Metric("timespy.finalscore", aggScore, "score", MetricRelativity.HigherIsBetter)); diff --git a/src/VirtualClient/VirtualClient.Actions/SPECjbb/SpecJbbMetricsParser.cs b/src/VirtualClient/VirtualClient.Actions/SPECjbb/SpecJbbMetricsParser.cs index 47e4990b76..050e0b4aae 100644 --- a/src/VirtualClient/VirtualClient.Actions/SPECjbb/SpecJbbMetricsParser.cs +++ b/src/VirtualClient/VirtualClient.Actions/SPECjbb/SpecJbbMetricsParser.cs @@ -43,7 +43,14 @@ public override IList Parse() string name = tokens[0]; string value = tokens[1]; - this.Metrics.Add(new Metric(name.Trim(), (value.Trim().Equals("N/A", StringComparison.OrdinalIgnoreCase)) ? null : Convert.ToDouble(value), SpecJbbMetricsParser.OperationPerSecond, MetricRelativity.HigherIsBetter)); + if (value.Trim().Equals("N/A", StringComparison.OrdinalIgnoreCase)) + { + this.metrics.Add(new Metric($"{name.Trim()}_N/A", 1, SpecJbbMetricsParser.OperationPerSecond, MetricRelativity.LowerIsBetter)); + } + else + { + this.metrics.Add(new Metric(name.Trim(), Convert.ToDouble(value), SpecJbbMetricsParser.OperationPerSecond, MetricRelativity.HigherIsBetter)); + } } return this.metrics; diff --git a/src/VirtualClient/VirtualClient.Contracts/Metric.cs b/src/VirtualClient/VirtualClient.Contracts/Metric.cs index 3181ca6d3a..27d54bfae1 100644 --- a/src/VirtualClient/VirtualClient.Contracts/Metric.cs +++ b/src/VirtualClient/VirtualClient.Contracts/Metric.cs @@ -18,7 +18,7 @@ public class Metric : IEquatable /// /// Creates a metric /// - public Metric(string name, double? value) + public Metric(string name, double value) { this.Name = name; this.Value = value; @@ -30,7 +30,7 @@ public Metric(string name, double? value) /// /// Creates a metric /// - public Metric(string name, double? value, IEnumerable tags = null, string description = null, IDictionary metadata = null) + public Metric(string name, double value, IEnumerable tags = null, string description = null, IDictionary metadata = null) : this(name, value) { this.Description = description; @@ -51,7 +51,7 @@ public Metric(string name, double? value, IEnumerable tags = null, strin /// /// Creates a metric /// - public Metric(string name, double? value, MetricRelativity relativity, IEnumerable tags = null, string description = null, IDictionary metadata = null) + public Metric(string name, double value, MetricRelativity relativity, IEnumerable tags = null, string description = null, IDictionary metadata = null) : this(name, value, tags: tags, description: description, metadata: metadata) { this.Relativity = relativity; @@ -60,7 +60,7 @@ public Metric(string name, double? value, MetricRelativity relativity, IEnumerab /// /// Creates a metric /// - public Metric(string name, double? value, string unit, IEnumerable tags = null, string description = null, IDictionary metadata = null) + public Metric(string name, double value, string unit, IEnumerable tags = null, string description = null, IDictionary metadata = null) : this(name, value, tags: tags, description: description, metadata: metadata) { this.Unit = unit; @@ -69,7 +69,7 @@ public Metric(string name, double? value, string unit, IEnumerable tags /// /// Creates a metric /// - public Metric(string name, double? value, string unit, MetricRelativity relativity, IEnumerable tags = null, string description = null, IDictionary metadata = null) + public Metric(string name, double value, string unit, MetricRelativity relativity, IEnumerable tags = null, string description = null, IDictionary metadata = null) : this(name, value, unit, tags: tags, description: description, metadata: metadata) { this.Relativity = relativity; @@ -108,7 +108,7 @@ public Metric(string name, double value, string unit, MetricRelativity relativit /// /// Result of test /// - public double? Value { get; } + public double Value { get; } /// /// Unit of result diff --git a/src/VirtualClient/VirtualClient.Contracts/MetricAggregate.cs b/src/VirtualClient/VirtualClient.Contracts/MetricAggregate.cs index c328fc03ff..788f71558e 100644 --- a/src/VirtualClient/VirtualClient.Contracts/MetricAggregate.cs +++ b/src/VirtualClient/VirtualClient.Contracts/MetricAggregate.cs @@ -12,7 +12,7 @@ namespace VirtualClient.Contracts /// Provides features for capturing Windows performance counter values over a period /// of time. /// - public class MetricAggregate : ConcurrentBag + public class MetricAggregate : ConcurrentBag { /// /// Initializes a new instance of the class. @@ -115,11 +115,11 @@ public Metric ToMetric(MetricAggregateType aggregateType) return Metric.None; } - double? value = 0; + double value = 0; switch (aggregateType) { case MetricAggregateType.Average: - double? sum = this.Sum(); + double sum = this.Sum(); value = sum / this.Count; break; diff --git a/src/VirtualClient/VirtualClient.Contracts/VirtualClientLoggingExtensions.cs b/src/VirtualClient/VirtualClient.Contracts/VirtualClientLoggingExtensions.cs index a560e65489..9f82cd0928 100644 --- a/src/VirtualClient/VirtualClient.Contracts/VirtualClientLoggingExtensions.cs +++ b/src/VirtualClient/VirtualClient.Contracts/VirtualClientLoggingExtensions.cs @@ -745,7 +745,7 @@ public static void LogMetrics( DateTime scenarioStartTime, DateTime scenarioEndTime, string metricName, - double? metricValue, + double metricValue, string metricUnits, string metricCategorization, string scenarioArguments, diff --git a/src/VirtualClient/VirtualClient.TestExtensions/MetricAssert.cs b/src/VirtualClient/VirtualClient.TestExtensions/MetricAssert.cs index 85011d4ae1..fe0c7f827a 100644 --- a/src/VirtualClient/VirtualClient.TestExtensions/MetricAssert.cs +++ b/src/VirtualClient/VirtualClient.TestExtensions/MetricAssert.cs @@ -27,7 +27,7 @@ public static void Exists( /// /// Asserts the metric exists in the given list of results /// - public static void Exists(IList results, string expectedMetric, double? expectedMetricValue, string expectedMetricUnit = null, List expectedTags = null) + public static void Exists(IList results, string expectedMetric, double expectedMetricValue, string expectedMetricUnit = null, List expectedTags = null) { List matchingMetrics = results.Where(m => m.Name == expectedMetric).ToList();