Skip to content

Commit

Permalink
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
specjbb parser parses N/A metrics gracefully
Browse files Browse the repository at this point in the history
nmalkapuram committed Jul 16, 2024
1 parent fcd1ea5 commit 78fba99
Showing 9 changed files with 325 additions and 30 deletions.

Large diffs are not rendered by default.

Original file line number Diff line number Diff line change
@@ -245,8 +245,8 @@ public void FioResultsParserReadsTheExpectedMeasurementValuesFromTheResults_Read
Assert.IsNotNull(metrics);
Assert.IsNotEmpty(metrics);

IEnumerable<Tuple<string, double>> expectedValues = FioMetricsParserTests.ExpectedReadMetrics.Select(entry => new Tuple<string, double>(entry.Key, entry.Value));
IEnumerable<Tuple<string, double>> actualValues = metrics.Select(m => new Tuple<string, double>(m.Name, m.Value));
IEnumerable<Tuple<string, double?>> expectedValues = FioMetricsParserTests.ExpectedReadMetrics.Select(entry => new Tuple<string, double?>(entry.Key, entry.Value));
IEnumerable<Tuple<string, double?>> actualValues = metrics.Select(m => new Tuple<string, double?>(m.Name, m.Value));

CollectionAssert.AreEquivalent(expectedValues, actualValues);
}
@@ -262,8 +262,8 @@ public void FioResultsParserReadsTheExpectedMeasurementValuesFromTheResults_Writ
Assert.IsNotNull(metrics);
Assert.IsNotEmpty(metrics);

IEnumerable<Tuple<string, double>> expectedValues = FioMetricsParserTests.ExpectedWriteMetrics.Select(entry => new Tuple<string, double>(entry.Key, entry.Value));
IEnumerable<Tuple<string, double>> actualValues = metrics.Select(m => new Tuple<string, double>(m.Name, m.Value));
IEnumerable<Tuple<string, double?>> expectedValues = FioMetricsParserTests.ExpectedWriteMetrics.Select(entry => new Tuple<string, double?>(entry.Key, entry.Value));
IEnumerable<Tuple<string, double?>> actualValues = metrics.Select(m => new Tuple<string, double?>(m.Name, m.Value));

CollectionAssert.AreEquivalent(expectedValues, actualValues);
}
@@ -282,11 +282,11 @@ public void FioResultsParserAppliesTheExpectedDefaultConversionFactorToMeasureme

// All latency metrics are emitted in milliseconds form (vs. nanoseconds).
double expectedConversionFactor = 0.000001;
IEnumerable<Tuple<string, double>> expectedValues = FioMetricsParserTests.ExpectedReadMetrics.Where(entry => entry.Key.Contains("latency"))
.Select(entry => new Tuple<string, double>(entry.Key, entry.Value * expectedConversionFactor));
IEnumerable<Tuple<string, double?>> expectedValues = FioMetricsParserTests.ExpectedReadMetrics.Where(entry => entry.Key.Contains("latency"))
.Select(entry => new Tuple<string, double?>(entry.Key, entry.Value * expectedConversionFactor));

IEnumerable<Tuple<string, double>> actualValues = metrics.Where(m => m.Name.Contains("latency"))
.Select(m => new Tuple<string, double>(m.Name, m.Value));
IEnumerable<Tuple<string, double?>> actualValues = metrics.Where(m => m.Name.Contains("latency"))
.Select(m => new Tuple<string, double?>(m.Name, m.Value));

Assert.IsTrue(expectedValues.Count() == actualValues.Count());
CollectionAssert.AreEquivalent(expectedValues, actualValues);
@@ -306,11 +306,11 @@ public void FioResultsParserAppliesTheExpectedDefaultConversionFactorToMeasureme

// All latency metrics are emitted in milliseconds form (vs. nanoseconds).
double expectedConversionFactor = 0.000001;
IEnumerable<Tuple<string, double>> expectedValues = FioMetricsParserTests.ExpectedWriteMetrics.Where(entry => entry.Key.Contains("latency"))
.Select(entry => new Tuple<string, double>(entry.Key, entry.Value * expectedConversionFactor));
IEnumerable<Tuple<string, double?>> expectedValues = FioMetricsParserTests.ExpectedWriteMetrics.Where(entry => entry.Key.Contains("latency"))
.Select(entry => new Tuple<string, double?>(entry.Key, entry.Value * expectedConversionFactor));

IEnumerable<Tuple<string, double>> actualValues = metrics.Where(m => m.Name.Contains("latency"))
.Select(m => new Tuple<string, double>(m.Name, m.Value));
IEnumerable<Tuple<string, double?>> actualValues = metrics.Where(m => m.Name.Contains("latency"))
.Select(m => new Tuple<string, double?>(m.Name, m.Value));

Assert.IsTrue(expectedValues.Count() == actualValues.Count());
CollectionAssert.AreEquivalent(expectedValues, actualValues);
Original file line number Diff line number Diff line change
@@ -40,5 +40,22 @@ public void SpecJbbParserVerifyMetricsFpRate()
MetricAssert.Exists(metrics, "max-jOPS", 4188, "jOPS");
MetricAssert.Exists(metrics, "critical-jOPS", 1666, "jOPS");
}


[Test]
public void SpecJbbParserVerifyNanMetricsFpRate()
{
string workingDirectory = Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location);
string outputPath = Path.Combine(workingDirectory, "Examples", "SPECjbb", "specjbbNanOutput1.txt");
this.rawText = File.ReadAllText(outputPath);
this.testParser = new SpecJbbMetricsParser(this.rawText);
IList<Metric> metrics = this.testParser.Parse();

Assert.AreEqual(4, metrics.Count);
MetricAssert.Exists(metrics, "hbIR (max attempted)", 304872, "jOPS");
MetricAssert.Exists(metrics, "hbIR (settled)", null, "jOPS");
MetricAssert.Exists(metrics, "max-jOPS", 234751, "jOPS");
MetricAssert.Exists(metrics, "critical-jOPS", null , "jOPS");
}
}
}
Original file line number Diff line number Diff line change
@@ -344,9 +344,9 @@ private Task ExecuteWorkloadAsync(EventContext telemetryContext, CancellationTok
private IList<Metric> CalculateTimeSpyAggregates(IList<Metric> metrics)
{
IList<Metric> aggregates = new List<Metric>();
double tsgt1 = 0;
double tsgt2 = 0;
double tsct = 0;
double? tsgt1 = 0;
double? tsgt2 = 0;
double? tsct = 0;
foreach (Metric metric in metrics)
{
if (metric.Name == "timespy.graphics.1")
@@ -366,9 +366,9 @@ private IList<Metric> CalculateTimeSpyAggregates(IList<Metric> metrics)
// Weighted Harmonic Mean of Individual Scores
if (tsgt1 != 0 && tsgt2 != 0 && tsct != 0)
{
double graphicsScore = 165 * (2 / ((1 / tsgt1) + (1 / tsgt2)));
double cpuScore = 298 * tsct;
double aggScore = 1 / ((0.85 / graphicsScore) + (0.15 / cpuScore));
double? graphicsScore = 165 * (2 / ((1 / tsgt1) + (1 / tsgt2)));
double? cpuScore = 298 * tsct;
double? aggScore = 1 / ((0.85 / graphicsScore) + (0.15 / cpuScore));
aggregates.Add(new Metric("timespy.graphics.agg", graphicsScore, "score", MetricRelativity.HigherIsBetter));
aggregates.Add(new Metric("timespy.cpu.agg", cpuScore, "score", MetricRelativity.HigherIsBetter));
aggregates.Add(new Metric("timespy.finalscore", aggScore, "score", MetricRelativity.HigherIsBetter));
Original file line number Diff line number Diff line change
@@ -43,7 +43,8 @@ public override IList<Metric> Parse()
string[] tokens = metric.Split("=");
string name = tokens[0];
string value = tokens[1];
this.Metrics.Add(new Metric(name.Trim(), Convert.ToDouble(value), SpecJbbMetricsParser.OperationPerSecond, MetricRelativity.HigherIsBetter));

this.Metrics.Add(new Metric(name.Trim(), (value.Trim().Equals("N/A", StringComparison.OrdinalIgnoreCase)) ? null : Convert.ToDouble(value), SpecJbbMetricsParser.OperationPerSecond, MetricRelativity.HigherIsBetter));
}

return this.Metrics;
12 changes: 6 additions & 6 deletions src/VirtualClient/VirtualClient.Contracts/Metric.cs
Original file line number Diff line number Diff line change
@@ -18,7 +18,7 @@ public class Metric : IEquatable<Metric>
/// <summary>
/// Creates a metric
/// </summary>
public Metric(string name, double value)
public Metric(string name, double? value)
{
this.Name = name;
this.Value = value;
@@ -30,7 +30,7 @@ public Metric(string name, double value)
/// <summary>
/// Creates a metric
/// </summary>
public Metric(string name, double value, IEnumerable<string> tags = null, string description = null, IDictionary<string, IConvertible> metadata = null)
public Metric(string name, double? value, IEnumerable<string> tags = null, string description = null, IDictionary<string, IConvertible> metadata = null)
: this(name, value)
{
this.Description = description;
@@ -51,7 +51,7 @@ public Metric(string name, double value, IEnumerable<string> tags = null, string
/// <summary>
/// Creates a metric
/// </summary>
public Metric(string name, double value, MetricRelativity relativity, IEnumerable<string> tags = null, string description = null, IDictionary<string, IConvertible> metadata = null)
public Metric(string name, double? value, MetricRelativity relativity, IEnumerable<string> tags = null, string description = null, IDictionary<string, IConvertible> metadata = null)
: this(name, value, tags: tags, description: description, metadata: metadata)
{
this.Relativity = relativity;
@@ -60,7 +60,7 @@ public Metric(string name, double value, MetricRelativity relativity, IEnumerabl
/// <summary>
/// Creates a metric
/// </summary>
public Metric(string name, double value, string unit, IEnumerable<string> tags = null, string description = null, IDictionary<string, IConvertible> metadata = null)
public Metric(string name, double? value, string unit, IEnumerable<string> tags = null, string description = null, IDictionary<string, IConvertible> metadata = null)
: this(name, value, tags: tags, description: description, metadata: metadata)
{
this.Unit = unit;
@@ -69,7 +69,7 @@ public Metric(string name, double value, string unit, IEnumerable<string> tags =
/// <summary>
/// Creates a metric
/// </summary>
public Metric(string name, double value, string unit, MetricRelativity relativity, IEnumerable<string> tags = null, string description = null, IDictionary<string, IConvertible> metadata = null)
public Metric(string name, double? value, string unit, MetricRelativity relativity, IEnumerable<string> tags = null, string description = null, IDictionary<string, IConvertible> metadata = null)
: this(name, value, unit, tags: tags, description: description, metadata: metadata)
{
this.Relativity = relativity;
@@ -99,7 +99,7 @@ public Metric(string name, double value, string unit, MetricRelativity relativit
/// <summary>
/// Result of test
/// </summary>
public double Value { get; }
public double? Value { get; }

/// <summary>
/// Unit of result
6 changes: 3 additions & 3 deletions src/VirtualClient/VirtualClient.Contracts/MetricAggregate.cs
Original file line number Diff line number Diff line change
@@ -12,7 +12,7 @@ namespace VirtualClient.Contracts
/// Provides features for capturing Windows performance counter values over a period
/// of time.
/// </summary>
public class MetricAggregate : ConcurrentBag<double>
public class MetricAggregate : ConcurrentBag<double?>
{
/// <summary>
/// Initializes a new instance of the <see cref="MetricAggregate"/> class.
@@ -115,11 +115,11 @@ public Metric ToMetric(MetricAggregateType aggregateType)
return Metric.None;
}

double value = 0;
double? value = 0;
switch (aggregateType)
{
case MetricAggregateType.Average:
double sum = this.Sum();
double? sum = this.Sum();
value = sum / this.Count;
break;

Original file line number Diff line number Diff line change
@@ -744,7 +744,7 @@ public static void LogMetrics(
DateTime scenarioStartTime,
DateTime scenarioEndTime,
string metricName,
double metricValue,
double? metricValue,
string metricUnits,
string metricCategorization,
string scenarioArguments,
Original file line number Diff line number Diff line change
@@ -27,7 +27,7 @@ public static void Exists(
/// <summary>
/// Asserts the metric exists in the given list of results
/// </summary>
public static void Exists(IList<Metric> results, string expectedMetric, double expectedMetricValue, string expectedMetricUnit = null, List<string> expectedTags = null)
public static void Exists(IList<Metric> results, string expectedMetric, double? expectedMetricValue, string expectedMetricUnit = null, List<string> expectedTags = null)
{
List<Metric> matchingMetrics = results.Where(m => m.Name == expectedMetric).ToList();

0 comments on commit 78fba99

Please sign in to comment.