Skip to content

Commit

Permalink
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
specjbb parser parses N/A metrics gracefully
Browse files Browse the repository at this point in the history
nmalkapuram committed Jul 16, 2024

Verified

This commit was created on GitHub.com and signed with GitHub’s verified signature.
1 parent fcd1ea5 commit 78fba99
Showing 9 changed files with 325 additions and 30 deletions.

Large diffs are not rendered by default.

Original file line number Diff line number Diff line change
@@ -245,8 +245,8 @@ public void FioResultsParserReadsTheExpectedMeasurementValuesFromTheResults_Read
Assert.IsNotNull(metrics);
Assert.IsNotEmpty(metrics);

IEnumerable<Tuple<string, double>> expectedValues = FioMetricsParserTests.ExpectedReadMetrics.Select(entry => new Tuple<string, double>(entry.Key, entry.Value));
IEnumerable<Tuple<string, double>> actualValues = metrics.Select(m => new Tuple<string, double>(m.Name, m.Value));
IEnumerable<Tuple<string, double?>> expectedValues = FioMetricsParserTests.ExpectedReadMetrics.Select(entry => new Tuple<string, double?>(entry.Key, entry.Value));
IEnumerable<Tuple<string, double?>> actualValues = metrics.Select(m => new Tuple<string, double?>(m.Name, m.Value));

CollectionAssert.AreEquivalent(expectedValues, actualValues);
}
@@ -262,8 +262,8 @@ public void FioResultsParserReadsTheExpectedMeasurementValuesFromTheResults_Writ
Assert.IsNotNull(metrics);
Assert.IsNotEmpty(metrics);

IEnumerable<Tuple<string, double>> expectedValues = FioMetricsParserTests.ExpectedWriteMetrics.Select(entry => new Tuple<string, double>(entry.Key, entry.Value));
IEnumerable<Tuple<string, double>> actualValues = metrics.Select(m => new Tuple<string, double>(m.Name, m.Value));
IEnumerable<Tuple<string, double?>> expectedValues = FioMetricsParserTests.ExpectedWriteMetrics.Select(entry => new Tuple<string, double?>(entry.Key, entry.Value));
IEnumerable<Tuple<string, double?>> actualValues = metrics.Select(m => new Tuple<string, double?>(m.Name, m.Value));

CollectionAssert.AreEquivalent(expectedValues, actualValues);
}
@@ -282,11 +282,11 @@ public void FioResultsParserAppliesTheExpectedDefaultConversionFactorToMeasureme

// All latency metrics are emitted in milliseconds form (vs. nanoseconds).
double expectedConversionFactor = 0.000001;
IEnumerable<Tuple<string, double>> expectedValues = FioMetricsParserTests.ExpectedReadMetrics.Where(entry => entry.Key.Contains("latency"))
.Select(entry => new Tuple<string, double>(entry.Key, entry.Value * expectedConversionFactor));
IEnumerable<Tuple<string, double?>> expectedValues = FioMetricsParserTests.ExpectedReadMetrics.Where(entry => entry.Key.Contains("latency"))
.Select(entry => new Tuple<string, double?>(entry.Key, entry.Value * expectedConversionFactor));

IEnumerable<Tuple<string, double>> actualValues = metrics.Where(m => m.Name.Contains("latency"))
.Select(m => new Tuple<string, double>(m.Name, m.Value));
IEnumerable<Tuple<string, double?>> actualValues = metrics.Where(m => m.Name.Contains("latency"))
.Select(m => new Tuple<string, double?>(m.Name, m.Value));

Assert.IsTrue(expectedValues.Count() == actualValues.Count());
CollectionAssert.AreEquivalent(expectedValues, actualValues);
@@ -306,11 +306,11 @@ public void FioResultsParserAppliesTheExpectedDefaultConversionFactorToMeasureme

// All latency metrics are emitted in milliseconds form (vs. nanoseconds).
double expectedConversionFactor = 0.000001;
IEnumerable<Tuple<string, double>> expectedValues = FioMetricsParserTests.ExpectedWriteMetrics.Where(entry => entry.Key.Contains("latency"))
.Select(entry => new Tuple<string, double>(entry.Key, entry.Value * expectedConversionFactor));
IEnumerable<Tuple<string, double?>> expectedValues = FioMetricsParserTests.ExpectedWriteMetrics.Where(entry => entry.Key.Contains("latency"))
.Select(entry => new Tuple<string, double?>(entry.Key, entry.Value * expectedConversionFactor));

IEnumerable<Tuple<string, double>> actualValues = metrics.Where(m => m.Name.Contains("latency"))
.Select(m => new Tuple<string, double>(m.Name, m.Value));
IEnumerable<Tuple<string, double?>> actualValues = metrics.Where(m => m.Name.Contains("latency"))
.Select(m => new Tuple<string, double?>(m.Name, m.Value));

Assert.IsTrue(expectedValues.Count() == actualValues.Count());
CollectionAssert.AreEquivalent(expectedValues, actualValues);
Original file line number Diff line number Diff line change
@@ -40,5 +40,22 @@ public void SpecJbbParserVerifyMetricsFpRate()
MetricAssert.Exists(metrics, "max-jOPS", 4188, "jOPS");
MetricAssert.Exists(metrics, "critical-jOPS", 1666, "jOPS");
}


[Test]
public void SpecJbbParserVerifyNanMetricsFpRate()
{
string workingDirectory = Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location);
string outputPath = Path.Combine(workingDirectory, "Examples", "SPECjbb", "specjbbNanOutput1.txt");
this.rawText = File.ReadAllText(outputPath);
this.testParser = new SpecJbbMetricsParser(this.rawText);
IList<Metric> metrics = this.testParser.Parse();

Assert.AreEqual(4, metrics.Count);
MetricAssert.Exists(metrics, "hbIR (max attempted)", 304872, "jOPS");
MetricAssert.Exists(metrics, "hbIR (settled)", null, "jOPS");
MetricAssert.Exists(metrics, "max-jOPS", 234751, "jOPS");
MetricAssert.Exists(metrics, "critical-jOPS", null , "jOPS");
}
}
}
Original file line number Diff line number Diff line change
@@ -344,9 +344,9 @@ private Task ExecuteWorkloadAsync(EventContext telemetryContext, CancellationTok
private IList<Metric> CalculateTimeSpyAggregates(IList<Metric> metrics)
{
IList<Metric> aggregates = new List<Metric>();
double tsgt1 = 0;
double tsgt2 = 0;
double tsct = 0;
double? tsgt1 = 0;
double? tsgt2 = 0;
double? tsct = 0;
foreach (Metric metric in metrics)
{
if (metric.Name == "timespy.graphics.1")
@@ -366,9 +366,9 @@ private IList<Metric> CalculateTimeSpyAggregates(IList<Metric> metrics)
// Weighted Harmonic Mean of Individual Scores
if (tsgt1 != 0 && tsgt2 != 0 && tsct != 0)
{
double graphicsScore = 165 * (2 / ((1 / tsgt1) + (1 / tsgt2)));
double cpuScore = 298 * tsct;
double aggScore = 1 / ((0.85 / graphicsScore) + (0.15 / cpuScore));
double? graphicsScore = 165 * (2 / ((1 / tsgt1) + (1 / tsgt2)));
double? cpuScore = 298 * tsct;
double? aggScore = 1 / ((0.85 / graphicsScore) + (0.15 / cpuScore));
aggregates.Add(new Metric("timespy.graphics.agg", graphicsScore, "score", MetricRelativity.HigherIsBetter));
aggregates.Add(new Metric("timespy.cpu.agg", cpuScore, "score", MetricRelativity.HigherIsBetter));
aggregates.Add(new Metric("timespy.finalscore", aggScore, "score", MetricRelativity.HigherIsBetter));
Original file line number Diff line number Diff line change
@@ -43,7 +43,8 @@ public override IList<Metric> Parse()
string[] tokens = metric.Split("=");
string name = tokens[0];
string value = tokens[1];
this.Metrics.Add(new Metric(name.Trim(), Convert.ToDouble(value), SpecJbbMetricsParser.OperationPerSecond, MetricRelativity.HigherIsBetter));

this.Metrics.Add(new Metric(name.Trim(), (value.Trim().Equals("N/A", StringComparison.OrdinalIgnoreCase)) ? null : Convert.ToDouble(value), SpecJbbMetricsParser.OperationPerSecond, MetricRelativity.HigherIsBetter));
}

return this.Metrics;
12 changes: 6 additions & 6 deletions src/VirtualClient/VirtualClient.Contracts/Metric.cs
Original file line number Diff line number Diff line change
@@ -18,7 +18,7 @@ public class Metric : IEquatable<Metric>
/// <summary>
/// Creates a metric
/// </summary>
public Metric(string name, double value)
public Metric(string name, double? value)
{
this.Name = name;
this.Value = value;
@@ -30,7 +30,7 @@ public Metric(string name, double value)
/// <summary>
/// Creates a metric
/// </summary>
public Metric(string name, double value, IEnumerable<string> tags = null, string description = null, IDictionary<string, IConvertible> metadata = null)
public Metric(string name, double? value, IEnumerable<string> tags = null, string description = null, IDictionary<string, IConvertible> metadata = null)
: this(name, value)
{
this.Description = description;
@@ -51,7 +51,7 @@ public Metric(string name, double value, IEnumerable<string> tags = null, string
/// <summary>
/// Creates a metric
/// </summary>
public Metric(string name, double value, MetricRelativity relativity, IEnumerable<string> tags = null, string description = null, IDictionary<string, IConvertible> metadata = null)
public Metric(string name, double? value, MetricRelativity relativity, IEnumerable<string> tags = null, string description = null, IDictionary<string, IConvertible> metadata = null)
: this(name, value, tags: tags, description: description, metadata: metadata)
{
this.Relativity = relativity;
@@ -60,7 +60,7 @@ public Metric(string name, double value, MetricRelativity relativity, IEnumerabl
/// <summary>
/// Creates a metric
/// </summary>
public Metric(string name, double value, string unit, IEnumerable<string> tags = null, string description = null, IDictionary<string, IConvertible> metadata = null)
public Metric(string name, double? value, string unit, IEnumerable<string> tags = null, string description = null, IDictionary<string, IConvertible> metadata = null)
: this(name, value, tags: tags, description: description, metadata: metadata)
{
this.Unit = unit;
@@ -69,7 +69,7 @@ public Metric(string name, double value, string unit, IEnumerable<string> tags =
/// <summary>
/// Creates a metric
/// </summary>
public Metric(string name, double value, string unit, MetricRelativity relativity, IEnumerable<string> tags = null, string description = null, IDictionary<string, IConvertible> metadata = null)
public Metric(string name, double? value, string unit, MetricRelativity relativity, IEnumerable<string> tags = null, string description = null, IDictionary<string, IConvertible> metadata = null)
: this(name, value, unit, tags: tags, description: description, metadata: metadata)
{
this.Relativity = relativity;
@@ -99,7 +99,7 @@ public Metric(string name, double value, string unit, MetricRelativity relativit
/// <summary>
/// Result of test
/// </summary>
public double Value { get; }
public double? Value { get; }

/// <summary>
/// Unit of result
6 changes: 3 additions & 3 deletions src/VirtualClient/VirtualClient.Contracts/MetricAggregate.cs
Original file line number Diff line number Diff line change
@@ -12,7 +12,7 @@ namespace VirtualClient.Contracts
/// Provides features for capturing Windows performance counter values over a period
/// of time.
/// </summary>
public class MetricAggregate : ConcurrentBag<double>
public class MetricAggregate : ConcurrentBag<double?>
{
/// <summary>
/// Initializes a new instance of the <see cref="MetricAggregate"/> class.
@@ -115,11 +115,11 @@ public Metric ToMetric(MetricAggregateType aggregateType)
return Metric.None;
}

double value = 0;
double? value = 0;
switch (aggregateType)
{
case MetricAggregateType.Average:
double sum = this.Sum();
double? sum = this.Sum();
value = sum / this.Count;
break;

Original file line number Diff line number Diff line change
@@ -744,7 +744,7 @@ public static void LogMetrics(
DateTime scenarioStartTime,
DateTime scenarioEndTime,
string metricName,
double metricValue,
double? metricValue,
string metricUnits,
string metricCategorization,
string scenarioArguments,
Original file line number Diff line number Diff line change
@@ -27,7 +27,7 @@ public static void Exists(
/// <summary>
/// Asserts the metric exists in the given list of results
/// </summary>
public static void Exists(IList<Metric> results, string expectedMetric, double expectedMetricValue, string expectedMetricUnit = null, List<string> expectedTags = null)
public static void Exists(IList<Metric> results, string expectedMetric, double? expectedMetricValue, string expectedMetricUnit = null, List<string> expectedTags = null)
{
List<Metric> matchingMetrics = results.Where(m => m.Name == expectedMetric).ToList();

0 comments on commit 78fba99

Please sign in to comment.