Skip to content

Commit fb7e018

Browse files
rafaelfrancarafaelfranca
authored
Merge pull request #439 from ruby/rmf-ResultsTableBuilder
Extract class to build the results table from benchmark data
2 parents b1a7224 + 54071c6 commit fb7e018

File tree

5 files changed

+551
-128
lines changed

5 files changed

+551
-128
lines changed

‎lib/benchmark_runner.rb‎

Lines changed: 0 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -16,16 +16,6 @@ def free_file_no(directory)
1616
end
1717
end
1818

19-
# Sort benchmarks with headlines first, then others, then micro
20-
def sort_benchmarks(bench_names, metadata)
21-
headline_benchmarks = metadata.select { |_, meta| meta['category'] == 'headline' }.keys
22-
micro_benchmarks = metadata.select { |_, meta| meta['category'] == 'micro' }.keys
23-
24-
headline_names, bench_names = bench_names.partition { |name| headline_benchmarks.include?(name) }
25-
micro_names, other_names = bench_names.partition { |name| micro_benchmarks.include?(name) }
26-
headline_names.sort + other_names.sort + micro_names.sort
27-
end
28-
2919
# Checked system - error or return info if the command fails
3020
def check_call(command, env: {}, raise_error: true, quiet: false)
3121
puts("+ #{command}") unless quiet

‎lib/results_table_builder.rb‎

Lines changed: 170 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,170 @@
1+
require_relative '../misc/stats'
2+
require 'yaml'
3+
4+
class ResultsTableBuilder
5+
SECONDS_TO_MS = 1000.0
6+
BYTES_TO_MIB = 1024.0 * 1024.0
7+
8+
def initialize(executable_names:, bench_data:, include_rss: false)
9+
@executable_names = executable_names
10+
@bench_data = bench_data
11+
@include_rss = include_rss
12+
@base_name = executable_names.first
13+
@other_names = executable_names[1..]
14+
@bench_names = compute_bench_names
15+
end
16+
17+
def build
18+
table = [build_header]
19+
format = build_format
20+
21+
@bench_names.each do |bench_name|
22+
next unless has_complete_data?(bench_name)
23+
24+
row = build_row(bench_name)
25+
table << row
26+
end
27+
28+
[table, format]
29+
end
30+
31+
private
32+
33+
def has_complete_data?(bench_name)
34+
@bench_data.all? { |(_k, v)| v[bench_name] }
35+
end
36+
37+
def build_header
38+
header = ["bench"]
39+
40+
@executable_names.each do |name|
41+
header << "#{name} (ms)" << "stddev (%)"
42+
header << "RSS (MiB)" if @include_rss
43+
end
44+
45+
@other_names.each do |name|
46+
header << "#{name} 1st itr"
47+
end
48+
49+
@other_names.each do |name|
50+
header << "#{@base_name}/#{name}"
51+
end
52+
53+
header
54+
end
55+
56+
def build_format
57+
format = ["%s"]
58+
59+
@executable_names.each do |_name|
60+
format << "%.1f" << "%.1f"
61+
format << "%.1f" if @include_rss
62+
end
63+
64+
@other_names.each do |_name|
65+
format << "%.3f"
66+
end
67+
68+
@other_names.each do |_name|
69+
format << "%.3f"
70+
end
71+
72+
format
73+
end
74+
75+
def build_row(bench_name)
76+
t0s = extract_first_iteration_times(bench_name)
77+
times_no_warmup = extract_benchmark_times(bench_name)
78+
rsss = extract_rss_values(bench_name)
79+
80+
base_t0, *other_t0s = t0s
81+
base_t, *other_ts = times_no_warmup
82+
base_rss, *other_rsss = rsss
83+
84+
row = [bench_name]
85+
build_base_columns(row, base_t, base_rss)
86+
build_comparison_columns(row, other_ts, other_rsss)
87+
build_ratio_columns(row, base_t0, other_t0s, base_t, other_ts)
88+
89+
row
90+
end
91+
92+
def build_base_columns(row, base_t, base_rss)
93+
row << mean(base_t)
94+
row << stddev_percent(base_t)
95+
row << base_rss if @include_rss
96+
end
97+
98+
def build_comparison_columns(row, other_ts, other_rsss)
99+
other_ts.zip(other_rsss).each do |other_t, other_rss|
100+
row << mean(other_t)
101+
row << stddev_percent(other_t)
102+
row << other_rss if @include_rss
103+
end
104+
end
105+
106+
def build_ratio_columns(row, base_t0, other_t0s, base_t, other_ts)
107+
ratio_1sts = other_t0s.map { |other_t0| base_t0 / other_t0 }
108+
ratios = other_ts.map { |other_t| mean(base_t) / mean(other_t) }
109+
row.concat(ratio_1sts)
110+
row.concat(ratios)
111+
end
112+
113+
def extract_first_iteration_times(bench_name)
114+
@executable_names.map do |name|
115+
data = bench_data_for(name, bench_name)
116+
(data['warmup'][0] || data['bench'][0]) * SECONDS_TO_MS
117+
end
118+
end
119+
120+
def extract_benchmark_times(bench_name)
121+
@executable_names.map do |name|
122+
bench_data_for(name, bench_name)['bench'].map { |v| v * SECONDS_TO_MS }
123+
end
124+
end
125+
126+
def extract_rss_values(bench_name)
127+
@executable_names.map do |name|
128+
bench_data_for(name, bench_name)['rss'] / BYTES_TO_MIB
129+
end
130+
end
131+
132+
def bench_data_for(name, bench_name)
133+
@bench_data[name][bench_name]
134+
end
135+
136+
def mean(values)
137+
Stats.new(values).mean
138+
end
139+
140+
def stddev(values)
141+
Stats.new(values).stddev
142+
end
143+
144+
def stddev_percent(values)
145+
100 * stddev(values) / mean(values)
146+
end
147+
148+
def compute_bench_names
149+
benchmarks_metadata = YAML.load_file('benchmarks.yml')
150+
sort_benchmarks(all_benchmark_names, benchmarks_metadata)
151+
end
152+
153+
def all_benchmark_names
154+
@bench_data.values.flat_map(&:keys).uniq
155+
end
156+
157+
# Sort benchmarks with headlines first, then others, then micro
158+
def sort_benchmarks(bench_names, metadata)
159+
bench_names.sort_by { |name| [category_priority(name, metadata), name] }
160+
end
161+
162+
def category_priority(bench_name, metadata)
163+
category = metadata.dig(bench_name, 'category') || 'other'
164+
case category
165+
when 'headline' then 0
166+
when 'micro' then 2
167+
else 1
168+
end
169+
end
170+
end

‎run_benchmarks.rb‎

Lines changed: 10 additions & 67 deletions
Original file line numberDiff line numberDiff line change
@@ -8,25 +8,12 @@
88
require 'rbconfig'
99
require 'etc'
1010
require 'yaml'
11-
require_relative 'misc/stats'
1211
require_relative 'lib/cpu_config'
1312
require_relative 'lib/benchmark_runner'
1413
require_relative 'lib/benchmark_suite'
1514
require_relative 'lib/table_formatter'
1615
require_relative 'lib/argument_parser'
17-
18-
def mean(values)
19-
Stats.new(values).mean
20-
end
21-
22-
def stddev(values)
23-
Stats.new(values).stddev
24-
end
25-
26-
def sort_benchmarks(bench_names)
27-
benchmarks_metadata = YAML.load_file('benchmarks.yml')
28-
BenchmarkRunner.sort_benchmarks(bench_names, benchmarks_metadata)
29-
end
16+
require_relative 'lib/results_table_builder'
3017

3118
args = ArgumentParser.parse(ARGV)
3219

@@ -36,15 +23,14 @@ def sort_benchmarks(bench_names)
3623
FileUtils.mkdir_p(args.out_path)
3724

3825
ruby_descriptions = {}
39-
args.executables.each do |name, executable|
40-
ruby_descriptions[name] = `#{executable.shelljoin} -v`.chomp
41-
end
4226

4327
# Benchmark with and without YJIT
4428
bench_start_time = Time.now.to_f
4529
bench_data = {}
4630
bench_failures = {}
4731
args.executables.each do |name, executable|
32+
ruby_descriptions[name] = `#{executable.shelljoin} -v`.chomp
33+
4834
suite = BenchmarkSuite.new(
4935
ruby: executable,
5036
ruby_description: ruby_descriptions[name],
@@ -61,9 +47,6 @@ def sort_benchmarks(bench_names)
6147
end
6248

6349
bench_end_time = Time.now.to_f
64-
# Get keys from all rows in case a benchmark failed for only some executables.
65-
bench_names = sort_benchmarks(bench_data.map { |k, v| v.keys }.flatten.uniq)
66-
6750
bench_total_time = (bench_end_time - bench_start_time).to_i
6851
puts("Total time spent benchmarking: #{bench_total_time}s")
6952

@@ -73,55 +56,15 @@ def sort_benchmarks(bench_names)
7356

7457
puts
7558

76-
# Table for the data we've gathered
59+
# Build results table
7760
all_names = args.executables.keys
7861
base_name, *other_names = all_names
79-
table = [["bench"]]
80-
format = ["%s"]
81-
all_names.each do |name|
82-
table[0] += ["#{name} (ms)", "stddev (%)"]
83-
format += ["%.1f", "%.1f"]
84-
if args.rss
85-
table[0] += ["RSS (MiB)"]
86-
format += ["%.1f"]
87-
end
88-
end
89-
other_names.each do |name|
90-
table[0] += ["#{name} 1st itr"]
91-
format += ["%.3f"]
92-
end
93-
other_names.each do |name|
94-
table[0] += ["#{base_name}/#{name}"]
95-
format += ["%.3f"]
96-
end
97-
98-
# Format the results table
99-
bench_names.each do |bench_name|
100-
# Skip this bench_name if we failed to get data for any of the executables.
101-
next unless bench_data.all? { |(_k, v)| v[bench_name] }
102-
103-
t0s = all_names.map { |name| (bench_data[name][bench_name]['warmup'][0] || bench_data[name][bench_name]['bench'][0]) * 1000.0 }
104-
times_no_warmup = all_names.map { |name| bench_data[name][bench_name]['bench'].map { |v| v * 1000.0 } }
105-
rsss = all_names.map { |name| bench_data[name][bench_name]['rss'] / 1024.0 / 1024.0 }
106-
107-
base_t0, *other_t0s = t0s
108-
base_t, *other_ts = times_no_warmup
109-
base_rss, *other_rsss = rsss
110-
111-
ratio_1sts = other_t0s.map { |other_t0| base_t0 / other_t0 }
112-
ratios = other_ts.map { |other_t| mean(base_t) / mean(other_t) }
113-
114-
row = [bench_name, mean(base_t), 100 * stddev(base_t) / mean(base_t)]
115-
row << base_rss if args.rss
116-
other_ts.zip(other_rsss).each do |other_t, other_rss|
117-
row += [mean(other_t), 100 * stddev(other_t) / mean(other_t)]
118-
row << other_rss if args.rss
119-
end
120-
121-
row += ratio_1sts + ratios
122-
123-
table << row
124-
end
62+
builder = ResultsTableBuilder.new(
63+
executable_names: all_names,
64+
bench_data: bench_data,
65+
include_rss: args.rss
66+
)
67+
table, format = builder.build
12568

12669
output_path = nil
12770
if args.out_override

‎test/benchmark_runner_test.rb‎

Lines changed: 0 additions & 51 deletions
Original file line numberDiff line numberDiff line change
@@ -49,57 +49,6 @@
4949
end
5050
end
5151

52-
describe '.sort_benchmarks' do
53-
before do
54-
@metadata = {
55-
'fib' => { 'category' => 'micro' },
56-
'railsbench' => { 'category' => 'headline' },
57-
'optcarrot' => { 'category' => 'headline' },
58-
'some_bench' => { 'category' => 'other' },
59-
'another_bench' => { 'category' => 'other' },
60-
'zebra' => { 'category' => 'other' }
61-
}
62-
end
63-
64-
it 'sorts benchmarks with headlines first, then others, then micro' do
65-
bench_names = ['fib', 'some_bench', 'railsbench', 'another_bench', 'optcarrot']
66-
result = BenchmarkRunner.sort_benchmarks(bench_names, @metadata)
67-
68-
# Headlines should be first
69-
headline_indices = [result.index('railsbench'), result.index('optcarrot')]
70-
assert_equal true, headline_indices.all? { |i| i < 2 }
71-
72-
# Micro should be last
73-
assert_equal 'fib', result.last
74-
75-
# Others in the middle
76-
other_indices = [result.index('some_bench'), result.index('another_bench')]
77-
assert_equal true, other_indices.all? { |i| i >= 2 && i < result.length - 1 }
78-
end
79-
80-
it 'sorts alphabetically within categories' do
81-
bench_names = ['zebra', 'another_bench', 'some_bench']
82-
result = BenchmarkRunner.sort_benchmarks(bench_names, @metadata)
83-
assert_equal ['another_bench', 'some_bench', 'zebra'], result
84-
end
85-
86-
it 'handles empty list' do
87-
result = BenchmarkRunner.sort_benchmarks([], @metadata)
88-
assert_equal [], result
89-
end
90-
91-
it 'handles single benchmark' do
92-
result = BenchmarkRunner.sort_benchmarks(['fib'], @metadata)
93-
assert_equal ['fib'], result
94-
end
95-
96-
it 'handles only headline benchmarks' do
97-
bench_names = ['railsbench', 'optcarrot']
98-
result = BenchmarkRunner.sort_benchmarks(bench_names, @metadata)
99-
assert_equal ['optcarrot', 'railsbench'], result
100-
end
101-
end
102-
10352
describe '.check_call' do
10453
it 'runs a successful command and returns success status' do
10554
result = nil

0 commit comments

Comments
 (0)