Combine measurements and stddev in one column

Author: jhawthorn

lib/results_table_builder.rb

@@ -39,7 +39,7 @@ def build_header
     header = ["bench"]
 
     @executable_names.each do |name|
-      header << "#{name} (ms)" << "stddev (%)"
+      header << "#{name} (ms)"
       header << "RSS (MiB)" if @include_rss
     end
 
@@ -67,7 +67,7 @@ def build_format
     format = ["%s"]
 
     @executable_names.each do |_name|
-      format << "%.1f" << "%.1f"
+      format << "%s"
       format << "%.1f" if @include_rss
     end
 
@@ -110,19 +110,21 @@ def build_row(bench_name)
   end
 
   def build_base_columns(row, base_t, base_rss)
-    row << mean(base_t)
-    row << stddev_percent(base_t)
+    row << format_time_with_stddev(base_t)
     row << base_rss if @include_rss
   end
 
   def build_comparison_columns(row, other_ts, other_rsss)
     other_ts.zip(other_rsss).each do |other_t, other_rss|
-      row << mean(other_t)
-      row << stddev_percent(other_t)
+      row << format_time_with_stddev(other_t)
       row << other_rss if @include_rss
     end
   end
 
+  def format_time_with_stddev(values)
+    "%.1f ± %.1f%%" % [mean(values), stddev_percent(values)]
+  end
+
   def build_ratio_columns(row, base_t0, other_t0s, base_t, other_ts)
     ratio_1sts = other_t0s.map { |other_t0| base_t0 / other_t0 }
     row.concat(ratio_1sts)

test/results_table_builder_test.rb

@@ -56,15 +56,25 @@
 
       table, format = builder.build
 
-      assert_equal ['bench', 'ruby (ms)', 'stddev (%)', 'ruby-yjit (ms)', 'stddev (%)', 'ruby-yjit 1st itr', 'ruby/ruby-yjit'], table[0]
+      assert_equal ['bench', 'ruby (ms)', 'ruby-yjit (ms)', 'ruby-yjit 1st itr', 'ruby/ruby-yjit'], table[0]
 
-      assert_equal ['%s', '%.1f', '%.1f', '%.1f', '%.1f', '%.3f', '%s'], format
+      assert_equal ['%s', '%s', '%s', '%.3f', '%s'], format
 
       assert_equal 'fib', table[1][0]
-      assert_in_delta 100.0, table[1][1], 1.0
-      assert_in_delta 50.0, table[1][3], 1.0
-      assert_in_delta 2.0, table[1][5], 0.1
-      assert_match(/^2\.0\d+/, table[1][6])
+
+      m = table[1][1].match(/\A(\d+\.\d) ± (\d+\.\d)%\z/)
+      assert m
+      assert_in_delta 100.0, m[1].to_f, 1.0
+
+      m = table[1][2].match(/\A(\d+\.\d) ± (\d+\.\d)%\z/)
+      assert m
+      assert_in_delta 50.0, m[1].to_f, 1.0
+
+      assert_in_delta 2.0, table[1][3], 0.1
+
+      m = table[1][4].match(/\A(\d+\.\d+)/)
+      assert m
+      assert_in_delta 2.0, m[1].to_f, 0.1
     end
 
     it 'includes RSS columns when include_rss is true' do
@@ -88,9 +98,9 @@
       table, format = builder.build
 
       # No RSS ratio column with a single executable
-      assert_equal ['bench', 'ruby (ms)', 'stddev (%)', 'RSS (MiB)'], table[0]
-      assert_equal ['%s', '%.1f', '%.1f', '%.1f'], format
-      assert_in_delta 10.0, table[1][3], 0.1
+      assert_equal ['bench', 'ruby (ms)', 'RSS (MiB)'], table[0]
+      assert_equal ['%s', '%s', '%.1f'], format
+      assert_in_delta 10.0, table[1][2], 0.1
     end
 
     it 'includes RSS ratio columns when include_rss is true with multiple executables' do
@@ -122,15 +132,15 @@
 
       expected_header = [
         'bench',
-        'ruby (ms)', 'stddev (%)', 'RSS (MiB)',
-        'ruby-yjit (ms)', 'stddev (%)', 'RSS (MiB)',
+        'ruby (ms)', 'RSS (MiB)',
+        'ruby-yjit (ms)', 'RSS (MiB)',
         'ruby-yjit 1st itr',
         'ruby/ruby-yjit',
         'RSS ruby/ruby-yjit'
       ]
       assert_equal expected_header, table[0]
 
-      expected_format = ['%s', '%.1f', '%.1f', '%.1f', '%.1f', '%.1f', '%.1f', '%.3f', '%s', '%.3f']
+      expected_format = ['%s', '%s', '%.1f', '%s', '%.1f', '%.3f', '%s', '%.3f']
       assert_equal expected_format, format
 
       # RSS ratio: 10 MiB / 20 MiB = 0.5
@@ -209,17 +219,17 @@
 
       expected_header = [
         'bench',
-        'ruby (ms)', 'stddev (%)',
-        'ruby-yjit (ms)', 'stddev (%)',
-        'ruby-rjit (ms)', 'stddev (%)',
+        'ruby (ms)',
+        'ruby-yjit (ms)',
+        'ruby-rjit (ms)',
         'ruby-yjit 1st itr',
         'ruby-rjit 1st itr',
         'ruby/ruby-yjit',
         'ruby/ruby-rjit'
       ]
       assert_equal expected_header, table[0]
 
-      expected_format = ['%s', '%.1f', '%.1f', '%.1f', '%.1f', '%.1f', '%.1f', '%.3f', '%.3f', '%s', '%s']
+      expected_format = ['%s', '%s', '%s', '%s', '%.3f', '%.3f', '%s', '%s']
       assert_equal expected_format, format
     end
 
@@ -245,7 +255,9 @@
 
       assert_equal 2, table.length
       assert_equal 'fib', table[1][0]
-      assert_in_delta 100.0, table[1][1], 5.0
+      m = table[1][1].match(/\A(\d+\.\d) ± (\d+\.\d)%\z/)
+      assert m
+      assert_in_delta 100.0, m[1].to_f, 5.0
     end
 
     it 'sorts benchmarks with headlines first, then others, then micro' do