Merge pull request #482 from ruby/mvh-rss-ratio

When --rss is used, include an "RSS ratio" column

Author: eightbitraptor

lib/benchmark_runner.rb

@@ -48,7 +48,7 @@ def write_csv(output_path, ruby_descriptions, table)
     end
 
     # Build output text string with metadata, table, and legend
-    def build_output_text(ruby_descriptions, table, format, bench_failures)
+    def build_output_text(ruby_descriptions, table, format, bench_failures, include_rss: false)
       base_name, *other_names = ruby_descriptions.keys
 
       output_str = +""
@@ -65,6 +65,9 @@ def build_output_text(ruby_descriptions, table, format, bench_failures)
         other_names.each do |name|
           output_str << "- #{name} 1st itr: ratio of #{base_name}/#{name} time for the first benchmarking iteration.\n"
           output_str << "- #{base_name}/#{name}: ratio of #{base_name}/#{name} time. Higher is better for #{name}. Above 1 represents a speedup.\n"
+          if include_rss
+            output_str << "- RSS #{base_name}/#{name}: ratio of #{base_name}/#{name} RSS. Higher is better for #{name}. Above 1 means lower memory usage.\n"
+          end
         end
         output_str << "- ***: p < 0.001, **: p < 0.01, *: p < 0.05 (Welch's t-test)\n"
       end

lib/benchmark_runner/cli.rb

@@ -85,7 +85,7 @@ def run
       BenchmarkRunner.write_csv(output_path, ruby_descriptions, table)
 
       # Save the output in a text file that we can easily refer to
-      output_str = BenchmarkRunner.build_output_text(ruby_descriptions, table, format, bench_failures)
+      output_str = BenchmarkRunner.build_output_text(ruby_descriptions, table, format, bench_failures, include_rss: args.rss)
       out_txt_path = output_path + ".txt"
       File.open(out_txt_path, "w") { |f| f.write output_str }
 

lib/results_table_builder.rb

@@ -54,6 +54,12 @@ def build_header
       end
     end
 
+    if @include_rss
+      @other_names.each do |name|
+        header << "RSS #{@base_name}/#{name}"
+      end
+    end
+
     header
   end
 
@@ -76,6 +82,12 @@ def build_format
       end
     end
 
+    if @include_rss
+      @other_names.each do |_name|
+        format << "%.3f"
+      end
+    end
+
     format
   end
 
@@ -92,6 +104,7 @@ def build_row(bench_name)
     build_base_columns(row, base_t, base_rss)
     build_comparison_columns(row, other_ts, other_rsss)
     build_ratio_columns(row, base_t0, other_t0s, base_t, other_ts)
+    build_rss_ratio_columns(row, base_rss, other_rsss)
 
     row
   end
@@ -124,6 +137,14 @@ def build_ratio_columns(row, base_t0, other_t0s, base_t, other_ts)
     end
   end
 
+  def build_rss_ratio_columns(row, base_rss, other_rsss)
+    return unless @include_rss
+
+    other_rsss.each do |other_rss|
+      row << base_rss / other_rss
+    end
+  end
+
   def format_ratio(ratio, pval)
     sym = significance_symbol(pval)
     formatted = "%.3f" % ratio

test/benchmark_runner_test.rb

@@ -390,6 +390,44 @@
       assert_includes result, "- ***: p < 0.001, **: p < 0.01, *: p < 0.05 (Welch's t-test)"
     end
 
+    it 'includes RSS ratio legend when include_rss is true' do
+      ruby_descriptions = {
+        'ruby-base' => 'ruby 3.3.0',
+        'ruby-yjit' => 'ruby 3.3.0 +YJIT'
+      }
+      table = [
+        ['bench', 'ruby-base (ms)', 'stddev (%)', 'RSS (MiB)', 'ruby-yjit (ms)', 'stddev (%)', 'RSS (MiB)', 'ruby-yjit 1st itr', 'ruby-base/ruby-yjit', 'RSS ruby-base/ruby-yjit'],
+        ['fib', '100.0', '5.0', '10.0', '50.0', '3.0', '12.0', '2.000', '2.000', '0.833']
+      ]
+      format = ['%s', '%.1f', '%.1f', '%.1f', '%.1f', '%.1f', '%.1f', '%.3f', '%s', '%.3f']
+      bench_failures = {}
+
+      result = BenchmarkRunner.build_output_text(
+        ruby_descriptions, table, format, bench_failures, include_rss: true
+      )
+
+      assert_includes result, '- RSS ruby-base/ruby-yjit: ratio of ruby-base/ruby-yjit RSS. Higher is better for ruby-yjit. Above 1 means lower memory usage.'
+    end
+
+    it 'omits RSS ratio legend when include_rss is false' do
+      ruby_descriptions = {
+        'ruby-base' => 'ruby 3.3.0',
+        'ruby-yjit' => 'ruby 3.3.0 +YJIT'
+      }
+      table = [
+        ['bench', 'ruby-base (ms)', 'stddev (%)', 'ruby-yjit (ms)', 'stddev (%)'],
+        ['fib', '100.0', '5.0', '50.0', '3.0']
+      ]
+      format = ['%s', '%.1f', '%.1f', '%.1f', '%.1f']
+      bench_failures = {}
+
+      result = BenchmarkRunner.build_output_text(
+        ruby_descriptions, table, format, bench_failures
+      )
+
+      refute_includes result, 'RSS ruby-base/ruby-yjit'
+    end
+
     it 'includes formatted table in output' do
       ruby_descriptions = { 'ruby' => 'ruby 3.3.0' }
       table = [

test/results_table_builder_test.rb

@@ -87,13 +87,56 @@
 
       table, format = builder.build
 
+      # No RSS ratio column with a single executable
       assert_equal ['bench', 'ruby (ms)', 'stddev (%)', 'RSS (MiB)'], table[0]
-
       assert_equal ['%s', '%.1f', '%.1f', '%.1f'], format
-
       assert_in_delta 10.0, table[1][3], 0.1
     end
 
+    it 'includes RSS ratio columns when include_rss is true with multiple executables' do
+      executable_names = ['ruby', 'ruby-yjit']
+      bench_data = {
+        'ruby' => {
+          'fib' => {
+            'warmup' => [0.1],
+            'bench' => [0.1, 0.11, 0.09],
+            'rss' => 1024 * 1024 * 10
+          }
+        },
+        'ruby-yjit' => {
+          'fib' => {
+            'warmup' => [0.05],
+            'bench' => [0.05, 0.06, 0.04],
+            'rss' => 1024 * 1024 * 20
+          }
+        }
+      }
+
+      builder = ResultsTableBuilder.new(
+        executable_names: executable_names,
+        bench_data: bench_data,
+        include_rss: true
+      )
+
+      table, format = builder.build
+
+      expected_header = [
+        'bench',
+        'ruby (ms)', 'stddev (%)', 'RSS (MiB)',
+        'ruby-yjit (ms)', 'stddev (%)', 'RSS (MiB)',
+        'ruby-yjit 1st itr',
+        'ruby/ruby-yjit',
+        'RSS ruby/ruby-yjit'
+      ]
+      assert_equal expected_header, table[0]
+
+      expected_format = ['%s', '%.1f', '%.1f', '%.1f', '%.1f', '%.1f', '%.1f', '%.3f', '%s', '%.3f']
+      assert_equal expected_format, format
+
+      # RSS ratio: 10 MiB / 20 MiB = 0.5
+      assert_in_delta 0.5, table[1].last, 0.01
+    end
+
     it 'skips benchmarks with missing data' do
       executable_names = ['ruby', 'ruby-yjit']
       bench_data = {