Compare performance to baseline

Author: OliverRietmann

.github/scripts/csv_to_md.py

@@ -2,11 +2,42 @@
 import csv
 import tabulate as tab
 
-csv_filename = sys.argv[1]
+csv_benchmark = sys.argv[1]
+csv_baseline = sys.argv[2]
 
-with open(csv_filename) as csv_file:
+def html_inline(x):
+  if x == 0:
+    color = "black"
+    x = "{:.1f}".format(x)
+  elif x < 0:
+    color = "green"
+    x = "{:.1f}".format(x)
+  elif x > 0:
+    color = "red"
+    x = "+{:.1f}".format(x)
+  return f"<span style=\"color:{color}\">{x}</span>"
+
+with open(csv_benchmark) as csv_file:
+  csv_reader = csv.reader(csv_file)
+  next(csv_reader)
+  table_benchmark = [row for row in csv_reader]
+ 
+with open(csv_baseline) as csv_file:
   csv_reader = csv.reader(csv_file)
-  header = next(csv_reader)
-  table = [row for row in csv_reader]
+  next(csv_reader)
+  table_baseline = [row for row in csv_reader]
+
+table = []
+for benchmark, baseline in zip(table_benchmark, table_baseline):
+  assert(benchmark[0] == baseline[0] and benchmark[2:4] == baseline[2:4])
+  name = benchmark[0]
+  time = benchmark[1]
+  d = float(baseline[1]) - float(benchmark[1])
+  difference = html_inline(d)
+  percent = html_inline(100 * d / float(baseline[1]))
+  count = benchmark[2]
+  events = benchmark[3]
+  table.append([name, time, difference, percent, count, events])
 
+header = ["name", "time", "difference", "percent", "count", "events"]
 print(tab.tabulate(table, header, tablefmt="github"))

.github/workflows/standalone-benchmark.yml

@@ -47,6 +47,9 @@ jobs:
 
           curl -fL --retry 3 -o ${STANDALONE_DIR}/o2-simple-GPU.out https://cernbox.cern.ch/remote.php/dav/public-files/SfYXgQOHFga2w75/o2-simple-GPU.out
 
+          mkdir -p ${STANDALONE_DIR}/baseline
+          curl -fL --retry 3 -o ${STANDALONE_DIR}/baseline/${{ matrix.name }}.csv https://cernbox.cern.ch/remote.php/dav/public-files/SfYXgQOHFga2w75/baseline/${{ matrix.name }}.csv
+
           mkdir -p ${STANDALONE_DIR}/events
           curl -fL --retry 3 -o ${STANDALONE_DIR}/events/o2-simple.tar.xz https://cernbox.cern.ch/remote.php/dav/public-files/SfYXgQOHFga2w75/events/o2-simple.tar.xz
           tar -xf ${STANDALONE_DIR}/events/o2-simple.tar.xz -C ${STANDALONE_DIR}/events
@@ -91,7 +94,8 @@ jobs:
         run: |
           source /etc/profile.d/modules.sh
           module load ninja/fortran-v1.11.1.g9-15 Vc/1.4.5-10 boost/v1.83.0-alice2-57 fmt/11.1.2-14 CMake/v3.31.6-10 ms_gsl/4.2.1-3 Clang/v20.1.7-9 TBB/v2022.3.0-3 ROOT/v6-36-04-alice9-15 ONNXRuntime/v1.22.0-71 GLFW/3.3.2-25
-          python3 ${GITHUB_WORKSPACE}/.github/scripts/csv_to_md.py ${ARTIFACT_FILE} >> ${GITHUB_STEP_SUMMARY}
+          python3 ${GITHUB_WORKSPACE}/.github/scripts/csv_to_md.py ${ARTIFACT_FILE} ${STANDALONE_DIR}/baseline/${{ matrix.name }}.csv >> ${GITHUB_STEP_SUMMARY}
+          rm -rf ${STANDALONE_DIR}/baseline
 
       - name: Upload Artifact
         uses: actions/upload-artifact@v4