Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
23 changes: 14 additions & 9 deletions .github/workflows/benchmarks.yml
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,7 @@ jobs:
- name: Get system info
id: sysinfo
run: |
echo "python_version=$(python --version)" >> $GITHUB_OUTPUT
echo "python_version=$(uv run python --version)" >> $GITHUB_OUTPUT
echo "os=$(uname -s)" >> $GITHUB_OUTPUT
echo "arch=$(uname -m)" >> $GITHUB_OUTPUT
echo "cpu_count=$(nproc)" >> $GITHUB_OUTPUT
Expand Down Expand Up @@ -159,9 +159,10 @@ jobs:
jq -r '
"| Endpoint | Baseline | SDK (100%) | Overhead | SDK (10%) | Overhead |",
"|----------|----------|------------|----------|-----------|----------|",
(.comparison_100 | to_entries[] |
"| \(.key) | \(.value.baseline_mean_ms | . * 10 | round / 10)ms | \(.value.sdk_mean_ms | . * 10 | round / 10)ms | +\(.value.mean_overhead_ms | . * 10 | round / 10)ms (\(.value.mean_overhead_pct | round)%) | - | - |"
)
(.comparison_100 | keys[]) as $key |
(.comparison_100[$key]) as $c100 |
(.comparison_10[$key] // null) as $c10 |
"| \($key) | \($c100.baseline_mean_ms * 10 | round / 10)ms | \($c100.sdk_mean_ms * 10 | round / 10)ms | +\($c100.mean_overhead_ms * 10 | round / 10)ms (\($c100.mean_overhead_pct | round)%) | \(if $c10 then "\($c10.sdk_mean_ms * 10 | round / 10)ms" else "-" end) | \(if $c10 then "+\($c10.mean_overhead_ms * 10 | round / 10)ms (\($c10.mean_overhead_pct | round)%)" else "-" end) |"
' benchmarks/results/realistic-workload.json >> "$SUMMARY_FILE"

cat >> "$SUMMARY_FILE" << 'EOF'
Expand All @@ -175,11 +176,15 @@ jobs:
jq -r '
"| QPS | Baseline | SDK (100%) | Overhead | SDK (10%) | Overhead |",
"|-----|----------|------------|----------|-----------|----------|",
(.baseline | to_entries[] |
. as $b |
($b.key | tostring) as $qps |
"| \($qps) | \($b.value.mean_ms | . * 10 | round / 10)ms | - | - | - | - |"
)
(.baseline | keys[] | tonumber) as $qps |
(.baseline["\($qps)"].mean_ms) as $base |
(.sdk_100["\($qps)"].mean_ms // null) as $sdk100 |
(.sdk_10["\($qps)"].mean_ms // null) as $sdk10 |
(if $sdk100 then (($sdk100 - $base) * 10 | round / 10) else null end) as $overhead100 |
(if $sdk10 then (($sdk10 - $base) * 10 | round / 10) else null end) as $overhead10 |
(if $sdk100 then ((($sdk100 - $base) / $base * 100) | round) else null end) as $pct100 |
(if $sdk10 then ((($sdk10 - $base) / $base * 100) | round) else null end) as $pct10 |
"| \($qps) | \($base * 10 | round / 10)ms | \(if $sdk100 then "\($sdk100 * 10 | round / 10)ms" else "-" end) | \(if $overhead100 then "+\($overhead100)ms (\($pct100)%)" else "-" end) | \(if $sdk10 then "\($sdk10 * 10 | round / 10)ms" else "-" end) | \(if $overhead10 then "+\($overhead10)ms (\($pct10)%)" else "-" end) |"
' benchmarks/results/fixed-qps-latency.json >> "$SUMMARY_FILE"

cat >> "$SUMMARY_FILE" << 'EOF'
Expand Down
2 changes: 2 additions & 0 deletions benchmarks/bench/fixed_qps_latency.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,8 @@
sends requests at a steady rate to measure realistic latency impact.
"""

from __future__ import annotations

import json
import os
import subprocess
Expand Down
2 changes: 2 additions & 0 deletions benchmarks/bench/realistic_workload.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,8 @@
These simulate production APIs that do actual work (I/O, validation, processing).
"""

from __future__ import annotations

import json
import os
import subprocess
Expand Down
2 changes: 2 additions & 0 deletions benchmarks/bench/sdk_sampling_rates.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,8 @@
#!/usr/bin/env python3
"""Benchmark: SDK with different sampling rates."""

from __future__ import annotations

import os
import shutil
import sys
Expand Down