|
| 1 | +// SPDX-License-Identifier: PMPL-1.0-or-later |
| 2 | +// Copyright (c) 2026 Jonathan D.A. Jewell (hyperpolymath) <j.d.a.jewell@open.ac.uk> |
| 3 | +// |
| 4 | +//! Routing and dispatch latency benchmarks for ECHIDNA. |
| 5 | +//! |
| 6 | +//! Measures: |
| 7 | +//! - LLM routing decision latency (deterministic prover selection) |
| 8 | +//! - Axiom scan throughput for various proof sizes |
| 9 | +//! - Trust level computation latency under varying factor combinations |
| 10 | +//! - ProverFactory instantiation cost (amortises cold-start cost) |
| 11 | +//! - Agentic planning throughput (goal queue processing rate) |
| 12 | +//! - BLAKE3 proof hashing throughput at different content sizes |
| 13 | +//! |
| 14 | +//! Run with: `cargo bench --bench routing_benchmarks` |
| 15 | +//! Generates HTML reports in `target/criterion/` |
| 16 | +
|
| 17 | +use criterion::{black_box, criterion_group, criterion_main, BenchmarkId, Criterion}; |
| 18 | + |
| 19 | +use echidna::agent::AgentConfig; |
| 20 | +use echidna::dispatch::DispatchConfig; |
| 21 | +use echidna::provers::{ProverConfig, ProverFactory, ProverKind}; |
| 22 | +use echidna::verification::axiom_tracker::{AxiomTracker, DangerLevel}; |
| 23 | +use echidna::verification::confidence::{compute_trust_level, TrustFactors, TrustLevel}; |
| 24 | + |
| 25 | +// ============================================================================ |
| 26 | +// Routing Decision Latency |
| 27 | +// |
| 28 | +// Measures the time to select a prover from content. In production this |
| 29 | +// happens for every incoming proof request before the prover is invoked. |
| 30 | +// ============================================================================ |
| 31 | + |
| 32 | +/// Benchmark: deterministic prover selection from content hash. |
| 33 | +/// |
| 34 | +/// This is the critical hot path for routing: a proof request arrives, |
| 35 | +/// the system must decide which of 48 backends to dispatch to. |
| 36 | +fn bench_routing_decision_latency(c: &mut Criterion) { |
| 37 | + let mut group = c.benchmark_group("routing_decision_latency"); |
| 38 | + |
| 39 | + let test_contents = [ |
| 40 | + ("short_smt", "(assert (= x x))"), |
| 41 | + ( |
| 42 | + "medium_lean4", |
| 43 | + "theorem comm (a b : Nat) : a + b = b + a := Nat.add_comm a b", |
| 44 | + ), |
| 45 | + ( |
| 46 | + "long_coq", |
| 47 | + "Theorem plus_comm : forall n m : nat, n + m = m + n. \ |
| 48 | + Proof. intros n m. induction n as [| n' IHn']. - simpl. rewrite <- plus_n_O. reflexivity. \ |
| 49 | + - simpl. rewrite -> IHn'. rewrite <- plus_n_Sm. reflexivity. Qed.", |
| 50 | + ), |
| 51 | + ]; |
| 52 | + |
| 53 | + for (label, content) in test_contents { |
| 54 | + group.bench_with_input( |
| 55 | + BenchmarkId::new("select_prover", label), |
| 56 | + &content, |
| 57 | + |b, content| { |
| 58 | + b.iter(|| { |
| 59 | + // Deterministic hash-based prover selection — the routing hot path. |
| 60 | + let hash: usize = content |
| 61 | + .as_bytes() |
| 62 | + .iter() |
| 63 | + .fold(0usize, |acc, &byte| acc.wrapping_mul(31).wrapping_add(byte as usize)); |
| 64 | + let provers = black_box(ProverKind::all()); |
| 65 | + let selected = provers[hash % provers.len()]; |
| 66 | + black_box(selected) |
| 67 | + }) |
| 68 | + }, |
| 69 | + ); |
| 70 | + } |
| 71 | + |
| 72 | + group.finish(); |
| 73 | +} |
| 74 | + |
| 75 | +// ============================================================================ |
| 76 | +// Axiom Scan Throughput |
| 77 | +// |
| 78 | +// The axiom scanner is called for every proof result before computing trust. |
| 79 | +// Its performance directly affects end-to-end proof verification latency. |
| 80 | +// ============================================================================ |
| 81 | + |
| 82 | +/// Benchmark: axiom scanning at various proof content sizes. |
| 83 | +fn bench_axiom_scan_throughput(c: &mut Criterion) { |
| 84 | + let mut group = c.benchmark_group("axiom_scan_throughput"); |
| 85 | + |
| 86 | + let sizes = [64, 256, 1024, 4096]; |
| 87 | + |
| 88 | + for size in sizes { |
| 89 | + // Generate a proof content string of approximately `size` bytes. |
| 90 | + let content: String = std::iter::repeat("(assert (= x x))\n") |
| 91 | + .flat_map(|s| s.chars()) |
| 92 | + .take(size) |
| 93 | + .collect(); |
| 94 | + |
| 95 | + group.bench_with_input( |
| 96 | + BenchmarkId::new("scan_bytes", size), |
| 97 | + &content, |
| 98 | + |b, content| { |
| 99 | + b.iter(|| { |
| 100 | + let tracker = AxiomTracker::new(); |
| 101 | + let result = tracker.scan(ProverKind::Z3, black_box(content)); |
| 102 | + black_box(result) |
| 103 | + }) |
| 104 | + }, |
| 105 | + ); |
| 106 | + } |
| 107 | + |
| 108 | + group.finish(); |
| 109 | +} |
| 110 | + |
| 111 | +// ============================================================================ |
| 112 | +// Trust Level Computation Latency |
| 113 | +// |
| 114 | +// Trust level computation happens after every proof verification. It must |
| 115 | +// be fast because it sits on the critical path. |
| 116 | +// ============================================================================ |
| 117 | + |
| 118 | +/// Benchmark: trust level computation under various factor combinations. |
| 119 | +fn bench_trust_level_computation(c: &mut Criterion) { |
| 120 | + let mut group = c.benchmark_group("trust_level_computation"); |
| 121 | + |
| 122 | + let factor_sets = [ |
| 123 | + ( |
| 124 | + "single_prover_no_cert", |
| 125 | + TrustFactors { |
| 126 | + prover: ProverKind::Z3, |
| 127 | + confirming_provers: 1, |
| 128 | + has_certificate: false, |
| 129 | + certificate_verified: false, |
| 130 | + worst_axiom_danger: DangerLevel::Safe, |
| 131 | + solver_integrity_ok: true, |
| 132 | + portfolio_confidence: None, |
| 133 | + }, |
| 134 | + ), |
| 135 | + ( |
| 136 | + "cross_checked_with_cert", |
| 137 | + TrustFactors { |
| 138 | + prover: ProverKind::Lean, |
| 139 | + confirming_provers: 3, |
| 140 | + has_certificate: true, |
| 141 | + certificate_verified: true, |
| 142 | + worst_axiom_danger: DangerLevel::Safe, |
| 143 | + solver_integrity_ok: true, |
| 144 | + portfolio_confidence: None, |
| 145 | + }, |
| 146 | + ), |
| 147 | + ( |
| 148 | + "reject_axiom_fast_path", |
| 149 | + TrustFactors { |
| 150 | + prover: ProverKind::Z3, |
| 151 | + confirming_provers: 5, |
| 152 | + has_certificate: true, |
| 153 | + certificate_verified: true, |
| 154 | + worst_axiom_danger: DangerLevel::Reject, |
| 155 | + solver_integrity_ok: true, |
| 156 | + portfolio_confidence: None, |
| 157 | + }, |
| 158 | + ), |
| 159 | + ]; |
| 160 | + |
| 161 | + for (label, factors) in factor_sets { |
| 162 | + group.bench_with_input( |
| 163 | + BenchmarkId::new("compute_trust", label), |
| 164 | + &factors, |
| 165 | + |b, f| { |
| 166 | + b.iter(|| { |
| 167 | + let level = compute_trust_level(black_box(f)); |
| 168 | + black_box(level) |
| 169 | + }) |
| 170 | + }, |
| 171 | + ); |
| 172 | + } |
| 173 | + |
| 174 | + group.finish(); |
| 175 | +} |
| 176 | + |
| 177 | +// ============================================================================ |
| 178 | +// ProverFactory Instantiation Cost |
| 179 | +// |
| 180 | +// Measures the cold-start cost of creating a prover backend. This is relevant |
| 181 | +// for server startup and for lazy initialisation strategies. |
| 182 | +// ============================================================================ |
| 183 | + |
| 184 | +/// Benchmark: ProverFactory::create for different backends. |
| 185 | +fn bench_prover_factory_instantiation(c: &mut Criterion) { |
| 186 | + let mut group = c.benchmark_group("prover_factory_instantiation"); |
| 187 | + |
| 188 | + let backends = [ |
| 189 | + ("z3", ProverKind::Z3), |
| 190 | + ("lean", ProverKind::Lean), |
| 191 | + ("coq", ProverKind::Coq), |
| 192 | + ("idris2", ProverKind::Idris2), |
| 193 | + ("agda", ProverKind::Agda), |
| 194 | + ]; |
| 195 | + |
| 196 | + let config = ProverConfig::default(); |
| 197 | + |
| 198 | + for (label, kind) in backends { |
| 199 | + group.bench_with_input( |
| 200 | + BenchmarkId::new("create_backend", label), |
| 201 | + &kind, |
| 202 | + |b, &kind| { |
| 203 | + b.iter(|| { |
| 204 | + let prover = ProverFactory::create(black_box(kind), config.clone()); |
| 205 | + black_box(prover) |
| 206 | + }) |
| 207 | + }, |
| 208 | + ); |
| 209 | + } |
| 210 | + |
| 211 | + group.finish(); |
| 212 | +} |
| 213 | + |
| 214 | +// ============================================================================ |
| 215 | +// Agentic Planning Throughput |
| 216 | +// |
| 217 | +// Measures how quickly AgentConfig can be constructed and cloned. |
| 218 | +// In the agentic planner, a new config may be derived for each sub-goal. |
| 219 | +// ============================================================================ |
| 220 | + |
| 221 | +/// Benchmark: AgentConfig construction and cloning throughput. |
| 222 | +fn bench_agentic_config_throughput(c: &mut Criterion) { |
| 223 | + let mut group = c.benchmark_group("agentic_planning_throughput"); |
| 224 | + |
| 225 | + // Benchmark: construction from scratch. |
| 226 | + group.bench_function("agent_config_construct", |b| { |
| 227 | + b.iter(|| { |
| 228 | + black_box(AgentConfig { |
| 229 | + max_concurrent: 8, |
| 230 | + max_attempts: 3, |
| 231 | + timeout_secs: 120, |
| 232 | + neural_enabled: true, |
| 233 | + reflection_enabled: true, |
| 234 | + planning_enabled: true, |
| 235 | + }) |
| 236 | + }) |
| 237 | + }); |
| 238 | + |
| 239 | + // Benchmark: clone from existing config. |
| 240 | + let base_config = AgentConfig::default(); |
| 241 | + group.bench_function("agent_config_clone", |b| { |
| 242 | + b.iter(|| { |
| 243 | + black_box(base_config.clone()) |
| 244 | + }) |
| 245 | + }); |
| 246 | + |
| 247 | + // Benchmark: JSON serialisation (used for config logging and audit). |
| 248 | + group.bench_function("agent_config_serialise", |b| { |
| 249 | + b.iter(|| { |
| 250 | + let json = serde_json::to_string(black_box(&base_config)).unwrap(); |
| 251 | + black_box(json) |
| 252 | + }) |
| 253 | + }); |
| 254 | + |
| 255 | + group.finish(); |
| 256 | +} |
| 257 | + |
| 258 | +// ============================================================================ |
| 259 | +// BLAKE3 Proof Hashing Throughput |
| 260 | +// |
| 261 | +// Proof identity hashing is on the critical path for certificate verification |
| 262 | +// and deduplication. This benchmark baselines the cost per proof size. |
| 263 | +// ============================================================================ |
| 264 | + |
| 265 | +/// Benchmark: BLAKE3 proof hashing at various content sizes. |
| 266 | +fn bench_proof_hash_throughput(c: &mut Criterion) { |
| 267 | + let mut group = c.benchmark_group("proof_hash_throughput"); |
| 268 | + |
| 269 | + let sizes = [64usize, 256, 1024, 4096, 16384]; |
| 270 | + |
| 271 | + for size in sizes { |
| 272 | + let content: Vec<u8> = (0..size).map(|i| (i % 256) as u8).collect(); |
| 273 | + group.bench_with_input( |
| 274 | + BenchmarkId::new("blake3_hash_bytes", size), |
| 275 | + &content, |
| 276 | + |b, data| { |
| 277 | + b.iter(|| { |
| 278 | + let hash = blake3::hash(black_box(data)); |
| 279 | + black_box(hash) |
| 280 | + }) |
| 281 | + }, |
| 282 | + ); |
| 283 | + } |
| 284 | + |
| 285 | + group.finish(); |
| 286 | +} |
| 287 | + |
| 288 | +// ============================================================================ |
| 289 | +// DispatchConfig Construction |
| 290 | +// |
| 291 | +// DispatchConfig is constructed per-request in the routing layer. |
| 292 | +// Its construction cost should be negligible. |
| 293 | +// ============================================================================ |
| 294 | + |
| 295 | +/// Benchmark: DispatchConfig construction and default/custom. |
| 296 | +fn bench_dispatch_config_construction(c: &mut Criterion) { |
| 297 | + let mut group = c.benchmark_group("dispatch_config_construction"); |
| 298 | + |
| 299 | + group.bench_function("default_config", |b| { |
| 300 | + b.iter(|| black_box(DispatchConfig::default())) |
| 301 | + }); |
| 302 | + |
| 303 | + group.bench_function("custom_config", |b| { |
| 304 | + b.iter(|| { |
| 305 | + black_box(DispatchConfig { |
| 306 | + cross_check: true, |
| 307 | + min_trust_level: TrustLevel::Level4, |
| 308 | + track_axioms: true, |
| 309 | + generate_certificates: true, |
| 310 | + timeout: 60, |
| 311 | + }) |
| 312 | + }) |
| 313 | + }); |
| 314 | + |
| 315 | + group.finish(); |
| 316 | +} |
| 317 | + |
| 318 | +// ============================================================================ |
| 319 | +// Criterion groups |
| 320 | +// ============================================================================ |
| 321 | + |
| 322 | +criterion_group!( |
| 323 | + routing_benches, |
| 324 | + bench_routing_decision_latency, |
| 325 | + bench_axiom_scan_throughput, |
| 326 | + bench_trust_level_computation, |
| 327 | + bench_prover_factory_instantiation, |
| 328 | + bench_agentic_config_throughput, |
| 329 | + bench_proof_hash_throughput, |
| 330 | + bench_dispatch_config_construction, |
| 331 | +); |
| 332 | + |
| 333 | +criterion_main!(routing_benches); |
0 commit comments