feat(eval): gate feedback coverage and lifecycle accuracy

Author: haasonsaas

TODO.md

@@ -146,7 +146,7 @@ This roadmap is derived from deep research into Greptile's public docs, blog, MC
 95. [ ] Add eval runs that compare single-pass review against agentic loop review.
 96. [ ] Add production replay evals using anonymized accepted/rejected review outcomes.
 97. [ ] Add leaderboard reporting for reviewer usefulness metrics, not just precision/recall.
-98. [ ] Add regression gates for feedback coverage, verifier health, and lifecycle-state accuracy.
+98. [x] Add regression gates for feedback coverage, verifier health, and lifecycle-state accuracy.
 99. [ ] Add model-routing policies that explicitly separate generation, verification, and auditing roles.
 100. [ ] Publish a repeatable "independent auditor" benchmark story in the UI and CLI so DiffScope's differentiation is measurable.
 

src/commands/eval/command/batch.rs

@@ -321,6 +321,7 @@ mod tests {
             min_micro_f1: None,
             min_macro_f1: None,
             min_verification_health: None,
+            min_lifecycle_accuracy: None,
             min_rule_f1: vec![],
             max_rule_f1_drop: vec![],
             matrix_models: vec![],

src/commands/eval/command/fixtures.rs

@@ -219,6 +219,7 @@ mod tests {
                 min_micro_f1: None,
                 min_macro_f1: None,
                 min_verification_health: None,
+                min_lifecycle_accuracy: None,
                 min_rule_f1: Vec::new(),
                 max_rule_f1_drop: Vec::new(),
                 matrix_models: Vec::new(),
@@ -262,6 +263,7 @@ mod tests {
                 min_micro_f1: None,
                 min_macro_f1: None,
                 min_verification_health: None,
+                min_lifecycle_accuracy: None,
                 min_rule_f1: Vec::new(),
                 max_rule_f1_drop: Vec::new(),
                 matrix_models: Vec::new(),

src/commands/eval/command/options.rs

@@ -37,6 +37,7 @@ pub(super) fn prepare_eval_options(
             min_micro_f1: options.min_micro_f1,
             min_macro_f1: options.min_macro_f1,
             min_verification_health: options.min_verification_health,
+            min_lifecycle_accuracy: options.min_lifecycle_accuracy,
             min_rule_f1: min_rule_thresholds,
             max_rule_f1_drop: max_rule_drop_thresholds,
         },

src/commands/eval/metrics.rs

@@ -1,5 +1,7 @@
 #[path = "metrics/comparisons.rs"]
 mod comparisons;
+#[path = "metrics/lifecycle.rs"]
+mod lifecycle;
 #[path = "metrics/rules.rs"]
 mod rules;
 #[path = "metrics/suites.rs"]
@@ -8,6 +10,7 @@ mod suites;
 pub(super) use comparisons::{
     build_named_breakdown_comparisons, build_suite_comparisons, build_verification_health,
 };
+pub(super) use lifecycle::build_lifecycle_accuracy;
 pub(super) use rules::{aggregate_rule_metrics, compute_rule_metrics, summarize_rule_metrics};
 pub(super) use suites::{
     build_benchmark_breakdowns, build_overall_benchmark_summary, build_suite_results,

src/commands/eval/metrics/lifecycle.rs

@@ -0,0 +1,87 @@
+use super::super::EvalFixtureResult;
+
+const LIFECYCLE_RULE_PREFIX: &str = "bug.lifecycle.";
+
+#[derive(Debug, Clone, Copy, PartialEq)]
+pub(in super::super) struct EvalPassRate {
+    pub(in super::super) passed: usize,
+    pub(in super::super) total: usize,
+    pub(in super::super) rate: f32,
+}
+
+pub(in super::super) fn build_lifecycle_accuracy(
+    results: &[EvalFixtureResult],
+) -> Option<EvalPassRate> {
+    let total = results
+        .iter()
+        .filter(|result| is_lifecycle_fixture(result))
+        .count();
+    if total == 0 {
+        return None;
+    }
+
+    let passed = results
+        .iter()
+        .filter(|result| is_lifecycle_fixture(result) && result.passed)
+        .count();
+
+    Some(EvalPassRate {
+        passed,
+        total,
+        rate: passed as f32 / total as f32,
+    })
+}
+
+fn is_lifecycle_fixture(result: &EvalFixtureResult) -> bool {
+    result
+        .rule_metrics
+        .iter()
+        .any(|metric| metric.expected > 0 && metric.rule_id.starts_with(LIFECYCLE_RULE_PREFIX))
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::commands::eval::EvalRuleMetrics;
+
+    fn fixture_result(passed: bool, rule_id: &str) -> EvalFixtureResult {
+        EvalFixtureResult {
+            passed,
+            rule_metrics: vec![EvalRuleMetrics {
+                rule_id: rule_id.to_string(),
+                expected: 1,
+                predicted: 1,
+                true_positives: usize::from(passed),
+                false_positives: usize::from(!passed),
+                false_negatives: usize::from(!passed),
+                precision: if passed { 1.0 } else { 0.0 },
+                recall: if passed { 1.0 } else { 0.0 },
+                f1: if passed { 1.0 } else { 0.0 },
+            }],
+            ..Default::default()
+        }
+    }
+
+    #[test]
+    fn build_lifecycle_accuracy_aggregates_lifecycle_fixture_pass_rate() {
+        let accuracy = build_lifecycle_accuracy(&[
+            fixture_result(true, "bug.lifecycle.context-only-addressed"),
+            fixture_result(false, "bug.lifecycle.api-drops-followup-addressed"),
+            fixture_result(true, "bug.readiness.current-head-staleness"),
+        ])
+        .unwrap();
+
+        assert_eq!(accuracy.passed, 1);
+        assert_eq!(accuracy.total, 2);
+        assert!((accuracy.rate - 0.5).abs() < f32::EPSILON);
+    }
+
+    #[test]
+    fn build_lifecycle_accuracy_returns_none_without_lifecycle_rules() {
+        assert!(build_lifecycle_accuracy(&[fixture_result(
+            true,
+            "bug.readiness.current-head-staleness"
+        )])
+        .is_none());
+    }
+}

src/commands/eval/report/output.rs

@@ -1,6 +1,7 @@
 use anyhow::Result;
 use std::path::Path;
 
+use super::super::metrics::build_lifecycle_accuracy;
 use super::super::EvalReport;
 
 pub(in super::super) fn print_eval_report(report: &EvalReport) {
@@ -307,6 +308,15 @@ pub(in super::super) fn print_eval_report(report: &EvalReport) {
         );
     }
 
+    if let Some(accuracy) = build_lifecycle_accuracy(&report.results) {
+        println!(
+            "Lifecycle accuracy: {:.0}% ({}/{})",
+            accuracy.rate * 100.0,
+            accuracy.passed,
+            accuracy.total
+        );
+    }
+
     for warning in &report.warnings {
         println!("Warning: {warning}");
     }

src/commands/eval/thresholds.rs

@@ -12,6 +12,7 @@ pub(super) struct EvalThresholdOptions {
     pub(super) min_micro_f1: Option<f32>,
     pub(super) min_macro_f1: Option<f32>,
     pub(super) min_verification_health: Option<f32>,
+    pub(super) min_lifecycle_accuracy: Option<f32>,
     pub(super) min_rule_f1: Vec<EvalRuleThreshold>,
     pub(super) max_rule_f1_drop: Vec<EvalRuleThreshold>,
 }

src/commands/eval/thresholds/evaluation/run.rs

@@ -3,6 +3,7 @@ use super::super::EvalThresholdOptions;
 use super::drops::check_drop_thresholds;
 use super::minimums::check_minimum_thresholds;
 use super::rules::build_rule_f1_map;
+use crate::commands::eval::metrics::build_lifecycle_accuracy;
 
 pub(in super::super::super) fn evaluate_eval_thresholds(
     current: &EvalReport,
@@ -35,6 +36,16 @@ pub(in super::super::super) fn evaluate_eval_thresholds(
             }
         }
     }
+    if let Some(threshold) = options.min_lifecycle_accuracy {
+        if let Some(accuracy) = build_lifecycle_accuracy(&current.results) {
+            if accuracy.total > 0 && accuracy.rate < threshold {
+                failures.push(format!(
+                    "lifecycle accuracy {:.3} fell below minimum {:.3} ({}/{})",
+                    accuracy.rate, threshold, accuracy.passed, accuracy.total
+                ));
+            }
+        }
+    }
     failures.extend(check_drop_thresholds(
         current,
         current_micro_f1,
@@ -90,6 +101,7 @@ mod tests {
             min_micro_f1: None,
             min_macro_f1: None,
             min_verification_health: None,
+            min_lifecycle_accuracy: None,
             min_rule_f1: vec![],
             max_rule_f1_drop: vec![],
         };
@@ -172,6 +184,7 @@ mod tests {
             min_micro_f1: None,
             min_macro_f1: None,
             min_verification_health: None,
+            min_lifecycle_accuracy: None,
             min_rule_f1: vec![],
             max_rule_f1_drop: vec![EvalRuleThreshold {
                 rule_id: "sec.sql.injection".to_string(),
@@ -233,6 +246,7 @@ mod tests {
             min_micro_f1: None,
             min_macro_f1: None,
             min_verification_health: None,
+            min_lifecycle_accuracy: None,
             min_rule_f1: vec![],
             max_rule_f1_drop: vec![],
         };
@@ -279,6 +293,7 @@ mod tests {
             min_micro_f1: None,
             min_macro_f1: None,
             min_verification_health: Some(0.8),
+            min_lifecycle_accuracy: None,
             min_rule_f1: vec![],
             max_rule_f1_drop: vec![],
         };
@@ -290,4 +305,76 @@ mod tests {
         assert!(failures[0].contains("minimum 0.800"));
         assert!(failures[0].contains("7/10"));
     }
+
+    #[test]
+    fn test_evaluate_eval_thresholds_checks_lifecycle_accuracy() {
+        let current = EvalReport {
+            run: Default::default(),
+            fixtures_total: 2,
+            fixtures_passed: 1,
+            fixtures_failed: 1,
+            rule_metrics: vec![],
+            rule_summary: Some(EvalRuleScoreSummary::default()),
+            benchmark_summary: None,
+            suite_results: vec![],
+            benchmark_by_category: Default::default(),
+            benchmark_by_language: Default::default(),
+            benchmark_by_difficulty: Default::default(),
+            suite_comparisons: vec![],
+            category_comparisons: vec![],
+            language_comparisons: vec![],
+            verification_health: None,
+            warnings: vec![],
+            threshold_failures: vec![],
+            results: vec![
+                crate::commands::eval::EvalFixtureResult {
+                    passed: true,
+                    rule_metrics: vec![EvalRuleMetrics {
+                        rule_id: "bug.lifecycle.context-only-addressed".to_string(),
+                        expected: 1,
+                        predicted: 1,
+                        true_positives: 1,
+                        false_positives: 0,
+                        false_negatives: 0,
+                        precision: 1.0,
+                        recall: 1.0,
+                        f1: 1.0,
+                    }],
+                    ..Default::default()
+                },
+                crate::commands::eval::EvalFixtureResult {
+                    passed: false,
+                    rule_metrics: vec![EvalRuleMetrics {
+                        rule_id: "bug.lifecycle.api-drops-followup-addressed".to_string(),
+                        expected: 1,
+                        predicted: 0,
+                        true_positives: 0,
+                        false_positives: 0,
+                        false_negatives: 1,
+                        precision: 0.0,
+                        recall: 0.0,
+                        f1: 0.0,
+                    }],
+                    ..Default::default()
+                },
+            ],
+        };
+        let options = EvalThresholdOptions {
+            max_micro_f1_drop: None,
+            max_suite_f1_drop: None,
+            max_category_f1_drop: None,
+            max_language_f1_drop: None,
+            min_micro_f1: None,
+            min_macro_f1: None,
+            min_verification_health: None,
+            min_lifecycle_accuracy: Some(0.8),
+            min_rule_f1: vec![],
+            max_rule_f1_drop: vec![],
+        };
+
+        let failures = evaluate_eval_thresholds(&current, None, &options);
+
+        assert_eq!(failures.len(), 1);
+        assert!(failures[0].contains("lifecycle accuracy 0.500"));
+    }
 }

src/commands/eval/types/options.rs

@@ -10,6 +10,7 @@ pub struct EvalRunOptions {
     pub min_micro_f1: Option<f32>,
     pub min_macro_f1: Option<f32>,
     pub min_verification_health: Option<f32>,
+    pub min_lifecycle_accuracy: Option<f32>,
     pub min_rule_f1: Vec<String>,
     pub max_rule_f1_drop: Vec<String>,
     pub matrix_models: Vec<String>,