From 42b4102f0b79cc211671738a3439a43ffa157228 Mon Sep 17 00:00:00 2001 From: WaffleBits <19478926+WaffleBits@users.noreply.github.com> Date: Wed, 17 Jun 2026 10:28:46 -0400 Subject: [PATCH] add numeric release gates --- .github/workflows/ci.yml | 2 +- README.md | 16 +- artifacts/release-gate-numeric-tolerance.json | 38 ++ artifacts/release-gate-promote.json | 22 +- artifacts/release-gate-rollback.json | 22 +- docs/ARCHITECTURE.md | 12 +- docs/RELEASE_VALIDATION.md | 26 +- fixtures/release_gate_numeric_tolerance.json | 102 +++++ src/lib.rs | 3 +- src/release.rs | 394 +++++++++++++++++- tests/release.rs | 103 ++++- 11 files changed, 715 insertions(+), 25 deletions(-) create mode 100644 artifacts/release-gate-numeric-tolerance.json create mode 100644 fixtures/release_gate_numeric_tolerance.json diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 16a2453..e03f857 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -28,4 +28,4 @@ jobs: run: | cargo run --release -- gate --input fixtures/release_gate_safe.json --output target/release-gate-promote.json cargo run --release -- gate --input fixtures/release_gate_bad.json --output target/release-gate-rollback.json - + cargo run --release -- gate --input fixtures/release_gate_numeric_tolerance.json --output target/release-gate-numeric-tolerance.json diff --git a/README.md b/README.md index a42f417..cdd9890 100644 --- a/README.md +++ b/README.md @@ -16,8 +16,9 @@ replayable scheduling traces, and canary/shadow release decisions. - Deterministic workload replay with a machine-readable trace fingerprint. - Baseline/candidate release validation with `promote`, `hold`, and `rollback` outcomes. -- Exact output checks, error-rate deltas, p95 latency regression policy, tests, - and CI. +- Exact output checks, model-aware numeric tolerances for backend drift, + per-segment release summaries, error-rate deltas, p95 latency regression + policy, tests, and CI. ## Quick Start @@ -35,10 +36,16 @@ cargo run --release -- gate \ cargo run --release -- gate \ --input fixtures/release_gate_bad.json \ --output artifacts/release-gate-rollback.json + +cargo run --release -- gate \ + --input fixtures/release_gate_numeric_tolerance.json \ + --output artifacts/release-gate-numeric-tolerance.json ``` The safe fixture produces `promote`. The candidate with an output mismatch and an added error produces `rollback`. +The numeric-tolerance fixture produces `promote` while reporting four tolerated +numeric comparisons across a baseline-runtime to candidate-runtime segment. The checked workload fixture completes four requests in 11 scheduler ticks, peaks at 12 of 20 KV pages, returns all pages on completion, and emits trace @@ -65,10 +72,15 @@ ticks, and completion count. ## Release Policy The gate joins mirrored baseline and candidate observations by request ID. +Outputs can be validated either by exact fingerprint or by a configured +numeric tolerance scoped to model, candidate backend, and accelerator. Reports +include aggregate metrics plus segment summaries so hardware/backend-specific +regressions remain visible. | Signal | Response | |---|---| | Output mismatch above policy | `rollback` | +| Numeric drift above model/backend policy | `rollback` | | Error-rate increase above policy | `rollback` | | p95 latency regression above policy | `hold` | | Missing or insufficient matched traffic | `hold` | diff --git a/artifacts/release-gate-numeric-tolerance.json b/artifacts/release-gate-numeric-tolerance.json new file mode 100644 index 0000000..6b1b169 --- /dev/null +++ b/artifacts/release-gate-numeric-tolerance.json @@ -0,0 +1,38 @@ +{ + "schema_version": 2, + "decision": "promote", + "matched_requests": 4, + "baseline_requests": 4, + "candidate_requests": 4, + "coverage_rate": 1.0, + "output_mismatch_rate": 0.0, + "numeric_pairs": 4, + "tolerated_numeric_outputs": 4, + "numeric_drift_rate": 0.0, + "max_numeric_abs_error": 0.001, + "max_numeric_rel_error": 0.008, + "baseline_error_rate": 0.0, + "candidate_error_rate": 0.0, + "error_rate_increase": 0.0, + "baseline_p95_latency_ms": 28.0, + "candidate_p95_latency_ms": 27.6, + "p95_latency_regression_pct": -1.428571, + "segments": [ + { + "model": "decoder-7b", + "baseline_backend": "baseline-runtime", + "candidate_backend": "candidate-runtime", + "accelerator": "h100", + "matched_requests": 4, + "output_mismatch_rate": 0.0, + "baseline_error_rate": 0.0, + "candidate_error_rate": 0.0, + "baseline_p95_latency_ms": 28.0, + "candidate_p95_latency_ms": 27.6, + "p95_latency_regression_pct": -1.428571 + } + ], + "reasons": [ + "candidate stayed within correctness, reliability, and latency policy" + ] +} diff --git a/artifacts/release-gate-promote.json b/artifacts/release-gate-promote.json index f19104c..b02dc13 100644 --- a/artifacts/release-gate-promote.json +++ b/artifacts/release-gate-promote.json @@ -1,17 +1,37 @@ { - "schema_version": 1, + "schema_version": 2, "decision": "promote", "matched_requests": 4, "baseline_requests": 4, "candidate_requests": 4, "coverage_rate": 1.0, "output_mismatch_rate": 0.0, + "numeric_pairs": 0, + "tolerated_numeric_outputs": 0, + "numeric_drift_rate": 0.0, + "max_numeric_abs_error": null, + "max_numeric_rel_error": null, "baseline_error_rate": 0.0, "candidate_error_rate": 0.0, "error_rate_increase": 0.0, "baseline_p95_latency_ms": 16.0, "candidate_p95_latency_ms": 16.7, "p95_latency_regression_pct": 4.375, + "segments": [ + { + "model": "unspecified", + "baseline_backend": "unspecified", + "candidate_backend": "unspecified", + "accelerator": "unspecified", + "matched_requests": 4, + "output_mismatch_rate": 0.0, + "baseline_error_rate": 0.0, + "candidate_error_rate": 0.0, + "baseline_p95_latency_ms": 16.0, + "candidate_p95_latency_ms": 16.7, + "p95_latency_regression_pct": 4.375 + } + ], "reasons": [ "candidate stayed within correctness, reliability, and latency policy" ] diff --git a/artifacts/release-gate-rollback.json b/artifacts/release-gate-rollback.json index aa2f696..1b5fb75 100644 --- a/artifacts/release-gate-rollback.json +++ b/artifacts/release-gate-rollback.json @@ -1,17 +1,37 @@ { - "schema_version": 1, + "schema_version": 2, "decision": "rollback", "matched_requests": 4, "baseline_requests": 4, "candidate_requests": 4, "coverage_rate": 1.0, "output_mismatch_rate": 0.333333, + "numeric_pairs": 0, + "tolerated_numeric_outputs": 0, + "numeric_drift_rate": 0.0, + "max_numeric_abs_error": null, + "max_numeric_rel_error": null, "baseline_error_rate": 0.0, "candidate_error_rate": 0.25, "error_rate_increase": 0.25, "baseline_p95_latency_ms": 16.0, "candidate_p95_latency_ms": 14.5, "p95_latency_regression_pct": -9.375, + "segments": [ + { + "model": "unspecified", + "baseline_backend": "unspecified", + "candidate_backend": "unspecified", + "accelerator": "unspecified", + "matched_requests": 4, + "output_mismatch_rate": 0.333333, + "baseline_error_rate": 0.0, + "candidate_error_rate": 0.25, + "baseline_p95_latency_ms": 16.0, + "candidate_p95_latency_ms": 14.5, + "p95_latency_regression_pct": -9.375 + } + ], "reasons": [ "output mismatch rate 0.3333 exceeded 0.0000", "error-rate increase 0.2500 exceeded 0.0100" diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 44d464e..adf43f8 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -36,15 +36,17 @@ It computes: - request coverage; - exact output-fingerprint mismatch rate; +- model/backend-scoped numeric drift when fingerprints differ; - error-rate increase; - successful-request p95 latency; and -- candidate p95 regression. +- candidate p95 regression; and +- segment summaries by model, baseline backend, candidate backend, and + accelerator. -Correctness or reliability regressions produce `rollback`. Latency regressions -or incomplete evidence produce `hold`. A complete candidate within policy -produces `promote`. +Correctness, numeric drift, or reliability regressions produce `rollback`. +Latency regressions or incomplete evidence produce `hold`. A complete candidate +within policy produces `promote`. This is a local validation component, not a deployment controller. Production integration would obtain observations from mirrored traffic, canary populations, telemetry, and an audited rollout system. - diff --git a/docs/RELEASE_VALIDATION.md b/docs/RELEASE_VALIDATION.md index 435c47b..9f496f9 100644 --- a/docs/RELEASE_VALIDATION.md +++ b/docs/RELEASE_VALIDATION.md @@ -8,6 +8,7 @@ Use `promote` when: - baseline and candidate request coverage is complete; - output fingerprints stay within the configured mismatch budget; +- configured numeric tolerances cover any expected model/backend output drift; - the candidate error-rate increase stays within policy; and - candidate p95 latency stays within the configured regression budget. @@ -23,17 +24,36 @@ Use `rollback` when the candidate changes successful outputs beyond policy or increases the request error rate beyond policy. These checks intentionally take precedence over latency. +## Numeric Tolerance + +Some accelerator backends legitimately produce small numeric differences even +when the generated output is acceptable. `numeric_tolerances` allow those +comparisons to be explicit instead of hidden behind broad fingerprint waivers. +Each tolerance can be scoped by model, candidate backend, and candidate +accelerator. + +When fingerprints differ, the gate promotes only if both observations provide +numeric output vectors of equal length and every value stays within either the +absolute or relative tolerance. Failed numeric comparisons count as output +mismatches and as numeric drift. + +The report includes: + +- `numeric_pairs` and `tolerated_numeric_outputs`; +- `numeric_drift_rate`; +- maximum absolute and relative numeric error; and +- segment summaries by model, baseline backend, candidate backend, and + accelerator. + ## Production Extension Points A real rollout system should add: - statistically meaningful sample sizes and confidence intervals; -- model-aware numeric tolerance rather than exact fingerprints where required; -- segmented checks by model, hardware, prompt class, and region; +- prompt-class and region segmentation; - SLO burn-rate and saturation signals; - canary population controls and audited rollback execution; and - provenance linking every decision to build, model, and configuration IDs. The checked fixtures are synthetic and exist to make the policy executable in CI. They are not claims about production traffic or fleet scale. - diff --git a/fixtures/release_gate_numeric_tolerance.json b/fixtures/release_gate_numeric_tolerance.json new file mode 100644 index 0000000..2eb2869 --- /dev/null +++ b/fixtures/release_gate_numeric_tolerance.json @@ -0,0 +1,102 @@ +{ + "thresholds": { + "min_matched_requests": 4, + "max_output_mismatch_rate": 0.0, + "max_error_rate_increase": 0.01, + "max_p95_latency_regression_pct": 10.0, + "max_numeric_drift_rate": 0.0, + "numeric_tolerances": [ + { + "model": "decoder-7b", + "candidate_backend": "candidate-runtime", + "candidate_accelerator": "h100", + "max_abs_error": 0.002, + "max_rel_error": 0.01 + } + ] + }, + "baseline": [ + { + "request_id": "prompt-a", + "output_fingerprint": "baseline-a", + "latency_ms": 18.0, + "ok": true, + "model": "decoder-7b", + "backend": "baseline-runtime", + "accelerator": "h100", + "output_values": [0.125, 0.5, 1.25] + }, + { + "request_id": "prompt-b", + "output_fingerprint": "baseline-b", + "latency_ms": 21.0, + "ok": true, + "model": "decoder-7b", + "backend": "baseline-runtime", + "accelerator": "h100", + "output_values": [0.25, 0.75, 1.5] + }, + { + "request_id": "prompt-c", + "output_fingerprint": "baseline-c", + "latency_ms": 25.0, + "ok": true, + "model": "decoder-7b", + "backend": "baseline-runtime", + "accelerator": "h100", + "output_values": [0.375, 1.0, 1.75] + }, + { + "request_id": "prompt-d", + "output_fingerprint": "baseline-d", + "latency_ms": 28.0, + "ok": true, + "model": "decoder-7b", + "backend": "baseline-runtime", + "accelerator": "h100", + "output_values": [0.5, 1.25, 2.0] + } + ], + "candidate": [ + { + "request_id": "prompt-a", + "output_fingerprint": "candidate-a", + "latency_ms": 18.2, + "ok": true, + "model": "decoder-7b", + "backend": "candidate-runtime", + "accelerator": "h100", + "output_values": [0.126, 0.501, 1.251] + }, + { + "request_id": "prompt-b", + "output_fingerprint": "candidate-b", + "latency_ms": 20.8, + "ok": true, + "model": "decoder-7b", + "backend": "candidate-runtime", + "accelerator": "h100", + "output_values": [0.251, 0.751, 1.501] + }, + { + "request_id": "prompt-c", + "output_fingerprint": "candidate-c", + "latency_ms": 24.4, + "ok": true, + "model": "decoder-7b", + "backend": "candidate-runtime", + "accelerator": "h100", + "output_values": [0.376, 1.001, 1.751] + }, + { + "request_id": "prompt-d", + "output_fingerprint": "candidate-d", + "latency_ms": 27.6, + "ok": true, + "model": "decoder-7b", + "backend": "candidate-runtime", + "accelerator": "h100", + "output_values": [0.501, 1.251, 2.001] + } + ] +} diff --git a/src/lib.rs b/src/lib.rs index 70cd79f..c40e22d 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -2,7 +2,8 @@ pub mod release; pub mod scheduler; pub use release::{ - GateDecision, GateInput, GateReport, GateThresholds, Observation, evaluate_release, + GateDecision, GateInput, GateReport, GateThresholds, NumericTolerance, Observation, + SegmentReport, evaluate_release, }; pub use scheduler::{ ReplayInput, ReplayReport, RequestSpec, RuntimeError, Scheduler, SchedulerConfig, TickTrace, diff --git a/src/release.rs b/src/release.rs index 8705310..a1a45e1 100644 --- a/src/release.rs +++ b/src/release.rs @@ -8,6 +8,53 @@ pub struct Observation { pub output_fingerprint: String, pub latency_ms: f64, pub ok: bool, + #[serde(default)] + pub model: Option, + #[serde(default)] + pub backend: Option, + #[serde(default)] + pub accelerator: Option, + #[serde(default)] + pub output_values: Option>, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct NumericTolerance { + #[serde(default)] + pub model: Option, + #[serde(default)] + pub candidate_backend: Option, + #[serde(default)] + pub candidate_accelerator: Option, + pub max_abs_error: f64, + pub max_rel_error: f64, +} + +impl NumericTolerance { + fn matches(&self, baseline: &Observation, candidate: &Observation) -> bool { + if !same_optional_value(&baseline.model, &candidate.model) { + return false; + } + + let model = candidate.model.as_ref().or(baseline.model.as_ref()); + if !matches_filter(&self.model, model) { + return false; + } + if !matches_filter(&self.candidate_backend, candidate.backend.as_ref()) { + return false; + } + if !matches_filter(&self.candidate_accelerator, candidate.accelerator.as_ref()) { + return false; + } + + self.max_abs_error >= 0.0 && self.max_rel_error >= 0.0 + } + + fn specificity(&self) -> usize { + usize::from(self.model.is_some()) + + usize::from(self.candidate_backend.is_some()) + + usize::from(self.candidate_accelerator.is_some()) + } } #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] @@ -16,6 +63,10 @@ pub struct GateThresholds { pub max_output_mismatch_rate: f64, pub max_error_rate_increase: f64, pub max_p95_latency_regression_pct: f64, + #[serde(default)] + pub max_numeric_drift_rate: f64, + #[serde(default)] + pub numeric_tolerances: Vec, } impl Default for GateThresholds { @@ -25,6 +76,8 @@ impl Default for GateThresholds { max_output_mismatch_rate: 0.0, max_error_rate_increase: 0.01, max_p95_latency_regression_pct: 10.0, + max_numeric_drift_rate: 0.0, + numeric_tolerances: Vec::new(), } } } @@ -45,6 +98,21 @@ pub enum GateDecision { Rollback, } +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct SegmentReport { + pub model: String, + pub baseline_backend: String, + pub candidate_backend: String, + pub accelerator: String, + pub matched_requests: usize, + pub output_mismatch_rate: f64, + pub baseline_error_rate: f64, + pub candidate_error_rate: f64, + pub baseline_p95_latency_ms: Option, + pub candidate_p95_latency_ms: Option, + pub p95_latency_regression_pct: Option, +} + #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] pub struct GateReport { pub schema_version: u32, @@ -54,12 +122,18 @@ pub struct GateReport { pub candidate_requests: usize, pub coverage_rate: f64, pub output_mismatch_rate: f64, + pub numeric_pairs: usize, + pub tolerated_numeric_outputs: usize, + pub numeric_drift_rate: f64, + pub max_numeric_abs_error: Option, + pub max_numeric_rel_error: Option, pub baseline_error_rate: f64, pub candidate_error_rate: f64, pub error_rate_increase: f64, pub baseline_p95_latency_ms: Option, pub candidate_p95_latency_ms: Option, pub p95_latency_regression_pct: Option, + pub segments: Vec, pub reasons: Vec, } @@ -103,11 +177,50 @@ pub fn evaluate_release(input: &GateInput) -> GateReport { .copied() .filter(|(baseline, candidate)| baseline.ok && candidate.ok) .collect(); - let mismatches = comparable_outputs + let matched_comparisons: Vec> = matched + .iter() + .map(|(baseline, candidate)| { + if baseline.ok && candidate.ok { + Some(compare_outputs(baseline, candidate, &input.thresholds)) + } else { + None + } + }) + .collect(); + let comparisons: Vec = + matched_comparisons.iter().flatten().copied().collect(); + let mismatches = comparisons .iter() - .filter(|(baseline, candidate)| baseline.output_fingerprint != candidate.output_fingerprint) + .filter(|comparison| !comparison.matches) .count(); let output_mismatch_rate = ratio(mismatches, comparable_outputs.len()); + let numeric_pairs = comparisons + .iter() + .filter(|comparison| comparison.used_numeric) + .count(); + let tolerated_numeric_outputs = comparisons + .iter() + .filter(|comparison| comparison.tolerated_numeric) + .count(); + let numeric_drifts = comparisons + .iter() + .filter(|comparison| comparison.numeric_drift) + .count(); + let numeric_drift_rate = ratio(numeric_drifts, numeric_pairs); + let max_numeric_abs_error = max_comparison_value(comparisons.iter().map(|comparison| { + if comparison.used_numeric { + Some(comparison.max_abs_error) + } else { + None + } + })); + let max_numeric_rel_error = max_comparison_value(comparisons.iter().map(|comparison| { + if comparison.used_numeric { + Some(comparison.max_rel_error) + } else { + None + } + })); let baseline_p95_latency_ms = percentile95( matched @@ -121,12 +234,8 @@ pub fn evaluate_release(input: &GateInput) -> GateReport { .filter(|(_, candidate)| candidate.ok) .map(|(_, candidate)| candidate.latency_ms), ); - let p95_latency_regression_pct = match (baseline_p95_latency_ms, candidate_p95_latency_ms) { - (Some(baseline), Some(candidate)) if baseline > 0.0 => { - Some(round6(((candidate - baseline) / baseline) * 100.0)) - } - _ => None, - }; + let p95_latency_regression_pct = + latency_regression_pct(baseline_p95_latency_ms, candidate_p95_latency_ms); let mut reasons = Vec::new(); let insufficient_coverage = matched_requests < input.thresholds.min_matched_requests @@ -150,6 +259,15 @@ pub fn evaluate_release(input: &GateInput) -> GateReport { )); } + let numeric_correctness_failed = + numeric_drift_rate > input.thresholds.max_numeric_drift_rate && numeric_pairs > 0; + if numeric_correctness_failed { + reasons.push(format!( + "numeric drift rate {:.4} exceeded {:.4}", + numeric_drift_rate, input.thresholds.max_numeric_drift_rate + )); + } + let reliability_failed = error_rate_increase > input.thresholds.max_error_rate_increase; if reliability_failed { reasons.push(format!( @@ -168,7 +286,7 @@ pub fn evaluate_release(input: &GateInput) -> GateReport { )); } - let decision = if correctness_failed || reliability_failed { + let decision = if correctness_failed || numeric_correctness_failed || reliability_failed { GateDecision::Rollback } else if insufficient_coverage || comparable_outputs.is_empty() || latency_failed { GateDecision::Hold @@ -178,23 +296,279 @@ pub fn evaluate_release(input: &GateInput) -> GateReport { }; GateReport { - schema_version: 1, + schema_version: 2, decision, matched_requests, baseline_requests, candidate_requests, coverage_rate, output_mismatch_rate, + numeric_pairs, + tolerated_numeric_outputs, + numeric_drift_rate, + max_numeric_abs_error, + max_numeric_rel_error, baseline_error_rate, candidate_error_rate, error_rate_increase, baseline_p95_latency_ms, candidate_p95_latency_ms, p95_latency_regression_pct, + segments: segment_reports(&matched, &matched_comparisons), reasons, } } +#[derive(Debug, Clone, Copy)] +struct OutputComparison { + matches: bool, + used_numeric: bool, + tolerated_numeric: bool, + numeric_drift: bool, + max_abs_error: f64, + max_rel_error: f64, +} + +fn compare_outputs( + baseline: &Observation, + candidate: &Observation, + thresholds: &GateThresholds, +) -> OutputComparison { + if baseline.output_fingerprint == candidate.output_fingerprint { + return OutputComparison { + matches: true, + used_numeric: false, + tolerated_numeric: false, + numeric_drift: false, + max_abs_error: 0.0, + max_rel_error: 0.0, + }; + } + + let Some(tolerance) = best_numeric_tolerance(baseline, candidate, thresholds) else { + return output_mismatch(); + }; + let Some(baseline_values) = baseline.output_values.as_ref() else { + return output_mismatch(); + }; + let Some(candidate_values) = candidate.output_values.as_ref() else { + return output_mismatch(); + }; + if baseline_values.len() != candidate_values.len() || baseline_values.is_empty() { + return numeric_drift(); + } + + let mut max_abs_error = 0.0_f64; + let mut max_rel_error = 0.0_f64; + let mut within_tolerance = true; + for (baseline_value, candidate_value) in baseline_values.iter().zip(candidate_values) { + let abs_error = (candidate_value - baseline_value).abs(); + let rel_error = if *baseline_value == 0.0 { + if abs_error == 0.0 { 0.0 } else { f64::INFINITY } + } else { + abs_error / baseline_value.abs() + }; + + max_abs_error = max_abs_error.max(abs_error); + max_rel_error = max_rel_error.max(rel_error); + if !baseline_value.is_finite() + || !candidate_value.is_finite() + || (abs_error > tolerance.max_abs_error && rel_error > tolerance.max_rel_error) + { + within_tolerance = false; + } + } + + OutputComparison { + matches: within_tolerance, + used_numeric: true, + tolerated_numeric: within_tolerance, + numeric_drift: !within_tolerance, + max_abs_error: round6(max_abs_error), + max_rel_error: round6(max_rel_error), + } +} + +fn output_mismatch() -> OutputComparison { + OutputComparison { + matches: false, + used_numeric: false, + tolerated_numeric: false, + numeric_drift: false, + max_abs_error: 0.0, + max_rel_error: 0.0, + } +} + +fn numeric_drift() -> OutputComparison { + OutputComparison { + matches: false, + used_numeric: true, + tolerated_numeric: false, + numeric_drift: true, + max_abs_error: f64::INFINITY, + max_rel_error: f64::INFINITY, + } +} + +fn best_numeric_tolerance<'a>( + baseline: &Observation, + candidate: &Observation, + thresholds: &'a GateThresholds, +) -> Option<&'a NumericTolerance> { + thresholds + .numeric_tolerances + .iter() + .filter(|tolerance| tolerance.matches(baseline, candidate)) + .max_by_key(|tolerance| tolerance.specificity()) +} + +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord)] +struct SegmentKey { + model: String, + baseline_backend: String, + candidate_backend: String, + accelerator: String, +} + +#[derive(Debug, Default)] +struct SegmentAccumulator { + matched_requests: usize, + baseline_errors: usize, + candidate_errors: usize, + output_pairs: usize, + output_mismatches: usize, + baseline_latencies: Vec, + candidate_latencies: Vec, +} + +fn segment_reports( + matched: &[(&Observation, &Observation)], + comparisons: &[Option], +) -> Vec { + let mut segments: BTreeMap = BTreeMap::new(); + for ((baseline, candidate), comparison) in matched.iter().zip(comparisons) { + let entry = segments + .entry(segment_key(baseline, candidate)) + .or_default(); + entry.matched_requests += 1; + if !baseline.ok { + entry.baseline_errors += 1; + } + if !candidate.ok { + entry.candidate_errors += 1; + } + if baseline.ok && candidate.ok { + entry.output_pairs += 1; + if comparison.is_some_and(|comparison| !comparison.matches) { + entry.output_mismatches += 1; + } + } + if baseline.ok { + entry.baseline_latencies.push(baseline.latency_ms); + } + if candidate.ok { + entry.candidate_latencies.push(candidate.latency_ms); + } + } + + segments + .into_iter() + .map(|(key, accumulator)| { + let baseline_p95_latency_ms = percentile95(accumulator.baseline_latencies.into_iter()); + let candidate_p95_latency_ms = + percentile95(accumulator.candidate_latencies.into_iter()); + SegmentReport { + model: key.model, + baseline_backend: key.baseline_backend, + candidate_backend: key.candidate_backend, + accelerator: key.accelerator, + matched_requests: accumulator.matched_requests, + output_mismatch_rate: ratio( + accumulator.output_mismatches, + accumulator.output_pairs, + ), + baseline_error_rate: ratio( + accumulator.baseline_errors, + accumulator.matched_requests, + ), + candidate_error_rate: ratio( + accumulator.candidate_errors, + accumulator.matched_requests, + ), + baseline_p95_latency_ms, + candidate_p95_latency_ms, + p95_latency_regression_pct: latency_regression_pct( + baseline_p95_latency_ms, + candidate_p95_latency_ms, + ), + } + }) + .collect() +} + +fn segment_key(baseline: &Observation, candidate: &Observation) -> SegmentKey { + SegmentKey { + model: if same_optional_value(&baseline.model, &candidate.model) { + candidate + .model + .as_ref() + .or(baseline.model.as_ref()) + .cloned() + .unwrap_or_else(|| "unspecified".into()) + } else { + format!( + "{} -> {}", + optional_label(&baseline.model), + optional_label(&candidate.model) + ) + }, + baseline_backend: optional_label(&baseline.backend), + candidate_backend: optional_label(&candidate.backend), + accelerator: candidate + .accelerator + .as_ref() + .or(baseline.accelerator.as_ref()) + .cloned() + .unwrap_or_else(|| "unspecified".into()), + } +} + +fn optional_label(value: &Option) -> String { + value.clone().unwrap_or_else(|| "unspecified".into()) +} + +fn same_optional_value(left: &Option, right: &Option) -> bool { + match (left, right) { + (Some(left), Some(right)) => left == right, + _ => true, + } +} + +fn matches_filter(filter: &Option, value: Option<&String>) -> bool { + match filter { + Some(filter) => value.is_some_and(|value| value == filter), + None => true, + } +} + +fn max_comparison_value(values: impl Iterator>) -> Option { + values + .flatten() + .filter(|value| value.is_finite()) + .max_by(f64::total_cmp) + .map(round6) +} + +fn latency_regression_pct(baseline: Option, candidate: Option) -> Option { + match (baseline, candidate) { + (Some(baseline), Some(candidate)) if baseline > 0.0 => { + Some(round6(((candidate - baseline) / baseline) * 100.0)) + } + _ => None, + } +} + fn ratio(numerator: usize, denominator: usize) -> f64 { if denominator == 0 { 0.0 diff --git a/tests/release.rs b/tests/release.rs index 2c3e79e..a434f62 100644 --- a/tests/release.rs +++ b/tests/release.rs @@ -1,5 +1,5 @@ use rust_inference_runtime::{ - GateDecision, GateInput, GateThresholds, Observation, evaluate_release, + GateDecision, GateInput, GateThresholds, NumericTolerance, Observation, evaluate_release, }; fn observation(id: &str, fingerprint: &str, latency_ms: f64, ok: bool) -> Observation { @@ -8,6 +8,29 @@ fn observation(id: &str, fingerprint: &str, latency_ms: f64, ok: bool) -> Observ output_fingerprint: fingerprint.into(), latency_ms, ok, + model: None, + backend: None, + accelerator: None, + output_values: None, + } +} + +fn numeric_observation( + id: &str, + fingerprint: &str, + backend: &str, + values: &[f64], + latency_ms: f64, +) -> Observation { + Observation { + request_id: id.into(), + output_fingerprint: fingerprint.into(), + latency_ms, + ok: true, + model: Some("decoder-7b".into()), + backend: Some(backend.into()), + accelerator: Some("h100".into()), + output_values: Some(values.to_vec()), } } @@ -69,3 +92,81 @@ fn holds_latency_regression_for_investigation() { assert_eq!(report.decision, GateDecision::Hold); assert!(report.p95_latency_regression_pct.unwrap() > 10.0); } + +#[test] +fn promotes_candidate_with_numeric_outputs_inside_model_tolerance() { + let report = evaluate_release(&GateInput { + thresholds: GateThresholds { + min_matched_requests: 3, + max_output_mismatch_rate: 0.0, + max_error_rate_increase: 0.01, + max_p95_latency_regression_pct: 10.0, + max_numeric_drift_rate: 0.0, + numeric_tolerances: vec![NumericTolerance { + model: Some("decoder-7b".into()), + candidate_backend: Some("candidate".into()), + candidate_accelerator: Some("h100".into()), + max_abs_error: 0.002, + max_rel_error: 0.01, + }], + }, + baseline: vec![ + numeric_observation("a", "baseline-a", "baseline", &[1.0, 2.0], 10.0), + numeric_observation("b", "baseline-b", "baseline", &[2.0, 4.0], 12.0), + numeric_observation("c", "baseline-c", "baseline", &[3.0, 6.0], 14.0), + ], + candidate: vec![ + numeric_observation("a", "candidate-a", "candidate", &[1.001, 2.001], 10.5), + numeric_observation("b", "candidate-b", "candidate", &[2.001, 4.001], 12.5), + numeric_observation("c", "candidate-c", "candidate", &[3.001, 6.001], 14.5), + ], + }); + + assert_eq!(report.decision, GateDecision::Promote); + assert_eq!(report.output_mismatch_rate, 0.0); + assert_eq!(report.numeric_pairs, 3); + assert_eq!(report.tolerated_numeric_outputs, 3); + assert_eq!(report.segments.len(), 1); + assert_eq!(report.segments[0].model, "decoder-7b"); + assert_eq!(report.segments[0].candidate_backend, "candidate"); +} + +#[test] +fn rolls_back_numeric_outputs_outside_model_tolerance() { + let report = evaluate_release(&GateInput { + thresholds: GateThresholds { + min_matched_requests: 3, + max_output_mismatch_rate: 0.0, + max_error_rate_increase: 0.01, + max_p95_latency_regression_pct: 10.0, + max_numeric_drift_rate: 0.0, + numeric_tolerances: vec![NumericTolerance { + model: Some("decoder-7b".into()), + candidate_backend: Some("candidate".into()), + candidate_accelerator: Some("h100".into()), + max_abs_error: 0.002, + max_rel_error: 0.01, + }], + }, + baseline: vec![ + numeric_observation("a", "baseline-a", "baseline", &[1.0, 2.0], 10.0), + numeric_observation("b", "baseline-b", "baseline", &[2.0, 4.0], 12.0), + numeric_observation("c", "baseline-c", "baseline", &[3.0, 6.0], 14.0), + ], + candidate: vec![ + numeric_observation("a", "candidate-a", "candidate", &[1.001, 2.001], 10.5), + numeric_observation("b", "candidate-b", "candidate", &[2.5, 4.001], 12.5), + numeric_observation("c", "candidate-c", "candidate", &[3.001, 6.001], 14.5), + ], + }); + + assert_eq!(report.decision, GateDecision::Rollback); + assert_eq!(report.output_mismatch_rate, 0.333333); + assert_eq!(report.numeric_drift_rate, 0.333333); + assert!( + report + .reasons + .iter() + .any(|reason| reason.contains("numeric drift rate")) + ); +}