add numeric release gates

.github/workflows/ci.yml

@@ -28,4 +28,4 @@ jobs:
         run: |
           cargo run --release -- gate --input fixtures/release_gate_safe.json --output target/release-gate-promote.json
           cargo run --release -- gate --input fixtures/release_gate_bad.json --output target/release-gate-rollback.json
-
+          cargo run --release -- gate --input fixtures/release_gate_numeric_tolerance.json --output target/release-gate-numeric-tolerance.json

README.md

@@ -16,8 +16,9 @@ replayable scheduling traces, and canary/shadow release decisions.
 - Deterministic workload replay with a machine-readable trace fingerprint.
 - Baseline/candidate release validation with `promote`, `hold`, and `rollback`
   outcomes.
-- Exact output checks, error-rate deltas, p95 latency regression policy, tests,
-  and CI.
+- Exact output checks, model-aware numeric tolerances for backend drift,
+  per-segment release summaries, error-rate deltas, p95 latency regression
+  policy, tests, and CI.
 
 ## Quick Start
 
@@ -35,10 +36,16 @@ cargo run --release -- gate \
 cargo run --release -- gate \
   --input fixtures/release_gate_bad.json \
   --output artifacts/release-gate-rollback.json
+
+cargo run --release -- gate \
+  --input fixtures/release_gate_numeric_tolerance.json \
+  --output artifacts/release-gate-numeric-tolerance.json
 ```
 
 The safe fixture produces `promote`. The candidate with an output mismatch and
 an added error produces `rollback`.
+The numeric-tolerance fixture produces `promote` while reporting four tolerated
+numeric comparisons across a baseline-runtime to candidate-runtime segment.
 
 The checked workload fixture completes four requests in 11 scheduler ticks,
 peaks at 12 of 20 KV pages, returns all pages on completion, and emits trace
@@ -65,10 +72,15 @@ ticks, and completion count.
 ## Release Policy
 
 The gate joins mirrored baseline and candidate observations by request ID.
+Outputs can be validated either by exact fingerprint or by a configured
+numeric tolerance scoped to model, candidate backend, and accelerator. Reports
+include aggregate metrics plus segment summaries so hardware/backend-specific
+regressions remain visible.
 
 | Signal | Response |
 |---|---|
 | Output mismatch above policy | `rollback` |
+| Numeric drift above model/backend policy | `rollback` |
 | Error-rate increase above policy | `rollback` |
 | p95 latency regression above policy | `hold` |
 | Missing or insufficient matched traffic | `hold` |

artifacts/release-gate-numeric-tolerance.json

@@ -0,0 +1,38 @@
+{
+  "schema_version": 2,
+  "decision": "promote",
+  "matched_requests": 4,
+  "baseline_requests": 4,
+  "candidate_requests": 4,
+  "coverage_rate": 1.0,
+  "output_mismatch_rate": 0.0,
+  "numeric_pairs": 4,
+  "tolerated_numeric_outputs": 4,
+  "numeric_drift_rate": 0.0,
+  "max_numeric_abs_error": 0.001,
+  "max_numeric_rel_error": 0.008,
+  "baseline_error_rate": 0.0,
+  "candidate_error_rate": 0.0,
+  "error_rate_increase": 0.0,
+  "baseline_p95_latency_ms": 28.0,
+  "candidate_p95_latency_ms": 27.6,
+  "p95_latency_regression_pct": -1.428571,
+  "segments": [
+    {
+      "model": "decoder-7b",
+      "baseline_backend": "baseline-runtime",
+      "candidate_backend": "candidate-runtime",
+      "accelerator": "h100",
+      "matched_requests": 4,
+      "output_mismatch_rate": 0.0,
+      "baseline_error_rate": 0.0,
+      "candidate_error_rate": 0.0,
+      "baseline_p95_latency_ms": 28.0,
+      "candidate_p95_latency_ms": 27.6,
+      "p95_latency_regression_pct": -1.428571
+    }
+  ],
+  "reasons": [
+    "candidate stayed within correctness, reliability, and latency policy"
+  ]
+}

artifacts/release-gate-promote.json

@@ -1,17 +1,37 @@
 {
-  "schema_version": 1,
+  "schema_version": 2,
   "decision": "promote",
   "matched_requests": 4,
   "baseline_requests": 4,
   "candidate_requests": 4,
   "coverage_rate": 1.0,
   "output_mismatch_rate": 0.0,
+  "numeric_pairs": 0,
+  "tolerated_numeric_outputs": 0,
+  "numeric_drift_rate": 0.0,
+  "max_numeric_abs_error": null,
+  "max_numeric_rel_error": null,
   "baseline_error_rate": 0.0,
   "candidate_error_rate": 0.0,
   "error_rate_increase": 0.0,
   "baseline_p95_latency_ms": 16.0,
   "candidate_p95_latency_ms": 16.7,
   "p95_latency_regression_pct": 4.375,
+  "segments": [
+    {
+      "model": "unspecified",
+      "baseline_backend": "unspecified",
+      "candidate_backend": "unspecified",
+      "accelerator": "unspecified",
+      "matched_requests": 4,
+      "output_mismatch_rate": 0.0,
+      "baseline_error_rate": 0.0,
+      "candidate_error_rate": 0.0,
+      "baseline_p95_latency_ms": 16.0,
+      "candidate_p95_latency_ms": 16.7,
+      "p95_latency_regression_pct": 4.375
+    }
+  ],
   "reasons": [
     "candidate stayed within correctness, reliability, and latency policy"
   ]

artifacts/release-gate-rollback.json

@@ -1,17 +1,37 @@
 {
-  "schema_version": 1,
+  "schema_version": 2,
   "decision": "rollback",
   "matched_requests": 4,
   "baseline_requests": 4,
   "candidate_requests": 4,
   "coverage_rate": 1.0,
   "output_mismatch_rate": 0.333333,
+  "numeric_pairs": 0,
+  "tolerated_numeric_outputs": 0,
+  "numeric_drift_rate": 0.0,
+  "max_numeric_abs_error": null,
+  "max_numeric_rel_error": null,
   "baseline_error_rate": 0.0,
   "candidate_error_rate": 0.25,
   "error_rate_increase": 0.25,
   "baseline_p95_latency_ms": 16.0,
   "candidate_p95_latency_ms": 14.5,
   "p95_latency_regression_pct": -9.375,
+  "segments": [
+    {
+      "model": "unspecified",
+      "baseline_backend": "unspecified",
+      "candidate_backend": "unspecified",
+      "accelerator": "unspecified",
+      "matched_requests": 4,
+      "output_mismatch_rate": 0.333333,
+      "baseline_error_rate": 0.0,
+      "candidate_error_rate": 0.25,
+      "baseline_p95_latency_ms": 16.0,
+      "candidate_p95_latency_ms": 14.5,
+      "p95_latency_regression_pct": -9.375
+    }
+  ],
   "reasons": [
     "output mismatch rate 0.3333 exceeded 0.0000",
     "error-rate increase 0.2500 exceeded 0.0100"

docs/ARCHITECTURE.md

@@ -36,15 +36,17 @@ It computes:
 
 - request coverage;
 - exact output-fingerprint mismatch rate;
+- model/backend-scoped numeric drift when fingerprints differ;
 - error-rate increase;
 - successful-request p95 latency; and
-- candidate p95 regression.
+- candidate p95 regression; and
+- segment summaries by model, baseline backend, candidate backend, and
+  accelerator.
 
-Correctness or reliability regressions produce `rollback`. Latency regressions
-or incomplete evidence produce `hold`. A complete candidate within policy
-produces `promote`.
+Correctness, numeric drift, or reliability regressions produce `rollback`.
+Latency regressions or incomplete evidence produce `hold`. A complete candidate
+within policy produces `promote`.
 
 This is a local validation component, not a deployment controller. Production
 integration would obtain observations from mirrored traffic, canary
 populations, telemetry, and an audited rollout system.
-

docs/RELEASE_VALIDATION.md

@@ -8,6 +8,7 @@ Use `promote` when:
 
 - baseline and candidate request coverage is complete;
 - output fingerprints stay within the configured mismatch budget;
+- configured numeric tolerances cover any expected model/backend output drift;
 - the candidate error-rate increase stays within policy; and
 - candidate p95 latency stays within the configured regression budget.
 
@@ -23,17 +24,36 @@ Use `rollback` when the candidate changes successful outputs beyond policy or
 increases the request error rate beyond policy. These checks intentionally take
 precedence over latency.
 
+## Numeric Tolerance
+
+Some accelerator backends legitimately produce small numeric differences even
+when the generated output is acceptable. `numeric_tolerances` allow those
+comparisons to be explicit instead of hidden behind broad fingerprint waivers.
+Each tolerance can be scoped by model, candidate backend, and candidate
+accelerator.
+
+When fingerprints differ, the gate promotes only if both observations provide
+numeric output vectors of equal length and every value stays within either the
+absolute or relative tolerance. Failed numeric comparisons count as output
+mismatches and as numeric drift.
+
+The report includes:
+
+- `numeric_pairs` and `tolerated_numeric_outputs`;
+- `numeric_drift_rate`;
+- maximum absolute and relative numeric error; and
+- segment summaries by model, baseline backend, candidate backend, and
+  accelerator.
+
 ## Production Extension Points
 
 A real rollout system should add:
 
 - statistically meaningful sample sizes and confidence intervals;
-- model-aware numeric tolerance rather than exact fingerprints where required;
-- segmented checks by model, hardware, prompt class, and region;
+- prompt-class and region segmentation;
 - SLO burn-rate and saturation signals;
 - canary population controls and audited rollback execution; and
 - provenance linking every decision to build, model, and configuration IDs.
 
 The checked fixtures are synthetic and exist to make the policy executable in
 CI. They are not claims about production traffic or fleet scale.
-

fixtures/release_gate_numeric_tolerance.json

@@ -0,0 +1,102 @@
+{
+  "thresholds": {
+    "min_matched_requests": 4,
+    "max_output_mismatch_rate": 0.0,
+    "max_error_rate_increase": 0.01,
+    "max_p95_latency_regression_pct": 10.0,
+    "max_numeric_drift_rate": 0.0,
+    "numeric_tolerances": [
+      {
+        "model": "decoder-7b",
+        "candidate_backend": "candidate-runtime",
+        "candidate_accelerator": "h100",
+        "max_abs_error": 0.002,
+        "max_rel_error": 0.01
+      }
+    ]
+  },
+  "baseline": [
+    {
+      "request_id": "prompt-a",
+      "output_fingerprint": "baseline-a",
+      "latency_ms": 18.0,
+      "ok": true,
+      "model": "decoder-7b",
+      "backend": "baseline-runtime",
+      "accelerator": "h100",
+      "output_values": [0.125, 0.5, 1.25]
+    },
+    {
+      "request_id": "prompt-b",
+      "output_fingerprint": "baseline-b",
+      "latency_ms": 21.0,
+      "ok": true,
+      "model": "decoder-7b",
+      "backend": "baseline-runtime",
+      "accelerator": "h100",
+      "output_values": [0.25, 0.75, 1.5]
+    },
+    {
+      "request_id": "prompt-c",
+      "output_fingerprint": "baseline-c",
+      "latency_ms": 25.0,
+      "ok": true,
+      "model": "decoder-7b",
+      "backend": "baseline-runtime",
+      "accelerator": "h100",
+      "output_values": [0.375, 1.0, 1.75]
+    },
+    {
+      "request_id": "prompt-d",
+      "output_fingerprint": "baseline-d",
+      "latency_ms": 28.0,
+      "ok": true,
+      "model": "decoder-7b",
+      "backend": "baseline-runtime",
+      "accelerator": "h100",
+      "output_values": [0.5, 1.25, 2.0]
+    }
+  ],
+  "candidate": [
+    {
+      "request_id": "prompt-a",
+      "output_fingerprint": "candidate-a",
+      "latency_ms": 18.2,
+      "ok": true,
+      "model": "decoder-7b",
+      "backend": "candidate-runtime",
+      "accelerator": "h100",
+      "output_values": [0.126, 0.501, 1.251]
+    },
+    {
+      "request_id": "prompt-b",
+      "output_fingerprint": "candidate-b",
+      "latency_ms": 20.8,
+      "ok": true,
+      "model": "decoder-7b",
+      "backend": "candidate-runtime",
+      "accelerator": "h100",
+      "output_values": [0.251, 0.751, 1.501]
+    },
+    {
+      "request_id": "prompt-c",
+      "output_fingerprint": "candidate-c",
+      "latency_ms": 24.4,
+      "ok": true,
+      "model": "decoder-7b",
+      "backend": "candidate-runtime",
+      "accelerator": "h100",
+      "output_values": [0.376, 1.001, 1.751]
+    },
+    {
+      "request_id": "prompt-d",
+      "output_fingerprint": "candidate-d",
+      "latency_ms": 27.6,
+      "ok": true,
+      "model": "decoder-7b",
+      "backend": "candidate-runtime",
+      "accelerator": "h100",
+      "output_values": [0.501, 1.251, 2.001]
+    }
+  ]
+}

src/lib.rs

@@ -2,7 +2,8 @@ pub mod release;
 pub mod scheduler;
 
 pub use release::{
-    GateDecision, GateInput, GateReport, GateThresholds, Observation, evaluate_release,
+    GateDecision, GateInput, GateReport, GateThresholds, NumericTolerance, Observation,
+    SegmentReport, evaluate_release,
 };
 pub use scheduler::{
     ReplayInput, ReplayReport, RequestSpec, RuntimeError, Scheduler, SchedulerConfig, TickTrace,