Skip to content

Commit 62eedf8

Browse files
committed
fix(box-qualify): the drift bands are ONE-SIDED, a box cannot get faster from contention
The bands shipped symmetric, so the first live run terminated boxes for BEATING their baseline: one-api read 75us against a 79us baseline (-5.06%) and tensorzero 75us against 81us (-7.41%). Both were rejected and replaced. With 7 of 13 baselines sitting above that run's box population the gate was on course to burn every replacement attempt and skip gateways outright, including busbar. One-sided is the correct semantics, not a loosening. Contention, throttling and noisy neighbours only ever ADD latency and REMOVE throughput. A box cannot randomly get faster, so a floor that beats its baseline is the box showing its true clean-hardware speed, and it implies the BASELINE was the noisy measurement rather than this run. The absolute envelope still bounds absurd values on both sides. bq_regression() replaces bq_abs() at the four gate sites and returns the magnitude of the deviation in the DEGRADING direction only: positive drift for stage-1 floor latency (higher is worse), negative drift for stage-2 peak throughput (higher is better). Verified against every real 2026-07-25 datapoint: the two wrongly rejected boxes now pass, gomodel still fails both gates (+5.4% floor, -86% peak), and kong's +13% faster peak passes.
1 parent 0c0462c commit 62eedf8

2 files changed

Lines changed: 41 additions & 8 deletions

File tree

‎lib/box_qualify.sh‎

Lines changed: 20 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -141,6 +141,18 @@ bq_drift_pct(){
141141
}
142142
# bq_abs <n> -> |n| (empty in, empty out)
143143
bq_abs(){ awk -v v="${1:-}" 'BEGIN{ if(v=="") exit; if(v+0<0) v=-v; printf "%.2f", v }'; }
144+
# bq_regression <drift_pct> <sense> -> the magnitude of the deviation in the DEGRADING direction only,
145+
# or 0 when the deviation is an improvement. THE BANDS ARE ONE-SIDED, and this is why:
146+
# a box cannot randomly get FASTER. Contention, throttling and noisy neighbours only ever ADD
147+
# latency and REMOVE throughput. So a floor that beats its baseline is not a contamination signal —
148+
# it is the box showing its true clean-hardware speed (and it implies the BASELINE was the noisy
149+
# measurement, not this run). Rejecting it would terminate good boxes, burn the replacement budget,
150+
# and eventually skip the gateway entirely.
151+
# sense=lat -> higher is worse (p99 latency): a POSITIVE drift is the regression.
152+
# sense=rps -> higher is better (throughput): a NEGATIVE drift is the regression.
153+
# Absurd values in the improving direction are still bounded by the absolute envelope
154+
# (BQ_FLOOR_ABS_MIN_US/BQ_FLOOR_ABS_MAX_US), which is two-sided on purpose.
155+
bq_regression(){ awk -v v="${1:-}" -v s="${2:-lat}" 'BEGIN{ if(v=="") exit; if(s=="rps") v=-v; if(v+0<0) v=0; printf "%.2f", v }'; }
144156
# bq_le <a> <b> -> 0 when a <= b. Empty a is treated as "not measured" -> NOT a violation (the caller
145157
# decides what an unmeasurable signal means; a gate must never fail on a value it never obtained).
146158
bq_le(){ [ -n "${1:-}" ] || return 0; awk -v a="$1" -v b="$2" 'BEGIN{exit !(a+0<=b+0)}'; }
@@ -361,15 +373,15 @@ bq_stage1_verdict(){
361373
return 0
362374
fi
363375
# 5. PRIMARY: drift vs the ROLLING baseline (median of the last N qualified runs, not the last run).
364-
if ! bq_le "$(bq_abs "$BQ_DRIFT_PCT")" "$BQ_FLOOR_DRIFT_PCT"; then
376+
if ! bq_le "$(bq_regression "$BQ_DRIFT_PCT" lat)" "$BQ_FLOOR_DRIFT_PCT"; then
365377
BQ_VERDICT=fail; BQ_REASON=floor_drift
366-
BQ_DETAIL="floor p99 ${med}us drifted ${BQ_DRIFT_PCT}% from this gateway+arch's rolling baseline ${base}us (band +/-${BQ_FLOOR_DRIFT_PCT}%)"
378+
BQ_DETAIL="floor p99 ${med}us is ${BQ_DRIFT_PCT}% SLOWER than this gateway+arch's rolling baseline ${base}us (one-sided band +${BQ_FLOOR_DRIFT_PCT}%; a faster floor never fails)"
367379
return 0
368380
fi
369381
# 6. ANTI-RATCHET: cumulative creep against the oldest retained qualified median.
370-
if [ -n "$oldest" ] && ! bq_le "$(bq_abs "$BQ_RATCHET_PCT")" "$BQ_FLOOR_RATCHET_PCT"; then
382+
if [ -n "$oldest" ] && ! bq_le "$(bq_regression "$BQ_RATCHET_PCT" lat)" "$BQ_FLOOR_RATCHET_PCT"; then
371383
BQ_VERDICT=fail; BQ_REASON=floor_ratchet
372-
BQ_DETAIL="floor p99 ${med}us is ${BQ_RATCHET_PCT}% from the OLDEST retained qualified floor ${oldest}us (anti-ratchet bound +/-${BQ_FLOOR_RATCHET_PCT}%): the box population has crept even though each step passed"
384+
BQ_DETAIL="floor p99 ${med}us is ${BQ_RATCHET_PCT}% SLOWER than the OLDEST retained qualified floor ${oldest}us (one-sided anti-ratchet bound +${BQ_FLOOR_RATCHET_PCT}%): the box population has crept slower even though each step passed"
373385
return 0
374386
fi
375387
BQ_DETAIL="floor p99 ${med}us, drift ${BQ_DRIFT_PCT}% vs baseline ${base}us (band +/-${BQ_FLOOR_DRIFT_PCT}%), spread ${BQ_JITTER_PCT}%, p99/p50 ${BQ_JITTER_RATIO}"
@@ -401,14 +413,14 @@ bq_stage2_verdict(){
401413
BQ_DETAIL="the peak replay of $cell delivered no successful throughput sample (baseline ${base} rps): the gateway or the box is not serving"
402414
return 0
403415
fi
404-
if ! bq_le "$(bq_abs "$BQ_DRIFT_PCT")" "$BQ_PEAK_DRIFT_PCT"; then
416+
if ! bq_le "$(bq_regression "$BQ_DRIFT_PCT" rps)" "$BQ_PEAK_DRIFT_PCT"; then
405417
BQ_VERDICT=fail; BQ_REASON=peak_drift
406-
BQ_DETAIL="peak replay of $cell reached ${obs} rps, ${BQ_DRIFT_PCT}% from the rolling baseline ${base} rps (band +/-${BQ_PEAK_DRIFT_PCT}%)"
418+
BQ_DETAIL="peak replay of $cell reached ${obs} rps, ${BQ_DRIFT_PCT}% BELOW the rolling baseline ${base} rps (one-sided band -${BQ_PEAK_DRIFT_PCT}%; exceeding the baseline never fails)"
407419
return 0
408420
fi
409-
if [ -n "$oldest" ] && ! bq_le "$(bq_abs "$BQ_RATCHET_PCT")" "$BQ_PEAK_RATCHET_PCT"; then
421+
if [ -n "$oldest" ] && ! bq_le "$(bq_regression "$BQ_RATCHET_PCT" rps)" "$BQ_PEAK_RATCHET_PCT"; then
410422
BQ_VERDICT=fail; BQ_REASON=peak_ratchet
411-
BQ_DETAIL="peak replay of $cell reached ${obs} rps, ${BQ_RATCHET_PCT}% from the OLDEST retained qualified peak ${oldest} rps (anti-ratchet bound +/-${BQ_PEAK_RATCHET_PCT}%)"
423+
BQ_DETAIL="peak replay of $cell reached ${obs} rps, ${BQ_RATCHET_PCT}% BELOW the OLDEST retained qualified peak ${oldest} rps (one-sided anti-ratchet bound -${BQ_PEAK_RATCHET_PCT}%)"
412424
return 0
413425
fi
414426
BQ_DETAIL="peak replay of $cell reached ${obs} rps, drift ${BQ_DRIFT_PCT}% vs baseline ${base} rps (band +/-${BQ_PEAK_DRIFT_PCT}%)"

‎lib/box_qualify_test.sh‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -82,6 +82,22 @@ check "tensorzero +3.8% -> pass (0.2pt headroom)" pass "$(s1 x 3.8 4 "$BASE"
8282
check "gomodel +5.4% -> FAIL" fail "$(s1 x 5.4 4 "$BASE")"
8383
s1 x 5.4 4 "$BASE" >/dev/null; check "gomodel reason" "floor_drift" "$BQ_REASON"
8484

85+
echo "box_qualify_test: STAGE 1 — the band is ONE-SIDED (a box cannot get faster from contention)"
86+
# THE INCIDENT THIS EXISTS FOR (2026-07-25, second run). The band shipped SYMMETRIC, so the gate
87+
# terminated boxes for BEATING their baseline: one-api read 75us against a 79us baseline (-5.06%) and
88+
# tensorzero 75us against 81us (-7.41%). Both were rejected and replaced, and with 7 of 13 baselines
89+
# sitting above that run's box population the gate was on course to burn every replacement and skip
90+
# gateways outright — including busbar.
91+
# WHY ONE-SIDED IS CORRECT, not a loosening: contention, throttling and noisy neighbours only ever ADD
92+
# latency. A box cannot randomly get FASTER. A floor that beats its baseline is the box showing its
93+
# true clean-hardware speed, and it implies the BASELINE was the noisy measurement. The absolute
94+
# envelope still bounds absurd values on both sides.
95+
check "one-api -5.06% (faster) -> pass" pass "$(s1 x -5.06 4 "$BASE")"
96+
check "tensorzero -7.41% (faster) -> pass" pass "$(s1 x -7.41 4 "$BASE")"
97+
check "far faster -40% (faster) -> pass" pass "$(s1 x -40 4 "$BASE")"
98+
# ...while the SLOW side keeps biting at exactly the shipped band.
99+
check "just over +4.01% (slower) -> FAIL" fail "$(s1 x 4.01 4 "$BASE")"
100+
85101
echo "box_qualify_test: STAGE 1 — why a static absolute threshold could NOT have caught it"
86102
# gomodel's absolute floor was 77-81us; the healthy population was 73-82us. Every one of those values
87103
# is INSIDE the shipped sanity envelope, so the envelope alone returns pass for the bad box too —
@@ -150,6 +166,11 @@ check "litellm-rust +8% -> pass" pass "$(p2 8 10000)"
150166
check "kong +13% -> pass" pass "$(p2 13 10000)"
151167
# THE ONE THE GATE EXISTS FOR — and note the margin: -86% against a -25% band.
152168
check "gomodel -86% -> FAIL" fail "$(p2 -86 10000)"
169+
# ONE-SIDED here too, mirrored: throughput is inverted (higher is better), so only a SHORTFALL is a
170+
# regression. A gateway that beats its recorded peak must never cost a box relaunch.
171+
check "kong +13% (faster) -> pass" pass "$(p2 13 10000)"
172+
check "far faster +200% (faster) -> pass" pass "$(p2 200 10000)"
173+
check "just under -25.1% -> FAIL" fail "$(p2 -25.1 10000)"
153174
p2 -86 10000 >/dev/null; check "gomodel stage-2 reason" "peak_drift" "$BQ_REASON"
154175
check "gomodel stage-2 drift recorded" "-86.00" "$BQ_DRIFT_PCT"
155176
# The band is why stage 2 is NOT 4%: six of the eight healthy gateways above would be rejected.

0 commit comments

Comments
 (0)