@@ -477,6 +477,10 @@ QUALIFY_DIR="$HERE/results/box-qualify"; mkdir -p "$QUALIFY_DIR"
477477# Where a gateway that exhausted its box budget is recorded. bench_gateway runs in a background
478478# subshell, so a file is the only way the final summary can see it.
479479QUALIFY_SKIPPED=" $QUALIFY_DIR /skipped-$RUN_ID .txt" ; : > " $QUALIFY_SKIPPED "
480+ # One line per gateway that produced a NEW snapshot in THIS run. Written by the boxes (which are
481+ # subshells, so a variable could not carry it back) and read by the field-wide publish at the end.
482+ # Empty means this run measured nothing anywhere, and nothing it derived may be pushed.
483+ FRESH_SNAPSHOTS=" $QUALIFY_DIR /measured-$RUN_ID .txt" ; : > " $FRESH_SNAPSHOTS "
480484
481485# _bq_json_field <file> <key> [subkey] -> the scalar, or EMPTY. Never fabricates: a missing key, a
482486# null, a non-scalar and an unparseable file all read as empty, which every gate treats as unmeasured.
@@ -639,13 +643,29 @@ bench_gateway_once() {
639643 # SAME code. Resolving it per box would let a push mid-run split the field across two revisions and
640644 # publish them side by side as though they were comparable.
641645 glog_echo " cloning $BENCH_REPO @ ${BENCH_COMMIT: 0: 12} ..." ; local _t0=$SECONDS
646+ # SPARSE: the box gets the gateway definitions and lib/, and nothing else.
647+ #
648+ # It used to clone the whole repository, which shipped results/ to every box - including the PREVIOUS
649+ # run's committed results/matrix/<gw>.json. The orchestrator then rsynced that file back off the box
650+ # and accepted it as this run's output, so a box that measured nothing still looked like it had. The
651+ # engine and the mock arrive as prebuilt binaries and the box builds nothing, so no other path is
652+ # needed: whatever is not checked out cannot be read, mistaken for fresh, or run stale.
653+ #
654+ # Same commit, so provenance is unchanged - a sparse checkout narrows WHICH paths are materialised,
655+ # not which revision they come from, and `git rev-parse HEAD` still has to match exactly.
642656 ssh $SSHOPT ubuntu@" $ip " " set -e
643657 rm -rf ~/benchmarking
644- git clone -q '$BENCH_REPO ' ~/benchmarking
658+ git clone -q --filter=blob:none --no-checkout '$BENCH_REPO ' ~/benchmarking
645659 cd ~/benchmarking
660+ git sparse-checkout set --no-cone gateways lib
646661 git checkout -q '$BENCH_COMMIT '
647662 # Refuse to measure a tree that is not exactly the requested commit.
648663 test \"\$ (git rev-parse HEAD)\" = '$BENCH_COMMIT '
664+ # Refuse to measure if the sparse checkout did not actually materialise what the run needs. A
665+ # silently empty gateways/ would surface much later as a validate failure with a confusing message.
666+ test -d gateways && test -r lib/rig.sh
667+ # And prove the exclusion held: results/ on a box is how the previous run's numbers got recycled.
668+ test ! -e results
649669 " >> " $glog " 2>&1
650670 local _up_rc=$?
651671 if [ " $_up_rc " -ne 0 ]; then
@@ -761,11 +781,18 @@ bench_gateway_once() {
761781 return 1
762782 }
763783
764- # Matrix is the SOLE producer now — the standalone perf/memory/stream/streamcpu/xlate/governed suites
765- # are RETIRED. Default to the same `matrix` run-all.sh uses (run-all.sh:75); an explicit SUITES override
766- # still lets an operator re-run a legacy suite ad hoc. Defaulting to the old 7-suite list here would
767- # override run-all.sh's matrix default on the box and re-run all six dormant suites (audit HIGH-2).
768- local ALL_SUITES=" ${SUITES:- matrix} "
784+ # THE SNAPSHOT IS THE ARTIFACT, and there are no per-suite JSONs any more.
785+ #
786+ # This defaulted to `matrix` and then required results/matrix/<gw>.json to exist before calling a run
787+ # DONE. The engine writes no such file - it writes results/snapshots/<gw>.json and the timestamped
788+ # result_<gw>_<measured_at>.json, and nothing else. The check passed anyway because the box cloned the
789+ # repository with the PREVIOUS run's results/matrix/<gw>.json committed inside it, so the orchestrator
790+ # rsynced git's own copy back off the box and accepted it as this run's output. That is how a run which
791+ # measured nothing could report DONE: the file it was asked to find was one it had shipped there itself.
792+ #
793+ # So: no legacy suite is pulled by default, and freshness is judged on the snapshot alone (below). An
794+ # explicit SUITES=... still drives the old per-suite pull for an ad-hoc re-run of a retired suite.
795+ local ALL_SUITES=" ${SUITES:- } "
769796 declare -A _pull_state=() _pull_rc=(); local suite
770797 for suite in $ALL_SUITES ; do _pull_state[$suite ]=unset; _pull_rc[$suite ]=0; done
771798
@@ -806,6 +833,9 @@ bench_gateway_once() {
806833# Every relative path below is relative to the repo, so anchor it rather than inheriting a cwd from
807834# the login shell that starts this script.
808835cd ~/benchmarking || exit 1
836+ # The checkout is sparse (gateways + lib only), so results/ does not exist here and the snapshot
837+ # writer does not create its own output directory - it reports an error and writes nothing.
838+ mkdir -p results/snapshots
809839source ~/.cargo/env 2>/dev/null || true
810840export CORES=0-3 LOADCORES=4-9 MOCKCORES=10-15
811841export CAP_MIB=24000
@@ -931,14 +961,28 @@ REMOTE
931961 # is "no config", not a pull failure, so it never contributes to pull_failed.
932962 if [ " $reachable " -eq 1 ]; then pull_config || true ; fi
933963 # Pull the run's snapshot artifact(s) BEFORE the teardown trap terminates the box — this is the only
934- # chance; the box and everything on it are gone straight after. Best-effort like the config sidecar.
935- if [ " $reachable " -eq 1 ]; then pull_snapshots || true ; fi
936- # DONE means a CLEAN, fully-pulled fresh run. If any suite's pull failed or the guard kept old data,
937- # this gateway did NOT cleanly refresh - say so loudly so the freshness guard's later hard-fail is
938- # never a surprise and the gateway can be re-run.
964+ # chance; the box and everything on it are gone straight after.
965+ #
966+ # THIS IS THE RUN'S SUCCESS CRITERION, not a best-effort extra. pull_snapshots returns 0 only when the
967+ # count of result_<gw>_*.json on disk actually GREW, so it answers the one question that matters -
968+ # "did THIS run produce a measurement?" - and it cannot be satisfied by an artifact that was already
969+ # here. Its result used to be discarded with `|| true`, which is how a box that aborted before
970+ # measuring anything still reached the DONE branch and published.
971+ local snap_fresh=0
972+ if [ " $reachable " -eq 1 ] && pull_snapshots; then snap_fresh=1; fi
973+ if [ " $snap_fresh " -eq 0 ]; then
974+ glog_echo " NO FRESH SNAPSHOT for $gw — this run measured nothing, so it publishes nothing and the board keeps whatever it already had"
975+ fi
976+ # DONE means a CLEAN run that MEASURED something and was fully pulled. Anything less is INCOMPLETE, so
977+ # the freshness guard's later hard-fail is never a surprise and the gateway can be re-run.
939978 local publish_failed=0
940- if [[ " $pull_failed " -eq 0 && " $run_failed " -eq 0 && -f " $HERE /results/matrix/ $gw .json " ]]; then
979+ if [[ " $pull_failed " -eq 0 && " $run_failed " -eq 0 && " $snap_fresh " -eq 1 ]]; then
941980 glog_echo " DONE"
981+ # Record that at least one gateway measured something this run. The field-wide history + charts
982+ # publish at the very end reads this: charts.py regenerates from whatever is in results/, so a run
983+ # where every box died would otherwise rebuild the PREVIOUS run's numbers and push them, which is
984+ # indistinguishable from a successful run to anyone reading the board.
985+ echo " $gw " >> " $FRESH_SNAPSHOTS "
942986 # INCREMENTAL PUBLISH: this box finished cleanly and the promote guard passed for every suite, so
943987 # commit + push ONLY this gateway's result now (gated on PUBLISH, serialized across boxes). The
944988 # board fills in gateway-by-gateway; a single-gateway invocation pushes just that one row. The
@@ -951,11 +995,11 @@ REMOTE
951995 publish_failed=1
952996 glog_echo " publish reported an issue for $gw (result IS committed/on disk; push may need a manual retry) — counting it as a run issue"
953997 fi
954- else glog_echo " INCOMPLETE (a suite crashed, failed to pull , or was guard-held ; this gateway did NOT fully refresh - re-run it)" ; fi
998+ else glog_echo " INCOMPLETE (the run crashed, measured nothing , or failed to pull ; this gateway did NOT refresh - re-run it)" ; fi
955999 # Propagate the issue to the caller's `wait "$p" || fail=…` so the summary's issue count is accurate
956- # and a run missing whole suites — OR a gateway whose publish never reached the remote — is never
1000+ # and a run that measured nothing — OR a gateway whose publish never reached the remote — is never
9571001 # reported as "0 issues" (audit R3-M4/M5 + R3-L1).
958- if [[ " $pull_failed " -ne 0 || " $run_failed " -ne 0 || " $publish_failed " -ne 0 || ! -f " $HERE /results/matrix/ $gw .json " ]]; then return 1; fi
1002+ if [[ " $pull_failed " -ne 0 || " $run_failed " -ne 0 || " $publish_failed " -ne 0 || " $snap_fresh " -eq 0 ]]; then return 1; fi
9591003 return 0
9601004}
9611005
@@ -980,6 +1024,26 @@ if [ -s "$QUALIFY_SKIPPED" ]; then
9801024 log " (re-run those gateways; their committed results are the PREVIOUS run's, not this one's)"
9811025fi
9821026
1027+ # ── A RUN THAT MEASURED NOTHING CHANGES NOTHING ───────────────────────────────────────────────────
1028+ # Everything below this line - the append-only history, charts.py, and the push - DERIVES from
1029+ # whatever happens to be sitting in results/. None of it re-reads the boxes. So when every box failed,
1030+ # history/append.py re-appends the old numbers, charts.py rebuilds the PREVIOUS run's charts, and the
1031+ # final sweep pushes them: from outside, a total failure is indistinguishable from a successful run.
1032+ #
1033+ # That is not hypothetical. It was demonstrated live: a box aborted before measuring, the orchestrator
1034+ # logged "all boxes done (1 job(s) reported an issue)", regenerated 24 charts from the previous run's
1035+ # data, and pushed them to origin/main.
1036+ #
1037+ # A failed run must leave the board exactly as it found it, so stop here and leave the working tree
1038+ # alone. `fail` is already non-zero from the boxes themselves, so the exit code still reports it.
1039+ if [ ! -s " $FRESH_SNAPSHOTS " ]; then
1040+ log " NO GATEWAY MEASURED ANYTHING THIS RUN — not appending history, not regenerating charts, not pushing."
1041+ log " The board keeps exactly what it had. Re-run the gateways above; their fanout logs say why each failed."
1042+ rm -f " $PUBLISH_LOCK " 2> /dev/null || true ; rmdir " ${PUBLISH_LOCK} .d" 2> /dev/null || true
1043+ exit " $fail "
1044+ fi
1045+ log " $( wc -l < " $FRESH_SNAPSHOTS " | tr -d ' ' ) gateway(s) produced a fresh snapshot this run — regenerating and publishing from them"
1046+
9831047# ── append this run to the append-only history (results/history/<gw>.jsonl) ─────────────────────
9841048# Do NOT swallow a failure with `|| true` (audit R3-LOW-4): a malformed result JSON or an unwritable
9851049# results/history/ would otherwise complete the run "successfully" with the append-only history
0 commit comments