Skip to content

Commit d3dcdb1

Browse files
committed
fix(site): C8 groups the board by instrument, not by commit sha
C8's claim is that one instrument measured every column, so the only thing differing between two columns is the gateway. The commit sha is a proxy for that, and it is loose in exactly one direction: a commit can change while the instrument does not. Plano's own config file moving cost the board a mixed-engine failure and would have cost a fourteen-box re-run to change no measurement. The proxy is never loose the other way, so attesting equivalence can only excuse a difference the binaries prove is not there. That is the rule the attestation file states and the rule the loader enforces: an entry is admitted only when it carries built-artifact hashes and they are identical. An entry reasoned from a diff is inert, because "this looks test-only" is how a real instrument change gets waved through. Evidence for the one entry: otb built release at both commits in clean worktrees is byte-identical (1fc78d7c...), and the mock-arm64 asset CI rebuilt from the newer commit is byte-identical to the hash the running boxes recorded (9a1c7b0e...). Also adds the override, for the rare case where the board must ship over C8's objection. It takes the REASON as its value rather than a boolean, refuses a value too short to be one, and hands what was overridden back to the caller so the fact rides along with what gets published. A publish guard that can be switched off invisibly is not a guard, and six months on nobody can say which numbers shipped over an objection. Tests cover the mix still failing unattested, the attested case publishing, an attestation with differing hashes staying inert, an unparseable file degrading to commit equality rather than to blanket agreement, the override refusing a bare flag, and the shipped file meeting the evidence rule it declares.
1 parent ade3c6e commit d3dcdb1

3 files changed

Lines changed: 169 additions & 5 deletions

File tree

‎site/check-consistency.mjs‎

Lines changed: 46 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -202,26 +202,67 @@ export function c6Inversions(gwKey, rawMatrix) {
202202
// - a gateway carries no engine stamp at all, while another does (silently pre-stamp data mixed in)
203203
// A board where NO gateway is stamped is left alone: that is entirely pre-stamp data, and failing it
204204
// would only punish history it cannot fix. The moment one stamped run lands, all of them must be.
205-
export function engineAgreement(gwKeys, resolve = (k) => newestSnapshotOnDisk(k)) {
205+
// Commits the repo ATTESTS are the same instrument, with built-artifact evidence. See
206+
// site/instrument-equivalence.json for the rule an entry has to meet. Returns commit -> instrument
207+
// id; a commit nobody attested maps to itself, so an unreadable or absent file degrades to the
208+
// original commit-equality behaviour rather than to a board that publishes anything.
209+
export function instrumentOf(fileText) {
210+
const map = new Map();
211+
let doc;
212+
try { doc = JSON.parse(fileText); } catch { return map; }
213+
for (const inst of (doc && doc.instruments) || []) {
214+
if (!inst || !inst.id || !Array.isArray(inst.commits)) continue;
215+
// An entry with no artifact evidence is INERT, not trusted: the file's own rule is that identical
216+
// binaries are what admits an entry, and a rule nothing enforces is a comment.
217+
const ev = inst.evidence && inst.evidence.otb_release_sha256;
218+
const hashes = ev ? Object.values(ev) : [];
219+
const proven = hashes.length >= inst.commits.length && new Set(hashes).size === 1;
220+
if (!proven) continue;
221+
for (const c of inst.commits) map.set(c, inst.id);
222+
}
223+
return map;
224+
}
225+
226+
export function engineAgreement(gwKeys, resolve = (k) => newestSnapshotOnDisk(k), opts = {}) {
206227
const errors = [];
207-
const seen = new Map(); // commit -> [gwKey]
228+
const seen = new Map(); // instrument id -> [gwKey]
229+
const commitsOf = new Map(); // instrument id -> Set(commit)
208230
const unstamped = [];
209231
let checked = 0;
232+
const equiv = opts.equivalence !== undefined
233+
? opts.equivalence
234+
: instrumentOf(readFileSync(join(ROOT, "site", "instrument-equivalence.json"), "utf8"));
210235
for (const k of gwKeys) {
211236
const found = resolve(k);
212237
const eng = found && found.snap && found.snap.rig && found.snap.rig.engine;
213238
if (!eng || !eng.commit) { unstamped.push(k); continue; }
214239
checked += 1;
215240
if (eng.dirty === true)
216241
errors.push(`C8: ${k} was measured by a DIRTY harness tree (engine.commit=${eng.commit.slice(0, 12)} with uncommitted edits) - the commit does not identify what ran, so this run is not reproducible and must be re-measured on a clean tree`);
217-
if (!seen.has(eng.commit)) seen.set(eng.commit, []);
218-
seen.get(eng.commit).push(k);
242+
// The instrument, not the commit. Two commits that build the same binaries measured the same way,
243+
// and failing that costs a whole field re-run to change nothing.
244+
const inst = equiv.get(eng.commit) || eng.commit;
245+
if (!seen.has(inst)) { seen.set(inst, []); commitsOf.set(inst, new Set()); }
246+
seen.get(inst).push(k);
247+
commitsOf.get(inst).add(eng.commit);
219248
}
220249
if (checked > 0 && unstamped.length > 0)
221250
errors.push(`C8: ${unstamped.length} gateway(s) carry no engine stamp (${unstamped.join(", ")}) while ${checked} do - the board would be mixing pre-stamp data with stamped data and cannot show they came from the same harness; re-measure the unstamped gateways`);
222251
if (seen.size > 1) {
223252
const groups = [...seen.entries()].map(([c, ks]) => `${c.slice(0, 12)}: ${ks.join(", ")}`).join(" | ");
224-
errors.push(`C8: the board mixes ${seen.size} harness engines (${groups}) - columns measured by different instruments are not comparable, so a defect fixed between those commits applies to only part of the field; re-run the lagging gateways on the newest engine`);
253+
const msg = `C8: the board mixes ${seen.size} harness engines (${groups}) - columns measured by different instruments are not comparable, so a defect fixed between those commits applies to only part of the field; re-run the lagging gateways on the newest engine`;
254+
// THE OVERRIDE, for the rare case where the board must ship over C8's objection. It is deliberately
255+
// awkward: it takes the reason as its value, it cannot be set by accident, and it does not silence
256+
// anything. The mix is still reported, still counted, and rides along in the returned result so the
257+
// caller publishes the fact that a human overrode a publish guard. An override nobody can see after
258+
// the fact is just a disabled check.
259+
const why = (opts.override !== undefined ? opts.override : process.env.OTB_ALLOW_MIXED_ENGINES) || "";
260+
if (why.trim().length >= 12) {
261+
return { errors, checked, commits: [...seen.keys()], overridden: { check: "C8.mix", reason: why.trim(), detail: msg } };
262+
}
263+
if (why.trim().length > 0)
264+
errors.push(`C8: OTB_ALLOW_MIXED_ENGINES was set to ${JSON.stringify(why)}, which is not a reason - the override takes the justification as its value (12 characters or more) so that what gets published records WHY a publish guard was overridden`);
265+
errors.push(msg);
225266
}
226267
return { errors, checked, commits: [...seen.keys()] };
227268
}

‎site/instrument-equivalence.json‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
{
2+
"_what": "Commits that are DIFFERENT commits but the SAME INSTRUMENT. C8 groups the board by instrument, not by commit sha; every commit listed under one instrument id is treated as that instrument. A commit absent from this file is its own instrument, which is the safe default: silence never merges anything.",
3+
"_why": "C8's claim is that one instrument measured every column, so the only thing differing between two columns is the gateway. The commit sha is a PROXY for the instrument, and it is a loose one in exactly one direction: a commit can change while the instrument does not (a gateway's own config, a test, a workflow). Failing those is a false positive that costs a full field re-run for no gain in truth. The proxy is never loose the other way, so this file can only ever excuse a difference the binaries prove is not there.",
4+
"_evidence_rule": "An entry is only admissible with BUILT-ARTIFACT evidence: build the shipped binaries at each commit and record their hashes. Identical bytes out of a reproducible build is proof, not argument. Do NOT add an entry reasoned from a diff - 'this looks test-only' is how a real instrument change gets waved through. If the binaries differ, the commits are different instruments and the field re-runs.",
5+
"instruments": [
6+
{
7+
"id": "otb-1fc78d7c",
8+
"commits": [
9+
"dd26a545026be44a0c38589242859138df05b2eb",
10+
"8d742122630fb6cf66ea37969cf50469c8affde9"
11+
],
12+
"reason": "The 14-box field ran on dd26a54. Plano failed to boot on it for two config reasons, and 8d74212 fixes plano's own config, corrects plano's matrix, adds tests, and wires an existing test into CI. Re-running the other thirteen to move a gateway config file would have changed no measurement.",
13+
"evidence": {
14+
"method": "cargo build --release --bin otb at each commit in a clean worktree, plus the mock-arm64 asset that bench-rig rebuilt from 8d74212, compared against the hash the running boxes recorded",
15+
"otb_release_sha256": {
16+
"dd26a545026be44a0c38589242859138df05b2eb": "1fc78d7c1e62db459e666e69b2bab4325c17ec385417b1f60cd603078f289631",
17+
"8d742122630fb6cf66ea37969cf50469c8affde9": "1fc78d7c1e62db459e666e69b2bab4325c17ec385417b1f60cd603078f289631"
18+
},
19+
"mock_arm64_sha256": {
20+
"recorded_by_boxes_on_dd26a54": "9a1c7b0e10f846602cf0e19a96f2d1e4a0e3f50b3b2634aa6786053a13cf9ab5",
21+
"rebuilt_by_ci_from_8d74212": "9a1c7b0e10f846602cf0e19a96f2d1e4a0e3f50b3b2634aa6786053a13cf9ab5"
22+
},
23+
"verified_at": "2026-07-28T05:50:00Z"
24+
}
25+
}
26+
]
27+
}

‎site/test.mjs‎

Lines changed: 96 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3004,3 +3004,99 @@ test("protocol grid: a matrix-less gateway renders an all-n/a row with its reaso
30043004
});
30053005

30063006
console.log(`\n${passed} tests passed`);
3007+
3008+
// ---- C8: THE BOARD IS GROUPED BY INSTRUMENT, NOT BY COMMIT SHA -----------------------------------
3009+
//
3010+
// C8's claim is that one instrument measured every column. The commit sha is a PROXY for that, and it
3011+
// is loose in exactly one direction: a commit can change while the instrument does not. A gateway's
3012+
// own config file, a test, a workflow - none of them can alter how a DIFFERENT gateway was measured,
3013+
// yet each moves the sha and each used to cost a full field re-run to satisfy a check that would have
3014+
// learned nothing. The proxy is never loose the other way, so attesting equivalence can only ever
3015+
// excuse a difference the built binaries prove is not there.
3016+
test("C8: commits the repo attests are one instrument, with artifact evidence, do not read as a mixed board", () => {
3017+
const snapOf = (commit) => ({ snap: { rig: { engine: { commit, dirty: false } } } });
3018+
const A = "a".repeat(40), B = "b".repeat(40);
3019+
const resolve = (k) => snapOf(k === "plano" ? B : A);
3020+
const keys = ["bifrost", "one-api", "plano"];
3021+
3022+
// RED-before: with no attestation, two commits are two instruments and the board is refused. This
3023+
// is the behaviour being preserved for every case that is NOT attested, so it is asserted first.
3024+
const bare = checkMod.engineAgreement(keys, resolve, { equivalence: new Map() });
3025+
assert.equal(bare.errors.length, 1, "an unattested mix must still fail");
3026+
assert.match(bare.errors[0], /mixes 2 harness engines/);
3027+
3028+
// GREEN-after: the same board, with the two commits attested as one instrument, publishes.
3029+
const attested = checkMod.instrumentOf(JSON.stringify({
3030+
instruments: [{ id: "otb-1fc78d7c", commits: [A, B],
3031+
evidence: { otb_release_sha256: { [A]: "1fc78d7c", [B]: "1fc78d7c" } } }],
3032+
}));
3033+
const ok = checkMod.engineAgreement(keys, resolve, { equivalence: attested });
3034+
assert.deepEqual(ok.errors, [], "commits proven to build the same binary are one instrument");
3035+
assert.equal(ok.checked, 3);
3036+
3037+
// An attestation whose evidence does NOT show identical binaries is inert. The file's rule is that
3038+
// identical bytes admit an entry; a rule nothing enforces is a comment, and this is the case where
3039+
// a real instrument change would otherwise be waved through by a hand-written claim.
3040+
const unproven = checkMod.instrumentOf(JSON.stringify({
3041+
instruments: [{ id: "wishful", commits: [A, B],
3042+
evidence: { otb_release_sha256: { [A]: "1111", [B]: "2222" } } }],
3043+
}));
3044+
assert.equal(unproven.size, 0, "differing binaries are differing instruments, whatever the file claims");
3045+
assert.equal(checkMod.engineAgreement(keys, resolve, { equivalence: unproven }).errors.length, 1);
3046+
3047+
// No evidence block at all is equally inert.
3048+
assert.equal(checkMod.instrumentOf(JSON.stringify({
3049+
instruments: [{ id: "bare", commits: [A, B] }] })).size, 0);
3050+
// Unparseable or absent file degrades to commit-equality, never to "everything agrees".
3051+
assert.equal(checkMod.instrumentOf("{ not json").size, 0);
3052+
});
3053+
3054+
// ---- C8: the override is loud, reasoned, and published ------------------------------------------
3055+
//
3056+
// A publish guard sometimes has to be overridden. The failure mode of an override is that it is
3057+
// silent: a boolean env var flips, the board ships, and six months later nobody can say which numbers
3058+
// were published over an objection. So the override takes the REASON as its value and hands it back to
3059+
// the caller to publish. An override nobody can see after the fact is a disabled check.
3060+
test("C8: the mixed-board override demands a reason and returns it for publication", () => {
3061+
const snapOf = (commit) => ({ snap: { rig: { engine: { commit, dirty: false } } } });
3062+
const resolve = (k) => snapOf(k === "plano" ? "b".repeat(40) : "a".repeat(40));
3063+
const keys = ["bifrost", "plano"];
3064+
const run = (override) => checkMod.engineAgreement(keys, resolve, { equivalence: new Map(), override });
3065+
3066+
assert.equal(run(undefined).errors.length, 1, "no override: the mix is refused");
3067+
3068+
// A truthy-but-meaningless value is NOT an override. This is the "1"/"true" habit the check exists
3069+
// to refuse, and it fails with a message saying what the field is for.
3070+
const bogus = run("1");
3071+
assert.equal(bogus.overridden, undefined, "a flag is not a justification");
3072+
assert.match(bogus.errors.join("\n"), /not a reason/);
3073+
assert.match(bogus.errors.join("\n"), /mixes 2 harness engines/, "and the original objection still stands");
3074+
3075+
// A real reason publishes, and the objection travels WITH the result rather than vanishing.
3076+
const forced = run("plano re-ran on a gateway-config-only commit; binaries verified identical");
3077+
assert.deepEqual(forced.errors, []);
3078+
assert.equal(forced.overridden.check, "C8.mix");
3079+
assert.match(forced.overridden.reason, /binaries verified identical/);
3080+
assert.match(forced.overridden.detail, /mixes 2 harness engines/,
3081+
"what was overridden is recorded, not just that something was");
3082+
});
3083+
3084+
// ---- the shipped attestation is itself valid ----------------------------------------------------
3085+
// The file in the repo is data the publish path trusts, so it is tested like code: it must parse, and
3086+
// every entry in it must meet the evidence rule it states for itself.
3087+
test("C8: the repo's own instrument-equivalence.json meets the evidence rule it declares", () => {
3088+
const raw = readFileSync(join(ROOT, "site", "instrument-equivalence.json"), "utf8");
3089+
const doc = JSON.parse(raw);
3090+
const admitted = checkMod.instrumentOf(raw);
3091+
for (const inst of doc.instruments) {
3092+
const hashes = Object.values((inst.evidence || {}).otb_release_sha256 || {});
3093+
assert.equal(hashes.length, inst.commits.length,
3094+
`${inst.id}: every attested commit needs a built-artifact hash, not a subset`);
3095+
assert.equal(new Set(hashes).size, 1, `${inst.id}: the hashes must actually be identical`);
3096+
assert.ok(inst.reason && inst.reason.length > 20, `${inst.id}: an entry states why it exists`);
3097+
for (const c of inst.commits) {
3098+
assert.match(c, /^[0-9a-f]{40}$/, `${inst.id}: commits are full shas, so they cannot be ambiguous`);
3099+
assert.equal(admitted.get(c), inst.id, `${inst.id}: ${c.slice(0, 12)} must be admitted`);
3100+
}
3101+
}
3102+
});

0 commit comments

Comments
 (0)