From 0bccb293720cdd6004c0bbafa3204b4ff5361b67 Mon Sep 17 00:00:00 2001 From: anupsv <6407789+anupsv@users.noreply.github.com> Date: Fri, 31 Jul 2026 00:39:40 -0700 Subject: [PATCH] DFlash: the golden R2 keys, settled by probe rather than by argument #826 added a "gautham-experiments/" prefix to the golden keys. That was wrong, and #824's original keys were right. Probe run 30613434387 established it in about a minute: correctness_prompts/laguna-xs-2.1-dflash/dflash_correctness_golden_hidden.json FOUND 185394b correctness_prompts/laguna-xs-2.1-dflash/dflash_benchmark_golden_hidden.json FOUND 185433b gautham-experiments/correctness_prompts/laguna-xs-2.1-dflash/... 404 NoSuchKey Those byte counts are exactly the generated goldens', and the SERIAL control key was FOUND in the same run, so the credentials and the bucket were never the issue. "gautham-experiments" is the BUCKET, already carried by R2_BUCKET_ENDPOINT. Worth being plain about how this went, because the pattern cost four dispatches: the operator's path string is ambiguous between bucket-plus-key and key, I picked one reading on the strength of the serial keys' shape, was told I had overridden them, switched to the other reading, and was wrong that time. Two confident guesses in opposite directions, ~35 minutes each. Neither reading was unreasonable; arguing about it was. The probe that ends the argument took a minute to write and a minute to run. Fixed in all three places -- the correctness step's env, the timed_prompt_pool entry, and DFlashGoldenKeyTests' pinned prefix -- and the test's doc comment now records the probe output as the evidence, so the next person reads a measurement instead of a rationale. swift test: 582 tests, 28 suites, green. Co-Authored-By: Claude Opus 5 (1M context) --- .github/workflows/dflash-benchmark.yml | 2 +- Tests/MLXFastTests/DFlashGoldenKeyTests.swift | 31 ++++++++++++------- fixtures/laguna_xs_2_1_dflash_track.json | 2 +- 3 files changed, 22 insertions(+), 13 deletions(-) diff --git a/.github/workflows/dflash-benchmark.yml b/.github/workflows/dflash-benchmark.yml index 844e185d..48676683 100644 --- a/.github/workflows/dflash-benchmark.yml +++ b/.github/workflows/dflash-benchmark.yml @@ -1622,7 +1622,7 @@ jobs: # benchmark (timed) golden comes from the per-run pool selection; the # correctness golden stays fixed because it is an untimed gate and is # not what the lottery would target. - MLXFAST_DFLASH_CORRECTNESS_GOLDEN_R2_PATH: gautham-experiments/correctness_prompts/laguna-xs-2.1-dflash/dflash_correctness_golden_hidden.json + MLXFAST_DFLASH_CORRECTNESS_GOLDEN_R2_PATH: correctness_prompts/laguna-xs-2.1-dflash/dflash_correctness_golden_hidden.json MLXFAST_DFLASH_BENCH_GOLDEN_R2_PATH: ${{ steps.select_dflash_target.outputs.r2_path }} MLXFAST_DFLASH_BENCH_GOLDEN_SHA256_SELECTED: ${{ steps.select_dflash_target.outputs.sha256 }} MLXFAST_DFLASH_BENCH_GOLDEN_BYTES_SELECTED: ${{ steps.select_dflash_target.outputs.bytes }} diff --git a/Tests/MLXFastTests/DFlashGoldenKeyTests.swift b/Tests/MLXFastTests/DFlashGoldenKeyTests.swift index 4c4622b9..db29df12 100644 --- a/Tests/MLXFastTests/DFlashGoldenKeyTests.swift +++ b/Tests/MLXFastTests/DFlashGoldenKeyTests.swift @@ -1,21 +1,30 @@ import Foundation import Testing -/// The R2 keys the DFlash goldens actually live under. +/// The R2 keys the DFlash goldens actually live under, PROVEN rather than inferred. /// -/// This exists because the prefix was dropped twice. The operator uploaded to -/// `gautham-experiments/correctness_prompts/laguna-xs-2.1-dflash/`, and I twice -/// pinned the key without the leading segment — first by assuming it was the -/// bucket name already carried by `R2_BUCKET_ENDPOINT` (the serial keys are -/// written with no bucket segment, which made the assumption feel safe), and -/// then again when the corrected commit missed the merge. +/// This constant was wrong twice, in opposite directions, because the operator's +/// phrase "gautham-experiments/correctness_prompts/laguna-xs-2.1-dflash/" is +/// ambiguous between a bucket-plus-key and a key: R2_BUCKET_ENDPOINT carries the +/// bucket, and the serial keys are written with no bucket segment. I stripped the +/// segment, then restored it when corrected, then had to strip it again. /// -/// A wrong key costs a full 30–40 minute ranked dispatch to discover, so it is -/// worth pinning rather than re-deriving. If the objects genuinely move, change -/// this test in the same commit that moves them. +/// Probe run 30613434387 settled it in about a minute: +/// +/// correctness_prompts/laguna-xs-2.1-dflash/dflash_correctness_golden_hidden.json FOUND 185394b +/// correctness_prompts/laguna-xs-2.1-dflash/dflash_benchmark_golden_hidden.json FOUND 185433b +/// gautham-experiments/correctness_prompts/... 404 NoSuchKey +/// +/// Those byte counts are exactly the generated goldens', and the serial control +/// key was FOUND in the same run, so the credentials and bucket were never in +/// question. `gautham-experiments` is the BUCKET. +/// +/// A wrong key costs a 30-40 minute ranked dispatch to discover, which is why +/// this is pinned and why dflash-probe-r2-keys.yml exists. If the objects move, +/// probe first, then change this test in the same commit that moves them. @Suite("DFlash golden R2 keys") struct DFlashGoldenKeyTests { - private static let prefix = "gautham-experiments/correctness_prompts/laguna-xs-2.1-dflash" + private static let prefix = "correctness_prompts/laguna-xs-2.1-dflash" @Test func theCorrectnessGoldenKeyMatchesWhereItWasUploaded() throws { diff --git a/fixtures/laguna_xs_2_1_dflash_track.json b/fixtures/laguna_xs_2_1_dflash_track.json index 78021fbd..77ad896c 100644 --- a/fixtures/laguna_xs_2_1_dflash_track.json +++ b/fixtures/laguna_xs_2_1_dflash_track.json @@ -6,7 +6,7 @@ "mlx_swift_lm_revision": "bc1c0ee67d15798343be17c9f8f61f7c0d977149", "timed_prompt_pool": [ { - "r2_path": "gautham-experiments/correctness_prompts/laguna-xs-2.1-dflash/dflash_benchmark_golden_hidden.json", + "r2_path": "correctness_prompts/laguna-xs-2.1-dflash/dflash_benchmark_golden_hidden.json", "sha256": "5b9e2c4a09904391213fd2d80f4b5d1ccde39a163b88de22990ec34f3f781ad8", "bytes": 185433 }