From 91a630becd671f45642ca20a49d0b6cf86086ac4 Mon Sep 17 00:00:00 2001 From: luciferlive112116 <291889058+luciferlive112116@users.noreply.github.com> Date: Wed, 15 Jul 2026 21:17:31 +0800 Subject: [PATCH] fix(docs): landing page still advertises the pre-8192 reference regime index/index.html named N = 12,000 as CCO's reference regime and shipped a copy-pasteable 'python -m eval --n 12000 --pairs 3' quickstart, but the engines and every other surface moved to 8192: strategy/matmul --n default is 8192, and README.md / BENCHMARKS.md both state 'Reference setup: 8192 x 8192'. The KPI beside it (3.5 TFLOP) was right only for the stale N -- 2*12000**3 = 3.456e12 -- so at the real default it must read 2*8192**3 = 1.0995e12 -> 1.1 TFLOP. The neighbouring 1,000 -> 2 GFLOP and 128,000 -> 4.2 PFLOP cards already use decimal N and stay as they are. Re-derive the three stale claims from the shipped default and add a CPU test that pins the page's advertised N, quickstart and KPI to strategy/matmul's --n default so the front page cannot drift away from the engines again. --- index/index.html | 8 +- tests/test_landing_page_reference_regime.py | 83 +++++++++++++++++++++ 2 files changed, 87 insertions(+), 4 deletions(-) create mode 100644 tests/test_landing_page_reference_regime.py diff --git a/index/index.html b/index/index.html index 24092bd..769d056 100644 --- a/index/index.html +++ b/index/index.html @@ -334,8 +334,8 @@

Matrix multiply cost grows with the cube of size.<

Trivial today — the N³ term means this is not where the real cost lives.

-
3.5 TFLOP
-
N = 12,000 — CCO's reference regime
+
1.1 TFLOP
+
N = 8,192 — CCO's reference regime

Full-rank data, deliberately the hard case — no structure to exploit for free.

@@ -461,7 +461,7 @@

From matmul, live today, to sub-quadratic attention.

Matmul arena

Approximate and exact-but-cheaper strategies for C = A × B, scored on full-rank, low-rank, and decaying-spectrum regimes. Open for contributions today.

-
subspaceN=12000open now
+
subspaceN=8192open now
@@ -524,7 +524,7 @@

Run the scorer

Clone the repo, install PyTorch, and reproduce the matmul reference scorecard on your own GPU — the same numbers a reviewer checks.

See the quickstart
diff --git a/tests/test_landing_page_reference_regime.py b/tests/test_landing_page_reference_regime.py new file mode 100644 index 0000000..72c4e56 --- /dev/null +++ b/tests/test_landing_page_reference_regime.py @@ -0,0 +1,83 @@ +"""Pin the landing page's reference-regime claims to the code (issue: stale N). + +`index/index.html` advertises CCO's reference regime and a copy-pasteable +quickstart command. Those must track the real default the engines ship with -- +`strategy`/`matmul` `--n` (8192), which README.md and BENCHMARKS.md also state as +"Reference setup: 8192 x 8192". The page had been left at the pre-8192 default of +N=12,000 (and its KPI showed 2*12000**3 = 3.5 TFLOP), so a visitor's first +command and headline number disagreed with every other surface in the repo. + +These tests re-derive the page's claims from the CLI default itself, so the +landing page cannot silently drift away from the shipped default again. +Pure parsing + arithmetic; no GPU needed. + +Run: python tests/test_landing_page_reference_regime.py +""" +import os +import re +import sys + +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +from strategy.cli import build_parser as strategy_parser +from matmul.cli import build_parser as matmul_parser + +_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +_PAGE = os.path.join(_ROOT, "index", "index.html") + + +def _page() -> str: + with open(_PAGE, encoding="utf-8") as fh: + return fh.read() + + +def _default_n() -> int: + n = strategy_parser().parse_args([]).n + assert n == matmul_parser().parse_args([]).n, "engines disagree on default --n" + return n + + +def test_quickstart_command_uses_the_shipped_default_n(): + """The copy-pasteable `python -m eval --n ...` on the page must be the default.""" + shown = re.findall(r"python -m eval --n (\d+)", _page()) + assert shown, "no `python -m eval --n ` quickstart found on the landing page" + for value in shown: + assert int(value) == _default_n() + + +def test_reference_regime_card_matches_the_default_n(): + """The 'CCO's reference regime' KPI card must name the same N.""" + m = re.search(r"N = ([\d,]+)\s*—|N = ([\d,]+)\s*—\s*CCO's reference regime", _page()) + assert m, "no 'CCO's reference regime' card found" + shown = int((m.group(1) or m.group(2)).replace(",", "")) + assert shown == _default_n() + + +def test_reference_regime_flop_kpi_is_2n3_at_the_default_n(): + """The KPI beside the reference regime is 2*N**3; check it in decimal units + (the same convention the 1,000 -> 2 GFLOP and 128,000 -> 4.2 PFLOP cards use).""" + n = _default_n() + expected_tflop = round(2.0 * n**3 / 1e12, 1) + assert expected_tflop == 1.1, f"sanity: 2*{n}**3 should be 1.1 TFLOP, got {expected_tflop}" + assert f"{expected_tflop} TFLOP" in _page() + # the pre-8192 figure (2*12000**3) must be gone + assert "3.5 TFLOP" not in _page() + + +def test_no_stale_12000_reference_remains(): + page = _page() + assert "12000" not in page and "12,000" not in page + + +if __name__ == "__main__": + fns = [v for k, v in sorted(globals().items()) if k.startswith("test_")] + failed = 0 + for fn in fns: + try: + fn() + print(f"PASS {fn.__name__}") + except AssertionError as e: + failed += 1 + print(f"FAIL {fn.__name__}: {e}") + print(f"\n{len(fns) - failed}/{len(fns)} passed") + sys.exit(1 if failed else 0)