Skip to content

Commit e9f427c

Browse files
arekay-nvclaude
andauthored
feat(config): register MLPerf Endpoints v1.0 seed set as a ruleset round (#508)
* feat(config): register MLPerf Endpoints v1.0 seed set as a ruleset round Adds the published Endpoints v1.0 seed set (cohort 2026-10-C1, set A) as the round `endpoints-v1.0-2026-10-C1-A`, selectable via `submission_ref.ruleset` as `mlperf-endpoints-v1.0-2026-10-C1-A` or the `mlperf-endpoints-current` alias. The two seeds are transcribed verbatim from `seedset.yaml` in mlcommons/endpoints_policies at commit 5279b845 and reach the runtime config through the existing `_apply_ruleset_seed_overrides` path, so no config schema changes are needed. Endpoints rotates independently of the MLPerf Inference rounds: MLCommons publishes a seed set every two publication cohorts and a submission keeps its bound set for its full Pareto-update window. `ENDPOINTS_ALL` therefore registers every published set rather than replacing the previous one, and the version string carries both the cohort ID and the set ID because `seed_sets` is a list keyed by `id` and a cohort may publish more than one set. The cohort's third seed, `model_seed`, is deliberately not pinned. Its purpose is undocumented upstream: the string appears in neither `endpoints_rules.md` nor `endpoints_submission_rules.md` at that commit, and submission rules §4.6 enumerates three RNG purposes (request-issue, sample order, and the per-query salt) of which this round maps two. Recorded as a comment pending a definition from MLCommons. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com> * fix(config): fail loudly on an unregistered submission_ref ruleset An unregistered ruleset previously raised only for `type: submission`; every other test type logged a warning and carried on with the default seeds 42/42. That makes a typo invisible in exactly the case it matters: the run reports itself as bound to a ruleset, dumps a config.yaml naming it, and issues load from seeds nobody chose. Naming a `submission_ref.ruleset` is a declaration that the run is bound to that ruleset's seeds, so an unresolvable name is now an error for every test type. The message lists the registered rulesets, because the likely mistake is a near-miss on a cohort-qualified name — `mlperf-endpoints-v1.0-2026-10-C1` against the registered `...-C1-A`, say, which is hard to spot unaided. `list_rulesets` is imported inside the handler rather than at module scope: `ruleset_registry` imports the rulesets package, which imports this module. This is the same deferral `SubmissionReference.get_ruleset_instance` already uses for `get_ruleset`. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com> --------- Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
1 parent 48c1f8e commit e9f427c

6 files changed

Lines changed: 252 additions & 32 deletions

File tree

‎src/inference_endpoint/config/ruleset_registry.py‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,8 @@
3131
from .rulesets.mlcommons.rules import ALL_ROUNDS as mlcommons_rounds
3232
from .rulesets.mlcommons.rules import CURRENT as mlcommons_current
3333
from .rulesets.mlcommons.rules import EDGE_CURRENT as mlcommons_edge_current
34+
from .rulesets.mlcommons.rules import ENDPOINTS_ALL as mlcommons_endpoints_all
35+
from .rulesets.mlcommons.rules import ENDPOINTS_CURRENT as mlcommons_endpoints_current
3436

3537
if TYPE_CHECKING:
3638
from .ruleset_base import BenchmarkSuiteRuleset
@@ -105,6 +107,14 @@ def _auto_register_mlcommons():
105107
f"mlperf-{mlcommons_edge_current.version}", mlcommons_edge_current
106108
) # -> "mlperf-edge-v0.1"
107109
_RULESET_REGISTRY.setdefault("mlperf-edge-current", mlcommons_edge_current)
110+
# Endpoints seed sets: by cohort-qualified version and as the endpoints
111+
# "current". A later cohort registers alongside rather than replacing, so a
112+
# submission bound to an earlier set stays resolvable for its full window.
113+
for ruleset in mlcommons_endpoints_all:
114+
_RULESET_REGISTRY.setdefault(f"mlperf-{ruleset.version}", ruleset)
115+
_RULESET_REGISTRY.setdefault(
116+
"mlperf-endpoints-current", mlcommons_endpoints_current
117+
)
108118

109119

110120
# Auto-register on import

‎src/inference_endpoint/config/rulesets/mlcommons/__init__.py‎

Lines changed: 14 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,18 @@
1515

1616
"""MLCommons benchmark ruleset implementation."""
1717

18-
from .rules import CURRENT, EDGE_CURRENT, OptimizationPriority, RoundRuleset
18+
from .rules import (
19+
CURRENT,
20+
EDGE_CURRENT,
21+
ENDPOINTS_CURRENT,
22+
OptimizationPriority,
23+
RoundRuleset,
24+
)
1925

20-
__all__ = ["CURRENT", "EDGE_CURRENT", "OptimizationPriority", "RoundRuleset"]
26+
__all__ = [
27+
"CURRENT",
28+
"EDGE_CURRENT",
29+
"ENDPOINTS_CURRENT",
30+
"OptimizationPriority",
31+
"RoundRuleset",
32+
]

‎src/inference_endpoint/config/rulesets/mlcommons/rules.py‎

Lines changed: 45 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -13,10 +13,13 @@
1313
# See the License for the specific language governing permissions and
1414
# limitations under the License.
1515

16-
"""This module contains a code representation of the rules for the current round of MLPerf Inference.
16+
"""Code representation of the MLCommons benchmark rounds and their pinned seeds.
1717
18-
These values are derived directly from the MLPerf Inference Policies document:
18+
The MLPerf Inference rounds derive from the MLPerf Inference Policies document:
1919
https://github.com/mlcommons/inference_policies/blob/master/inference_rules.adoc
20+
21+
The MLPerf Endpoints seed sets come from a separate repository on an
22+
independent rotation; each block cites its own pinned upstream source.
2023
"""
2124

2225
import copy
@@ -330,3 +333,43 @@ def apply_user_config(
330333
)
331334

332335
EDGE_CURRENT = _edge_v0_1
336+
337+
338+
# --- MLPerf Endpoints v1.0 seed set ---
339+
#
340+
# Endpoints publishes its own seeds on its own rotation, independent of the
341+
# MLPerf Inference rounds above: a seed set per publication cohort, refreshed
342+
# every two cohorts, each set adoptable for four. A submission binds to one set
343+
# and keeps it for its full Pareto-update window. Values are transcribed
344+
# verbatim from cohort 2026-10-C1, set A, pinned to a specific upstream commit
345+
# so the transcription stays re-verifiable (the branch itself is mutable):
346+
# https://github.com/mlcommons/endpoints_policies/blob/5279b845b8492b02742a66d31cee09d9512f7e1d/seedset.yaml
347+
#
348+
# The cohort ID and set ID are both carried in the version string: `seed_sets`
349+
# is a list keyed by `id`, so a cohort may publish more than one set, and the
350+
# version is the registry key. A later cohort — or a second set within this
351+
# cohort — adds a sibling round rather than editing this one. Endpoints
352+
# characterizes a Pareto curve rather than gating per-model TTFT/TPOT, so there
353+
# are no per-model rulesets to declare: this round exists to pin seeds, and the
354+
# legacy per-model `apply_user_config` path correctly refuses it.
355+
#
356+
# The set also publishes a third seed, `model_seed: 9315206023656308754`, which
357+
# is not pinned here. Its purpose is undocumented: `model_seed` appears nowhere
358+
# in endpoints_rules.md or endpoints_submission_rules.md at the commit above —
359+
# only in seedset.yaml, which gives no definition. Submission rules §4.6 says
360+
# the set drives "request-issue / sample order, and the per-query salt"; this
361+
# round maps the first two, leaving the salt RNG unaccounted for. Pinning the
362+
# third seed awaits a definition from MLCommons of what it seeds.
363+
_endpoints_v1_0_2026_10_C1_A = RoundRuleset(
364+
version="endpoints-v1.0-2026-10-C1-A",
365+
scheduler_rng_seed=10487924139932647040,
366+
sample_index_rng_seed=586478644936801402,
367+
benchmark_rulesets={},
368+
)
369+
370+
# Every published cohort stays registered: a submission binds to one set for
371+
# its full update window, so an older set must keep resolving after a newer
372+
# one is published. Add new sets here, do not replace.
373+
ENDPOINTS_ALL = [_endpoints_v1_0_2026_10_C1_A]
374+
375+
ENDPOINTS_CURRENT = _endpoints_v1_0_2026_10_C1_A

‎src/inference_endpoint/config/schema.py‎

Lines changed: 18 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -1315,10 +1315,11 @@ def _apply_ruleset_seed_overrides(self) -> None:
13151315
13161316
MLPerf rounds pin the RNG seeds; this mirrors LoadGen locking the core
13171317
seeds from ``user.conf`` (a submitter cannot substitute their own).
1318-
If ``submission_ref`` is unset, the config is left unchanged. If it
1319-
names an unregistered ruleset, a ``type=SUBMISSION`` config errors (a
1320-
submission cannot silently fall back to default seeds), while any other
1321-
type is left unchanged so non-submission/placeholder configs still work.
1318+
If ``submission_ref`` is unset, the config is left unchanged. Naming an
1319+
unregistered ruleset is always an error, for every test type: declaring
1320+
a ``submission_ref`` asserts the run is bound to that ruleset's seeds,
1321+
so a typo that silently fell back to the defaults would produce a run
1322+
that looks bound while issuing load from seed 42.
13221323
13231324
The warmup phase is reseeded from the sample-index (dataloader) seed so
13241325
its sample order derives from the same pinned seed as the perf phase.
@@ -1330,18 +1331,19 @@ def _apply_ruleset_seed_overrides(self) -> None:
13301331
try:
13311332
ruleset = self.submission_ref.get_ruleset_instance()
13321333
except KeyError as e:
1333-
if self.type == TestType.SUBMISSION:
1334-
raise ValueError(
1335-
f"submission_ref.ruleset {self.submission_ref.ruleset!r} is not "
1336-
"registered; a submission must pin official RNG seeds and cannot "
1337-
"fall back to defaults."
1338-
) from e
1339-
logger.warning(
1340-
"submission_ref.ruleset %r is not registered; skipping ruleset "
1341-
"seed overrides.",
1342-
self.submission_ref.ruleset,
1343-
)
1344-
return
1334+
# Imported here, not at module scope: ruleset_registry imports the
1335+
# rulesets package, which imports this module (same reason
1336+
# SubmissionReference.get_ruleset_instance defers it).
1337+
from .ruleset_registry import list_rulesets
1338+
1339+
# The likely mistake is a near-miss on a cohort-qualified name, so
1340+
# name the alternatives rather than only the rejected value.
1341+
raise ValueError(
1342+
f"submission_ref.ruleset {self.submission_ref.ruleset!r} is not "
1343+
"registered, so the run cannot be bound to its RNG seeds and "
1344+
"would silently fall back to the defaults. Registered rulesets: "
1345+
+ ", ".join(sorted(list_rulesets()))
1346+
) from e
13451347

13461348
# A ruleset used as a submission_ref must pin both seeds. ``None`` means
13471349
# "unseeded" in the general ruleset contract (ruleset_base.py), but an

‎tests/unit/config/rulesets/mlcommons/test_rules.py‎

Lines changed: 119 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,13 +20,23 @@
2020
list_rulesets,
2121
register_ruleset,
2222
)
23+
from inference_endpoint.config.rulesets.mlcommons import (
24+
ENDPOINTS_CURRENT as package_endpoints_current,
25+
)
2326
from inference_endpoint.config.rulesets.mlcommons import datasets, models
2427
from inference_endpoint.config.rulesets.mlcommons.rules import (
2528
ALL_ROUNDS,
2629
CURRENT,
2730
EDGE_CURRENT,
31+
ENDPOINTS_ALL,
32+
ENDPOINTS_CURRENT,
2833
OptimizationPriority,
2934
)
35+
from inference_endpoint.config.schema import (
36+
BenchmarkConfig,
37+
SubmissionReference,
38+
TestType,
39+
)
3040
from inference_endpoint.config.user_config import UserConfig
3141

3242

@@ -233,3 +243,112 @@ def test_edge_ruleset_apply_user_config():
233243
assert rt_settings.min_issue_duration_ms == 0
234244
assert rt_settings.max_issue_duration_ms == 4 * 60 * 60 * 1000
235245
assert rt_settings.n_samples_from_dataset == 995
246+
247+
248+
# Verbatim from mlcommons/endpoints_policies seedset.yaml, cohort 2026-10-C1 set A.
249+
_EP_SCHED_SEED = 10487924139932647040
250+
_EP_SAMPLE_SEED = 586478644936801402
251+
252+
253+
@pytest.mark.unit
254+
def test_endpoints_v1_0_official_seeds():
255+
"""Seeds are the published Endpoints v1.0 cohort 2026-10-C1 set A values.
256+
257+
A drift here means a run would issue load from seeds MLCommons never
258+
published, which the reviewer-side seeded-RNG check would reject.
259+
"""
260+
ep = get_ruleset("mlperf-endpoints-v1.0-2026-10-C1-A")
261+
assert ep.scheduler_rng_seed == _EP_SCHED_SEED
262+
assert ep.sample_index_rng_seed == _EP_SAMPLE_SEED
263+
264+
265+
@pytest.mark.unit
266+
def test_endpoints_ruleset_registered():
267+
assert get_ruleset("mlperf-endpoints-v1.0-2026-10-C1-A") is ENDPOINTS_CURRENT
268+
assert get_ruleset("mlperf-endpoints-current") is ENDPOINTS_CURRENT
269+
assert "mlperf-endpoints-v1.0-2026-10-C1-A" in list_rulesets()
270+
assert ENDPOINTS_CURRENT.version == "endpoints-v1.0-2026-10-C1-A"
271+
272+
273+
@pytest.mark.unit
274+
def test_endpoints_seeds_differ_from_every_other_registered_ruleset():
275+
"""Endpoints cohorts rotate independently of the other rulesets; a shared
276+
value would mean one of the two was transcribed from the wrong source."""
277+
for other in [*ALL_ROUNDS, EDGE_CURRENT]:
278+
assert ENDPOINTS_CURRENT.scheduler_rng_seed != other.scheduler_rng_seed
279+
assert ENDPOINTS_CURRENT.sample_index_rng_seed != other.sample_index_rng_seed
280+
281+
282+
@pytest.mark.unit
283+
def test_every_published_endpoints_cohort_stays_registered():
284+
"""A submission keeps its bound seed set for its full update window, so
285+
publishing a newer cohort must not unregister an older one."""
286+
names = list_rulesets()
287+
for ruleset in ENDPOINTS_ALL:
288+
assert f"mlperf-{ruleset.version}" in names
289+
assert ENDPOINTS_CURRENT in ENDPOINTS_ALL
290+
291+
292+
@pytest.mark.unit
293+
def test_endpoints_cohort_versions_are_unique():
294+
"""Duplicate versions would make the registry silently drop a cohort."""
295+
assert len(ENDPOINTS_ALL) == len({r.version for r in ENDPOINTS_ALL})
296+
297+
298+
@pytest.mark.unit
299+
def test_endpoints_round_refuses_the_per_model_config_path():
300+
"""The round declares no per-model rules, so the legacy per-model path must
301+
refuse it rather than emit runtime settings with no rules behind them."""
302+
with pytest.raises(ValueError, match="not found in rules"):
303+
ENDPOINTS_CURRENT.apply_user_config(
304+
model=models.Llama3_1_8b, user_config=UserConfig(1.0)
305+
)
306+
307+
308+
@pytest.mark.unit
309+
def test_endpoints_current_is_re_exported_from_the_package():
310+
assert package_endpoints_current is ENDPOINTS_CURRENT
311+
312+
313+
@pytest.mark.unit
314+
@pytest.mark.parametrize(
315+
"ruleset_name",
316+
["mlperf-endpoints-v1.0-2026-10-C1-A", "mlperf-endpoints-current"],
317+
)
318+
def test_binding_the_round_pins_the_published_seeds_on_a_config(ruleset_name):
319+
"""Closes the loop through the only consumer that matters: the seeds must
320+
survive pydantic revalidation in _apply_ruleset_seed_overrides and land on
321+
the runtime config. scheduler_rng_seed exceeds int64, so a future bound on
322+
that field would break this round while every other test stayed green.
323+
"""
324+
cfg = BenchmarkConfig(
325+
type=TestType.OFFLINE,
326+
model_params={"name": "test-model"},
327+
endpoint_config={"endpoints": ["http://localhost:8000"]},
328+
datasets=[{"path": "perf.jsonl"}],
329+
submission_ref=SubmissionReference(model="test-model", ruleset=ruleset_name),
330+
)
331+
assert cfg.settings.runtime.scheduler_random_seed == _EP_SCHED_SEED
332+
assert cfg.settings.runtime.dataloader_random_seed == _EP_SAMPLE_SEED
333+
# Warmup derives its sample order from the same pinned seed as the perf phase.
334+
assert cfg.settings.warmup.warmup_random_seed == _EP_SAMPLE_SEED
335+
336+
337+
@pytest.mark.unit
338+
def test_pinned_seeds_survive_a_yaml_round_trip(tmp_path):
339+
"""config.yaml in the report dir is the reproducibility record, so it must
340+
carry the pinned values rather than the pre-resolution defaults."""
341+
cfg = BenchmarkConfig(
342+
type=TestType.OFFLINE,
343+
model_params={"name": "test-model"},
344+
endpoint_config={"endpoints": ["http://localhost:8000"]},
345+
datasets=[{"path": "perf.jsonl"}],
346+
submission_ref=SubmissionReference(
347+
model="test-model", ruleset="mlperf-endpoints-current"
348+
),
349+
)
350+
out = tmp_path / "config.yaml"
351+
cfg.to_yaml_file(out)
352+
reloaded = BenchmarkConfig.from_yaml_file(out)
353+
assert reloaded.settings.runtime.scheduler_random_seed == _EP_SCHED_SEED
354+
assert reloaded.settings.runtime.dataloader_random_seed == _EP_SAMPLE_SEED

‎tests/unit/config/test_schema.py‎

Lines changed: 46 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -1081,18 +1081,52 @@ def test_unregistered_ruleset_submission_raises(self):
10811081
self._submission("does-not-exist")
10821082

10831083
@pytest.mark.unit
1084-
def test_unregistered_ruleset_non_submission_is_lenient(self):
1085-
"""Non-submission configs are unaffected: an unknown ruleset leaves the
1086-
runtime seeds at their defaults rather than erroring."""
1087-
cfg = BenchmarkConfig(
1088-
type=TestType.OFFLINE,
1089-
model_params={"name": "test"},
1090-
endpoint_config={"endpoints": ["http://localhost:8000"]},
1091-
datasets=[{"path": "test.jsonl"}],
1092-
submission_ref=SubmissionReference(model="test", ruleset="does-not-exist"),
1093-
)
1094-
assert cfg.settings.runtime.scheduler_random_seed == 42
1095-
assert cfg.settings.runtime.dataloader_random_seed == 42
1084+
@pytest.mark.parametrize(
1085+
("test_type", "settings"),
1086+
[
1087+
(TestType.OFFLINE, {}),
1088+
(
1089+
TestType.ONLINE,
1090+
{
1091+
"load_pattern": {
1092+
"type": LoadPatternType.CONCURRENCY,
1093+
"target_concurrency": 1,
1094+
}
1095+
},
1096+
),
1097+
],
1098+
)
1099+
def test_unregistered_ruleset_raises_for_any_type(self, test_type, settings):
1100+
"""Naming a ruleset is a declaration that the run is bound to its seeds.
1101+
A typo must fail loudly rather than silently running on the defaults —
1102+
the run would otherwise look bound while issuing load from seed 42.
1103+
"""
1104+
with pytest.raises(ValidationError, match="not registered"):
1105+
BenchmarkConfig(
1106+
type=test_type,
1107+
model_params={"name": "test"},
1108+
endpoint_config={"endpoints": ["http://localhost:8000"]},
1109+
datasets=[{"path": "test.jsonl"}],
1110+
settings=settings,
1111+
submission_ref=SubmissionReference(
1112+
model="test", ruleset="does-not-exist"
1113+
),
1114+
)
1115+
1116+
@pytest.mark.unit
1117+
def test_unregistered_ruleset_error_names_the_available_rulesets(self):
1118+
"""The failure has to be actionable: a near-miss on a cohort-qualified
1119+
name is the likely mistake, so the message lists what is registered."""
1120+
with pytest.raises(ValidationError, match="mlperf-endpoints-current"):
1121+
BenchmarkConfig(
1122+
type=TestType.OFFLINE,
1123+
model_params={"name": "test"},
1124+
endpoint_config={"endpoints": ["http://localhost:8000"]},
1125+
datasets=[{"path": "test.jsonl"}],
1126+
submission_ref=SubmissionReference(
1127+
model="test", ruleset="mlperf-endpoints-v1.0-2026-10-C1"
1128+
),
1129+
)
10961130

10971131
@pytest.mark.unit
10981132
def test_no_submission_ref_keeps_defaults(self):

0 commit comments

Comments
 (0)