|
20 | 20 | list_rulesets, |
21 | 21 | register_ruleset, |
22 | 22 | ) |
| 23 | +from inference_endpoint.config.rulesets.mlcommons import ( |
| 24 | + ENDPOINTS_CURRENT as package_endpoints_current, |
| 25 | +) |
23 | 26 | from inference_endpoint.config.rulesets.mlcommons import datasets, models |
24 | 27 | from inference_endpoint.config.rulesets.mlcommons.rules import ( |
25 | 28 | ALL_ROUNDS, |
26 | 29 | CURRENT, |
27 | 30 | EDGE_CURRENT, |
| 31 | + ENDPOINTS_ALL, |
| 32 | + ENDPOINTS_CURRENT, |
28 | 33 | OptimizationPriority, |
29 | 34 | ) |
| 35 | +from inference_endpoint.config.schema import ( |
| 36 | + BenchmarkConfig, |
| 37 | + SubmissionReference, |
| 38 | + TestType, |
| 39 | +) |
30 | 40 | from inference_endpoint.config.user_config import UserConfig |
31 | 41 |
|
32 | 42 |
|
@@ -233,3 +243,112 @@ def test_edge_ruleset_apply_user_config(): |
233 | 243 | assert rt_settings.min_issue_duration_ms == 0 |
234 | 244 | assert rt_settings.max_issue_duration_ms == 4 * 60 * 60 * 1000 |
235 | 245 | assert rt_settings.n_samples_from_dataset == 995 |
| 246 | + |
| 247 | + |
| 248 | +# Verbatim from mlcommons/endpoints_policies seedset.yaml, cohort 2026-10-C1 set A. |
| 249 | +_EP_SCHED_SEED = 10487924139932647040 |
| 250 | +_EP_SAMPLE_SEED = 586478644936801402 |
| 251 | + |
| 252 | + |
| 253 | +@pytest.mark.unit |
| 254 | +def test_endpoints_v1_0_official_seeds(): |
| 255 | + """Seeds are the published Endpoints v1.0 cohort 2026-10-C1 set A values. |
| 256 | +
|
| 257 | + A drift here means a run would issue load from seeds MLCommons never |
| 258 | + published, which the reviewer-side seeded-RNG check would reject. |
| 259 | + """ |
| 260 | + ep = get_ruleset("mlperf-endpoints-v1.0-2026-10-C1-A") |
| 261 | + assert ep.scheduler_rng_seed == _EP_SCHED_SEED |
| 262 | + assert ep.sample_index_rng_seed == _EP_SAMPLE_SEED |
| 263 | + |
| 264 | + |
| 265 | +@pytest.mark.unit |
| 266 | +def test_endpoints_ruleset_registered(): |
| 267 | + assert get_ruleset("mlperf-endpoints-v1.0-2026-10-C1-A") is ENDPOINTS_CURRENT |
| 268 | + assert get_ruleset("mlperf-endpoints-current") is ENDPOINTS_CURRENT |
| 269 | + assert "mlperf-endpoints-v1.0-2026-10-C1-A" in list_rulesets() |
| 270 | + assert ENDPOINTS_CURRENT.version == "endpoints-v1.0-2026-10-C1-A" |
| 271 | + |
| 272 | + |
| 273 | +@pytest.mark.unit |
| 274 | +def test_endpoints_seeds_differ_from_every_other_registered_ruleset(): |
| 275 | + """Endpoints cohorts rotate independently of the other rulesets; a shared |
| 276 | + value would mean one of the two was transcribed from the wrong source.""" |
| 277 | + for other in [*ALL_ROUNDS, EDGE_CURRENT]: |
| 278 | + assert ENDPOINTS_CURRENT.scheduler_rng_seed != other.scheduler_rng_seed |
| 279 | + assert ENDPOINTS_CURRENT.sample_index_rng_seed != other.sample_index_rng_seed |
| 280 | + |
| 281 | + |
| 282 | +@pytest.mark.unit |
| 283 | +def test_every_published_endpoints_cohort_stays_registered(): |
| 284 | + """A submission keeps its bound seed set for its full update window, so |
| 285 | + publishing a newer cohort must not unregister an older one.""" |
| 286 | + names = list_rulesets() |
| 287 | + for ruleset in ENDPOINTS_ALL: |
| 288 | + assert f"mlperf-{ruleset.version}" in names |
| 289 | + assert ENDPOINTS_CURRENT in ENDPOINTS_ALL |
| 290 | + |
| 291 | + |
| 292 | +@pytest.mark.unit |
| 293 | +def test_endpoints_cohort_versions_are_unique(): |
| 294 | + """Duplicate versions would make the registry silently drop a cohort.""" |
| 295 | + assert len(ENDPOINTS_ALL) == len({r.version for r in ENDPOINTS_ALL}) |
| 296 | + |
| 297 | + |
| 298 | +@pytest.mark.unit |
| 299 | +def test_endpoints_round_refuses_the_per_model_config_path(): |
| 300 | + """The round declares no per-model rules, so the legacy per-model path must |
| 301 | + refuse it rather than emit runtime settings with no rules behind them.""" |
| 302 | + with pytest.raises(ValueError, match="not found in rules"): |
| 303 | + ENDPOINTS_CURRENT.apply_user_config( |
| 304 | + model=models.Llama3_1_8b, user_config=UserConfig(1.0) |
| 305 | + ) |
| 306 | + |
| 307 | + |
| 308 | +@pytest.mark.unit |
| 309 | +def test_endpoints_current_is_re_exported_from_the_package(): |
| 310 | + assert package_endpoints_current is ENDPOINTS_CURRENT |
| 311 | + |
| 312 | + |
| 313 | +@pytest.mark.unit |
| 314 | +@pytest.mark.parametrize( |
| 315 | + "ruleset_name", |
| 316 | + ["mlperf-endpoints-v1.0-2026-10-C1-A", "mlperf-endpoints-current"], |
| 317 | +) |
| 318 | +def test_binding_the_round_pins_the_published_seeds_on_a_config(ruleset_name): |
| 319 | + """Closes the loop through the only consumer that matters: the seeds must |
| 320 | + survive pydantic revalidation in _apply_ruleset_seed_overrides and land on |
| 321 | + the runtime config. scheduler_rng_seed exceeds int64, so a future bound on |
| 322 | + that field would break this round while every other test stayed green. |
| 323 | + """ |
| 324 | + cfg = BenchmarkConfig( |
| 325 | + type=TestType.OFFLINE, |
| 326 | + model_params={"name": "test-model"}, |
| 327 | + endpoint_config={"endpoints": ["http://localhost:8000"]}, |
| 328 | + datasets=[{"path": "perf.jsonl"}], |
| 329 | + submission_ref=SubmissionReference(model="test-model", ruleset=ruleset_name), |
| 330 | + ) |
| 331 | + assert cfg.settings.runtime.scheduler_random_seed == _EP_SCHED_SEED |
| 332 | + assert cfg.settings.runtime.dataloader_random_seed == _EP_SAMPLE_SEED |
| 333 | + # Warmup derives its sample order from the same pinned seed as the perf phase. |
| 334 | + assert cfg.settings.warmup.warmup_random_seed == _EP_SAMPLE_SEED |
| 335 | + |
| 336 | + |
| 337 | +@pytest.mark.unit |
| 338 | +def test_pinned_seeds_survive_a_yaml_round_trip(tmp_path): |
| 339 | + """config.yaml in the report dir is the reproducibility record, so it must |
| 340 | + carry the pinned values rather than the pre-resolution defaults.""" |
| 341 | + cfg = BenchmarkConfig( |
| 342 | + type=TestType.OFFLINE, |
| 343 | + model_params={"name": "test-model"}, |
| 344 | + endpoint_config={"endpoints": ["http://localhost:8000"]}, |
| 345 | + datasets=[{"path": "perf.jsonl"}], |
| 346 | + submission_ref=SubmissionReference( |
| 347 | + model="test-model", ruleset="mlperf-endpoints-current" |
| 348 | + ), |
| 349 | + ) |
| 350 | + out = tmp_path / "config.yaml" |
| 351 | + cfg.to_yaml_file(out) |
| 352 | + reloaded = BenchmarkConfig.from_yaml_file(out) |
| 353 | + assert reloaded.settings.runtime.scheduler_random_seed == _EP_SCHED_SEED |
| 354 | + assert reloaded.settings.runtime.dataloader_random_seed == _EP_SAMPLE_SEED |
0 commit comments